Merge pull request #3487 from facebook/dev

release v1.5.4

.circleci/config.yml

@@ -0,0 +1,123 @@
+version: 2
+
+jobs:
+  # the first half of the jobs are in this test
+  short-tests-0:
+    # TODO: Create a small custom docker image with all the dependencies we need
+    #       preinstalled to reduce installation time.
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    steps:
+      - checkout
+      - run:
+          name: Test
+          command: |
+            ./tests/test-license.py
+            cc -v
+            CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
+            make c99build; make clean
+            make c11build; make clean
+            make -j regressiontest; make clean
+            make shortest; make clean
+            make cxxtest; make clean
+  # the second half of the jobs are in this test
+  short-tests-1:
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    steps:
+      - checkout
+      - run:
+          name: Test
+          command: |
+            make gnu90build; make clean
+            make gnu99build; make clean
+            make ppc64build V=1; make clean
+            make ppcbuild   V=1; make clean
+            make armbuild   V=1; make clean
+            make aarch64build V=1; make clean
+            make -C tests test-legacy test-longmatch; make clean
+            make -C lib libzstd-nomt; make clean
+  # This step should only be run in a cron job
+  regression-test:
+    docker:
+      - image: fbopensource/zstd-circleci-primary:0.0.1
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    steps:
+      - checkout
+      # Restore the cached resources.
+      - restore_cache:
+          # We try our best to bust the cache when the data changes by hashing
+          # data.c. If that doesn't work, simply update the version number here
+          # and below. If we fail to bust the cache, the regression testing will
+          # still work, since it has its own stamp, but will need to redownload
+          # everything.
+          keys:
+            - regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+      - run:
+          name: Regression Test
+          command: |
+            make -C programs zstd
+            make -C tests/regression test
+            mkdir -p $CIRCLE_ARTIFACTS
+            ./tests/regression/test                     \
+                --cache  tests/regression/cache         \
+                --output $CIRCLE_ARTIFACTS/results.csv  \
+                --zstd   programs/zstd
+            echo "NOTE: The new results.csv is uploaded as an artifact to this job"
+            echo "      If this fails, go to the Artifacts pane in CircleCI, "
+            echo "      download /tmp/circleci-artifacts/results.csv, and if they "
+            echo "      are still good, copy it into the repo and commit it."
+            echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
+            diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
+      # Only save the cache on success (default), since if the failure happened
+      # before we stamp the data cache, we will have a bad cache for this key.
+      - save_cache:
+          key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
+          paths:
+            - tests/regression/cache
+      - store_artifacts:
+          path: /tmp/circleci-artifacts
+
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      # Run the tests in parallel
+      - short-tests-0
+      - short-tests-1
+      - regression-test
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - release
+                - dev
+                - master
+    jobs:
+      # Run daily regression tests
+      - regression-test
+
+
+
+  # Longer tests
+    #- make -C tests test-zstd-nolegacy && make clean
+    #- pyenv global 3.4.4; make -C tests versionsTest && make clean
+    #- make zlibwrapper         && make clean
+    #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
+    #- make uasan               && make clean
+    #- make asan32              && make clean
+    #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
+  # Valgrind tests
+    #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
+    #- make -C tests valgrindTest && make clean
+  # ARM, AArch64, PowerPC, PowerPC64 tests
+    #- make ppctest             && make clean
+    #- make ppc64test           && make clean
+    #- make armtest             && make clean
+    #- make aarch64test         && make clean

.circleci/images/primary/Dockerfile

@@ -0,0 +1,9 @@
+FROM circleci/buildpack-deps:focal
+
+RUN sudo dpkg --add-architecture i386
+RUN sudo apt-get -y -qq update
+RUN sudo apt-get -y install \
+    gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
+    libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
+    libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
+    libcurl4-openssl-dev

.cirrus.yml

@@ -1,9 +1,10 @@
 task:
-  name: FreeBSD (make check)
+  name: FreeBSD (shortest)
   freebsd_instance:
     matrix:
-      image_family: freebsd-14-2
+      image_family: freebsd-13-0
+      image_family: freebsd-12-2
   install_script: pkg install -y gmake coreutils
   script: |
     MOREFLAGS="-Werror" gmake -j all
-    gmake check
+    gmake shortest

.github/workflows/android-ndk-build.yml

@@ -1,39 +0,0 @@
-name: Android NDK Build
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-  push:
-    branches: [ actionsTest, '*ndk*' ]
-
-permissions: read-all
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-
-    - name: Set up JDK 17
-      uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
-      with:
-        java-version: '17'
-        distribution: 'temurin'
-
-    - name: Setup Android SDK
-      uses: android-actions/setup-android@9fc6c4e9069bf8d3d10b2204b1fb8f6ef7065407 # v3.2.2
-
-    - name: Install Android NDK
-      run: |
-        sdkmanager --install "ndk;27.0.12077973"
-        echo "ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973" >> $GITHUB_ENV
-
-    - name: Build with NDK
-      run: |
-        export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
-        make CC=aarch64-linux-android21-clang \
-             AR=llvm-ar \
-             RANLIB=llvm-ranlib \
-             STRIP=llvm-strip
-

.github/workflows/cmake-tests.yml

@@ -1,153 +0,0 @@
-name: cmake-tests
-# CMake-specific build and test workflows
-# This workflow validates zstd builds across different CMake configurations,
-# platforms, and edge cases to ensure broad compatibility.
-
-concurrency:
-  group: cmake-${{ github.ref }}
-  cancel-in-progress: true
-
-on:
-  pull_request:
-    branches: [ dev, release, actionsTest ]
-
-permissions: read-all
-
-env:
-  # Centralized test timeouts for consistency
-  QUICK_TEST_TIME: "30s"
-  STANDARD_TEST_TIME: "1mn"
-  # Common CMake flags
-  COMMON_CMAKE_FLAGS: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
-
-jobs:
-  # Ubuntu-based cmake build using make wrapper
-  # This test uses the make-driven cmake build to ensure compatibility
-  # with the existing build system integration
-  cmake-ubuntu-basic:
-    name: "CMake Ubuntu Basic Build"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: Install dependencies
-      run: |
-        sudo apt install liblzma-dev  # Required for compression algorithms
-    - name: CMake build and test via make
-      run: |
-        # Use make wrapper for cmake build with quick test timeouts
-        FUZZERTEST=-T${{ env.STANDARD_TEST_TIME }} ZSTREAM_TESTTIME=-T${{ env.STANDARD_TEST_TIME }} make cmakebuild V=1
-
-  # Cross-platform cmake build with edge case: source paths containing spaces
-  # This test ensures cmake handles filesystem paths with spaces correctly
-  # across different operating systems and build generators
-  cmake-cross-platform-spaces:
-    name: "CMake Cross-Platform (Spaces in Path)"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            generator: "Unix Makefiles"
-            name: "Linux"
-          - os: windows-latest
-            generator: "NMake Makefiles"
-            name: "Windows NMake"
-          - os: macos-latest
-            generator: "Unix Makefiles"
-            name: "macOS"
-    env:
-      SRC_DIR: "source directory with spaces"
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-      with:
-        path: "${{ env.SRC_DIR }}"
-    - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
-      if: ${{ matrix.generator == 'NMake Makefiles' }}
-    - name: "CMake build and install (${{ matrix.name }})"
-      run: |
-        # Test Release build with installation to verify packaging
-        cmake -S "${{ env.SRC_DIR }}/build/cmake" -B build -DBUILD_TESTING=ON -G "${{ matrix.generator }}" -DCMAKE_BUILD_TYPE=Release --install-prefix "${{ runner.temp }}/install"
-        cmake --build build --config Release
-        cmake --install build --config Release
-
-  # Windows-specific cmake testing with Visual Studio 2022
-  # Tests multiple generators and toolchains to ensure broad Windows compatibility
-  # including MSVC (x64, Win32, ARM64), MinGW, and Clang-CL with various architectures and optimizations
-  cmake-windows-comprehensive:
-    name: "CMake Windows VS2022 (${{ matrix.name }})"
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - generator: "Visual Studio 17 2022"
-            flags: "-A x64"
-            name: "MSVC x64"
-            runner: "windows-2022"
-            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
-          - generator: "Visual Studio 17 2022"
-            flags: "-A Win32"
-            name: "MSVC Win32"
-            runner: "windows-2022"
-            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
-          - generator: "Visual Studio 17 2022"
-            flags: "-A x64"
-            name: "MSVC x64 (No ZSTD_BUILD_TESTS)"
-            runner: "windows-2022"
-            # Intentionally omit ZSTD_BUILD_TESTS to reproduce the CXX language configuration bug
-            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON"
-          # - generator: "Visual Studio 17 2022"
-          #   flags: "-A ARM64"
-          #   name: "MSVC ARM64"
-          #   runner: "windows-2022-arm64"  # Disabled due to very long queue times
-          - generator: "MinGW Makefiles"
-            flags: ""
-            name: "MinGW"
-            runner: "windows-2022"
-            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
-          - generator: "Visual Studio 17 2022"
-            flags: "-T ClangCL"
-            name: "Clang-CL"
-            runner: "windows-2022"
-            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
-          - generator: "Visual Studio 17 2022"
-            flags: "-T ClangCL -A x64 -DCMAKE_C_FLAGS=/arch:AVX2"
-            name: "Clang-CL AVX2"
-            runner: "windows-2022"
-            cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
-    - name: "Configure CMake (${{ matrix.name }})"
-      run: |
-        cd build\cmake
-        mkdir build
-        cd build
-        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_extra_flags }} -DZSTD_ZSTREAM_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FUZZER_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FULLBENCH_FLAGS=-i0 ..
-    - name: "Build (${{ matrix.name }})"
-      run: |
-        cd build\cmake\build
-        cmake.exe --build .
-    - name: "Test (${{ matrix.name }})"
-      run: |
-        cd build\cmake\build
-        ctest.exe -V -C Debug
-
-  # macOS ARM64 (Apple Silicon) specific cmake testing
-  # Validates zstd builds and runs correctly on Apple Silicon architecture
-  # Uses native ARM64 hardware for optimal performance and compatibility testing
-  cmake-macos-arm64:
-    name: "CMake macOS ARM64 (Apple Silicon)"
-    runs-on: macos-14  # ARM64 runner
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: "CMake build and test (ARM64)"
-      run: |
-        # Configure and build with ARM64-specific optimizations
-        cd build/cmake
-        mkdir build
-        cd build
-        cmake -DCMAKE_BUILD_TYPE=Release ${{ env.COMMON_CMAKE_FLAGS }} -DZSTD_ZSTREAM_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FUZZER_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FULLBENCH_FLAGS=-i1 ..
-        make -j$(sysctl -n hw.ncpu)
-        ctest -V

.github/workflows/commit.yml

@@ -1,104 +0,0 @@
-name: facebook/zstd/commit
-on:
-  push:
-    branches:
-    - dev
-  pull_request:
-    branches:
-    - dev
-
-permissions: read-all
-
-jobs:
-  short-tests-0:
-    runs-on: ubuntu-latest
-    services:
-      docker:
-        image: fbopensource/zstd-circleci-primary:0.0.1
-        options: --entrypoint /bin/bash
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install Dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install libcurl4-gnutls-dev
-    - name: Test
-      run: |
-        ./tests/test-license.py
-        cc -v
-        CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
-        make c99build; make clean
-        make c11build; make clean
-        make -j regressiontest; make clean
-        make check; make clean
-        make cxxtest; make clean
-
-  short-tests-1:
-    runs-on: ubuntu-latest
-    services:
-      docker:
-        image: fbopensource/zstd-circleci-primary:0.0.1
-        options: --entrypoint /bin/bash
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install Dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-13-dev-powerpc-cross
-    - name: gnu90 build
-      run: make gnu90build && make clean
-    - name: gnu99 build
-      run: make gnu99build && make clean
-    - name: ppc64 build
-      run: make ppc64build V=1 && make clean
-    - name: ppc build
-      run: make ppcbuild V=1 && make clean
-    - name: arm build
-      run: make armbuild V=1 && make clean
-    - name: aarch64 build
-      run: make aarch64build V=1 && make clean
-    - name: test-legacy
-      run: make -C tests test-legacy V=1 && make clean
-    - name: test-longmatch
-      run: make -C tests test-longmatch V=1 && make clean
-    - name: libzstd-nomt build
-      run: make -C lib libzstd-nomt V=1 && make clean
-
-  regression-test:
-    runs-on: ubuntu-latest
-    services:
-      docker:
-        image: fbopensource/zstd-circleci-primary:0.0.1
-        options: --entrypoint /bin/bash
-    env:
-      CIRCLE_ARTIFACTS: "/tmp/circleci-artifacts"
-    steps:
-    - uses: actions/checkout@v4
-    - name: restore_cache
-      uses: actions/cache@v4
-      with:
-        key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
-        path: tests/regression/cache
-        restore-keys: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
-    - name: Install Dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install libcurl4-gnutls-dev
-    - name: Regression Test
-      run: |
-        make -C programs zstd
-        make -C tests/regression test
-        mkdir -p $CIRCLE_ARTIFACTS
-        ./tests/regression/test                     \
-            --cache  tests/regression/cache         \
-            --output $CIRCLE_ARTIFACTS/results.csv  \
-            --zstd   programs/zstd
-        echo "NOTE: The new results.csv is uploaded as an artifact to this job"
-        echo "      If this fails, go to the Artifacts pane in CircleCI, "
-        echo "      download /tmp/circleci-artifacts/results.csv, and if they "
-        echo "      are still good, copy it into the repo and commit it."
-        echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
-        diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
-    - uses: actions/upload-artifact@v4
-      with:
-        path: "/tmp/circleci-artifacts"

.github/workflows/dev-long-tests.yml

@@ -1,5 +1,5 @@
 name: dev-long-tests
-# Tests generally longer than 10mn
+# Tests longer than 10mn
 
 concurrency:
   group: long-${{ github.ref }}
@@ -9,109 +9,78 @@ on:
   pull_request:
     branches: [ dev, release, actionsTest ]
 
-permissions: read-all
-
 jobs:
-  # lasts ~7mn
   make-all:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make all
       run: make all
 
-  # lasts ~19mn
+  # lasts ~24mn
   make-test:
     runs-on: ubuntu-latest
     env:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make test
-      run: |
-        make test
-        make -j zstd
-        ./tests/test_process_substitution.bash ./zstd
-
-  # lasts ~16mn
-  make-test-macos:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: make test on macos
       run: make test
 
-  # lasts ~10mn
-  make-test-32bit:
-    runs-on: ubuntu-latest
-    env:
-      DEVNULLRIGHTS: 1
-      READFROMBLOCKDEVICE: 1
+  # lasts ~26mn
+  make-test-osx:
+    runs-on: macos-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: make test # note: `make -j test success` seems to require a clean state
-      run: |
-        sudo apt-get -qqq update
-        make libc6install
-        make clean
-        CFLAGS="-m32 -O2" make -j test V=1
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: OS-X test
+      run: make test # make -c lib all doesn't work because of the fact that it's not a tty
 
-  # lasts ~7mn
-  test-largeDictionary:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: largeDictionary
-      run: |
-        CFLAGS="-Werror -O3" make -j -C tests test-largeDictionary
-
-  # lasts ~9mn
   no-intrinsics-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
-  # lasts ~8mn
   tsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
-  uasan-zstreamtest:
+  ubsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: ub + address sanitizer on zstreamtest
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: undefined behavior sanitizer zstreamtest
       run: CC=clang make uasan-test-zstream
 
-  # lasts ~11mn
+  # lasts ~15mn
   tsan-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
+
   big-tests-zstreamtest32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: zstream tests in 32bit mode, with big tests
       run: |
         sudo apt-get -qqq update
         make libc6install
         CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"
 
-  # lasts ~13mn
+  # lasts ~23mn
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
       # See https://askubuntu.com/a/1428822
       run: |
@@ -123,14 +92,14 @@ jobs:
   clang-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Test Zstd
       run: CC=clang make -j uasan-test-zstd </dev/null V=1
 
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
         sudo apt-get -qqq update
@@ -144,7 +113,7 @@ jobs:
   gcc-8-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
       # See https://askubuntu.com/a/1428822
       run: |
@@ -156,98 +125,76 @@ jobs:
   clang-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Fuzz Test
       run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
         make libc6install
         CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
-  clang-asan-fuzz32:
+  clang-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: clang + ASan + Fuzz Test 32bit
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: clang + ASan + UBSan + Fuzz Test 32bit
       run: |
         sudo apt-get -qqq update
         make libc6install
-        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make asan-fuzztest
-
-# The following test seems to have issues on github CI specifically,
-# it does not provide the `__mulodi4` instruction emulation
-# required for signed 64-bit multiplication.
-# Replaced by asan-only test (above)
-#
-#  clang-asan-ubsan-fuzz32:
-#    runs-on: ubuntu-20.04
-#    steps:
-#    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-#    - name: clang + ASan + UBSan + Fuzz Test 32bit
-#      run: |
-#        sudo apt-get -qqq update
-#        make libc6install
-#        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
-  clang-asan-ubsan-regression:
+  clang-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + ASan + UBSan + Regression Test
       run: CC=clang make -j uasanregressiontest
 
   msan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: MSan + Regression Test
       run: make -j msanregressiontest
 
-  clang-msan-fuzz-unoptimized:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-      - name: clang + MSan + Fuzz Test
-        run: |
-          sudo apt-get -qqq update
-          sudo apt-get install clang
-          CC=clang MOREFLAGS="-O0" make clean msan-fuzztest
-
   clang-msan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + MSan + Fuzz Test
       run: |
         sudo apt-get -qqq update
         sudo apt-get install clang
         CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
 
+  # lasts ~24mn
   clang-msan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + MSan + Test Zstd
       run: |
-        CC=clang make -j msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
+        sudo apt-get update
+        sudo apt-get install clang
+        CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
 
   armfuzz:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Qemu ARM emulation + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -257,7 +204,7 @@ jobs:
   valgrind-fuzz-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: valgrind + fuzz test stack mode    # ~ 7mn
       shell: 'script -q -e -c "bash {0}"'
       run: |
@@ -273,8 +220,8 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - uses: msys2/setup-msys2@40677d36a502eb2cf0fb808cc9dec31bf6152638 # tag=v2.28.0
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - uses: msys2/setup-msys2@v2
       with:
         msystem: MINGW64
         install: make
@@ -317,7 +264,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # tag=v4.3.1
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts

.github/workflows/dev-short-tests.yml

@@ -10,55 +10,56 @@ on:
   pull_request:
     branches: [ dev, release, actionsTest ]
 
-permissions: read-all
-
 jobs:
   linux-kernel:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: linux kernel, library + build + test
       run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
 
   benchmarking:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make benchmarking
       run: make benchmarking
 
   check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make check on 32-bit
       run: |
         sudo apt update
         APT_PACKAGES="gcc-multilib" make apt-install
         CFLAGS="-m32 -O1 -fstack-protector" make check V=1
-        CFLAGS="-m32 -O1 -fstack-protector" make V=1 -C tests test-cli-tests
+
+  check-x32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
+      env:
+        CHECK_CONSTRAINED_MEM: true
+      run: |
+        sudo apt update
+        APT_PACKAGES="gcc-multilib" make apt-install
+        CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
 
   build-c89:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: ensure zstd can be built with c89/c90 compilers (+ long long support + variadic macros)
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: ensure zstd can be build with c89/c90 compilers (+ long long support + variadic macros)
       run: |
         make c89build V=1
 
-  build-zstd-dll:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: build zstd bin against a dynamic lib (debuglevel for more dependencies)
-      run: |
-        make -C lib lib-mt-release
-        DEBUGLEVEL=2 make -C programs zstd-dll
 
   gcc-7-libzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-7 + libzstdmt compilation
       # See https://askubuntu.com/a/1428822
       run: |
@@ -69,13 +70,24 @@ jobs:
         make clean
         LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
 
-    # candidate test (for discussion) : underlink test
+    # candidate test (to check) : underlink test
     # LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
 
+  cmake-build-and-test-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: cmake build and test check
+      run: |
+        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
+        cp -r ./ "../zstd source"
+        cd "../zstd source"
+        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
+
   cpp-gnu90-c99-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: C++, gnu90 and c99 compatibility
       run: |
         make cxxtest
@@ -89,7 +101,7 @@ jobs:
   mingw-cross-compilation:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: mingw cross-compilation
       run: |
         # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -100,7 +112,7 @@ jobs:
   armbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ARM Build Test
       run: |
         sudo apt-get -qqq update
@@ -110,7 +122,7 @@ jobs:
   bourne-shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Bourne shell compatibility (shellcheck)
       run: |
         wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -120,22 +132,21 @@ jobs:
   zlib-wrapper:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: install valgrind
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: zlib wrapper test
       run: |
         sudo apt-get -qqq update
-        make valgrindinstall V=1
-    - name: zlib wrapper test
-      run: make -C zlibWrapper test V=1
-    - name: zlib wrapper test under valgrind
-      run: make -C zlibWrapper test-valgrind V=1
+        make valgrindinstall
+        make -C zlibWrapper test
+        make -C zlibWrapper test-valgrind
 
   lz4-threadpool-libs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: LZ4, thread pool, and libs build testslib wrapper test
       run: |
+        make lz4install
         make -C tests test-lz4
         make check < /dev/null | tee   # mess with lz4 console detection
         make clean
@@ -143,41 +154,20 @@ jobs:
         make clean
         bash tests/libzstd_builds.sh
 
-  gcc-make-all-avx2:
+  gcc-make-tests-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: Make all, with AVX2
-      run: |
-        sudo apt-get -qqq update
-        make libc6install
-        CFLAGS="-Werror -mavx2" make -j all
-
-  gcc-make-all-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Make all, 32bit mode
       run: |
         sudo apt-get -qqq update
         make libc6install
         CFLAGS="-Werror -m32" make -j all32
 
-  gcc-make-all-32bit-avx2:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: Make all, 32bit + AVX2 mode
-      run: |
-        sudo apt-get -qqq update
-        make libc6install
-        CPPFLAGS="-DSTATIC_BMI2=1" CFLAGS="-Werror -m32 -mavx2 -mbmi2" make -j all32
-
-
   gcc-8-make:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: gcc-8 build
         # See https://askubuntu.com/a/1428822
         run: |
@@ -186,31 +176,10 @@ jobs:
           make gcc8install
           CC=gcc-8 CFLAGS="-Werror" make -j all
 
-  make-external-compressors:
-    strategy:
-      matrix:
-        include:
-          - name: "no external compressors"
-            flags: "HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0"
-          - name: "only zlib"
-            flags: "HAVE_ZLIB=1 HAVE_LZ4=0 HAVE_LZMA=0"
-          - name: "only lz4"
-            flags: "HAVE_ZLIB=0 HAVE_LZ4=1 HAVE_LZMA=0"
-          - name: "only lzma"
-            flags: "HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=1"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-      - name: Build with ${{matrix.name}}
-        run: |
-          sudo apt install liblzma-dev
-          ${{matrix.flags}} make zstd
-
-
   implicit-fall-through:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: -Wimplicit-fallthrough build
         run: |
           make clean
@@ -221,11 +190,11 @@ jobs:
   meson-linux:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Install packages
         run: |
           sudo apt-get update
-          sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev liblzma-dev
+          sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev
           pip install --pre meson
       - name: Build with Meson
         run: |
@@ -237,67 +206,25 @@ jobs:
             -Dbin_tests=true \
             -Dbin_contrib=true \
             -Ddefault_library=both \
-            build/meson mesonBuild
-          ninja -C mesonBuild/
-          meson test -C mesonBuild/ --print-errorlogs
-          meson install -C mesonBuild --destdir staging/
-
-  meson-mingw-cross-compilation:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-      - name: Install packages
-        run: |
-          sudo apt-get -qqq update
-          sudo apt-get -y install build-essential python3-pip ninja-build {gcc,g++}-mingw-w64-x86-64
-          pip install --pre meson
-      - name: Build with Meson
-        run: |
-          cat > cross.ini <<EOF
-          [binaries]
-          ar = 'x86_64-w64-mingw32-ar'
-          c = 'x86_64-w64-mingw32-gcc'
-          cpp = 'x86_64-w64-mingw32-g++'
-          ld = 'x86_64-w64-mingw32-ld'
-          objcopy = 'x86_64-w64-mingw32-objcopy'
-          objdump = 'x86_64-w64-mingw32-objdump'
-          strip = 'x86_64-w64-mingw32-strip'
-          windres = 'x86_64-w64-mingw32-windres'
-
-          [host_machine]
-          system = 'windows'
-          endian = 'little'
-          cpu_family = 'x86_64'
-          cpu = 'x86_64'
-          EOF
-
-          # pzstd doesn't build; skip -Dbin_contrib=true
-          meson setup \
-            --buildtype=debugoptimized \
-            --cross-file=cross.ini \
-            -Db_lundef=false \
-            -Dbin_programs=true \
-            -Dbin_tests=true \
-            -Ddefault_library=both \
             build/meson builddir
           ninja -C builddir/
-          if grep -- -pthread builddir/meson-private/libzstd.pc; then
-            echo "Error: found stray pthread dependency"
-            exit 1
-          fi
+          meson test -C builddir/ --print-errorlogs
+          meson install -C builddir --destdir staging/
 
   meson-windows:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Install packages
         run: pip install --pre meson
+      - name: Initialize the MSVC dev command prompt
+        uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89
       - name: Configure with Meson
         run: |
-          meson setup --vsenv build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true
+          meson setup build/meson/ builddir -Dbin_tests=true -Dbin_programs=true -Dbin_contrib=true
       - name: Build with Meson
         run: |
-          meson compile -C builddir/
+          ninja -C builddir/
       - name: Test with Meson
         run: |
           meson test -C builddir/ --print-errorlogs
@@ -305,47 +232,59 @@ jobs:
         run: |
           meson install -C builddir --destdir staging/
 
+  cmake-visual-2022:
+    strategy:
+      matrix:
+        include:
+          - generator: "Visual Studio 17 2022"
+            flags: "-A x64"
+          - generator: "Visual Studio 17 2022"
+            flags: "-A Win32"
+          - generator: "MinGW Makefiles"
+    runs-on: windows-2022
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: Add MSBuild to PATH
+      uses: microsoft/setup-msbuild@v1.3
+    - name: Build
+      working-directory: ${{env.GITHUB_WORKSPACE}}
+      run: |
+        cd build\cmake
+        mkdir build
+        cd build
+        cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} ..
+        cmake.exe --build .
+
   msbuild-visual-studio:
     strategy:
-      fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:
         include: [
-          { name: "VS 2022 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2022", arch: "" },
-          { name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022", arch: "" },
-          { name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
-          { name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
-          { name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
-          { name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
-          { name: "VS 2022 x64 Release AVX2", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: "AdvancedVectorExtensions2" },
+          { name: "VS 2022 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022"},
+          { name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019"},
+          { name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019"},
         ]
     runs-on: ${{matrix.runner}}
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
-    - name: Build ${{matrix.name}}
+      uses: microsoft/setup-msbuild@v1.3
+    - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
       # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      if: ${{ matrix.arch == '' }}
       run: >
         msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=${{matrix.toolset}}
-        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} /warnaserror
-    - name: Build ${{matrix.name}}
-      working-directory: ${{env.GITHUB_WORKSPACE}}
-      # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
-      if: ${{ matrix.arch != '' }}
-      run: >
-        msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=${{matrix.toolset}}
-        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} /warnaserror
-        /p:InstructionSet=${{matrix.arch}}
+        /t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
 
-  # This tests that we don't accidentally grow the size too much.
+  # This tests that we don't accidently grow the size too much.
   # If the size grows intentionally, you can raise these numbers.
   # But we do need to think about binary size, since it is a concern.
   libzstd-size:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: libzstd size test
       run: |
         make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
@@ -356,7 +295,7 @@ jobs:
   minimal-decompressor-macros:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: minimal decompressor macros
       run: |
         make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -367,13 +306,11 @@ jobs:
         make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
         make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
         make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
-        make clean && make check ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1 MOREFLAGS="-Werror"
-        make clean && make check ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1 MOREFLAGS="-Werror"
 
   dynamic-bmi2:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: dynamic bmi2 tests
       run: |
         make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -385,34 +322,33 @@ jobs:
   test-variants:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make all variants & validate
       run: |
         make -j -C programs allVariants MOREFLAGS=-O0
         ./tests/test-variants.sh
 
+
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false  # 'false' means Don't stop matrix workflows even if some matrix failed.
       matrix:
         include: [
           { name: ARM,      xcc_pkg: gcc-arm-linux-gnueabi,     xcc: arm-linux-gnueabi-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-arm-static     },
-          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-aarch64,xemu: qemu-aarch64-static },
+          { name: ARM64,    xcc_pkg: gcc-aarch64-linux-gnu,     xcc: aarch64-linux-gnu-gcc,     xemu_pkg: qemu-system-arm,    xemu: qemu-aarch64-static },
           { name: PPC,      xcc_pkg: gcc-powerpc-linux-gnu,     xcc: powerpc-linux-gnu-gcc,     xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc-static     },
           { name: PPC64LE,  xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc,    xemu: qemu-ppc64le-static },
           { name: S390X,    xcc_pkg: gcc-s390x-linux-gnu,       xcc: s390x-linux-gnu-gcc,       xemu_pkg: qemu-system-s390x,  xemu: qemu-s390x-static   },
           { name: MIPS,     xcc_pkg: gcc-mips-linux-gnu,        xcc: mips-linux-gnu-gcc,        xemu_pkg: qemu-system-mips,   xemu: qemu-mips-static    },
-          { name: RISC-V,   xcc_pkg: gcc-riscv64-linux-gnu,     xcc: riscv64-linux-gnu-gcc,     xemu_pkg: qemu-system-riscv64,xemu: qemu-riscv64-static },
           { name: M68K,     xcc_pkg: gcc-m68k-linux-gnu,        xcc: m68k-linux-gnu-gcc,        xemu_pkg: qemu-system-m68k,   xemu: qemu-m68k-static    },
-          { name: SPARC,    xcc_pkg: gcc-sparc64-linux-gnu,     xcc: sparc64-linux-gnu-gcc,     xemu_pkg: qemu-system-sparc,  xemu: qemu-sparc64-static },
         ]
     env:                        # Set environment variables
       XCC: ${{ matrix.xcc }}
       XEMU: ${{ matrix.xemu }}
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: apt update & install
       run: |
         sudo apt-get update
@@ -432,15 +368,7 @@ jobs:
     - name: ARM64
       if: ${{ matrix.name == 'ARM64' }}
       run: |
-        make clean
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
-        CFLAGS="-march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
-        CFLAGS="-march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
-# This test is only compatible with standard libraries that support BTI (Branch Target Identification).
-# Unfortunately, the standard library provided on Ubuntu 24.04 does not have this feature enabled.
-#        make clean
-#        LDFLAGS="-static -z force-bti" MOREFLAGS="-mbranch-protection=standard" CC=$XCC QEMU_SYS=$XEMU make check V=1
+        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
     - name: PPC
       if: ${{ matrix.name == 'PPC' }}
       run: |
@@ -457,18 +385,10 @@ jobs:
       if: ${{ matrix.name == 'MIPS' }}
       run: |
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: RISC-V
-      if: ${{ matrix.name == 'RISC-V' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
     - name: M68K
       if: ${{ matrix.name == 'M68K' }}
       run: |
         LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
-    - name: SPARC
-      if: ${{ matrix.name == 'SPARC' }}
-      run: |
-        LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
 
   mingw-short-test:
     runs-on: windows-latest
@@ -484,8 +404,8 @@ jobs:
       run:
         shell: msys2 {0}
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - uses: msys2/setup-msys2@40677d36a502eb2cf0fb808cc9dec31bf6152638 # tag=v2.28.0
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - uses: msys2/setup-msys2@v2
       with:
         msystem: ${{ matrix.msystem }}
         install: make diffutils
@@ -520,9 +440,9 @@ jobs:
         platform: [x64, Win32]
         configuration: [Release]
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
+      uses: microsoft/setup-msbuild@v1.3
     - name: Build and run tests
       working-directory: ${{env.GITHUB_WORKSPACE}}
       env:
@@ -541,8 +461,8 @@ jobs:
     runs-on: windows-latest
     steps:
     - run: git config --global core.autocrlf input
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - uses: cygwin/cygwin-install-action@f61179d72284ceddc397ed07ddb444d82bf9e559 # tag=v5
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - uses: cygwin/cygwin-install-action@f5e0f048310c425e84bc789f493a828c6dc80a25 # tag=master
       with:
         platform: x86_64
         packages: >-
@@ -562,18 +482,32 @@ jobs:
         make -j allzstd &&
         make -C tests fuzzer &&
         ./tests/fuzzer.exe -v -T1m
-    - name: cygwin install test
-      shell: C:\cygwin\bin\bash.exe --noprofile --norc -eo pipefail '{0}'
-      run: >-
-        make -j &&
-        make install
+
+  intel-cet-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: Build Zstd
+      run: |
+        make -j zstd V=1
+        readelf -n zstd
+    - name: Get Intel SDE
+      run: |
+        curl -LO https://downloadmirror.intel.com/684899/sde-external-9.0.0-2021-11-07-lin.tar.xz
+        tar xJvf sde-external-9.0.0-2021-11-07-lin.tar.xz
+    - name: Configure Permissions
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+    - name: Run Under SDE
+      run: |
+        sde-external-9.0.0-2021-11-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
 
   pkg-config:
     runs-on: ubuntu-latest
     container:
       image: debian:testing
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Install dependencies
         run: |
           apt -y update
@@ -588,7 +522,7 @@ jobs:
   versions-compatibility:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Versions Compatibility Test
       run: |
         make -C tests versionsTest
@@ -596,15 +530,27 @@ jobs:
   clangbuild:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make clangbuild
       run: |
         make clangbuild
 
+  clang-pgo:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: Build PGO Zstd with Clang
+      env:
+        CC: clang-14
+        LLVM_PROFDATA: llvm-profdata-14
+      run: |
+        make -C programs zstd-pgo
+        ./programs/zstd -b
+
   gcc-pgo:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: Build PGO Zstd with GCC
       env:
         CC: gcc
@@ -612,67 +558,26 @@ jobs:
         make -C programs zstd-pgo
         ./programs/zstd -b
 
-  clang-pgo:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: Build PGO Zstd with Clang
-      env:
-        CC: clang
-      run: |
-        sudo apt install -y llvm
-        llvm-profdata --version
-        make -C programs zstd-pgo
-        ./programs/zstd -b
 
-  musl-build:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-      - name: Install musl-tools
-        run: |
-          sudo apt install -y musl-tools
-      - name: Compile with musl-gcc and test-zstd
-        run: |
-          CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1
-
-  intel-cet-compatibility:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: Build Zstd
-      run: |
-        make -j zstd V=1
-        readelf -n zstd
-    - name: Get Intel SDE
-      run: |
-        curl -LO https://downloadmirror.intel.com/813591/sde-external-9.33.0-2024-01-07-lin.tar.xz
-        tar xJvf sde-external-9.33.0-2024-01-07-lin.tar.xz
-    - name: Configure Permissions
-      run: |
-        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
-    - name: Run Under SDE
-      run: |
-        sde-external-9.33.0-2024-01-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
-
-  icx:
-    # install instructions: https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/2025-0/apt-005.html
-    name: icx-check
-    runs-on: ubuntu-latest
-    steps:
-    - name: install icx
-      run: |
-        # download the key to system keyring
-        wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-        | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
-
-        # add signed entry to apt sources and configure the APT client to use Intel repository:
-        echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-        sudo apt-get update
-        sudo apt-get install -y intel-basekit intel-hpckit
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - name: make check
-      run: |
-        source /opt/intel/oneapi/setvars.sh
-        make CC=icx check
-        make CC=icx -C tests test-cli-tests
+# For reference : icc tests
+# icc tests are currently failing on Github Actions, likely to issues during installation stage
+# To be fixed later
+#
+#  icc:
+#    name: icc-check
+#    runs-on: ubuntu-latest
+#    steps:
+#    - name: install icc
+#      run: |
+#        export DEBIAN_FRONTEND=noninteractive
+#        sudo apt-get -qqq update
+#        sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
+#        sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
+#        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
+#        sudo apt-get update
+#        sudo apt-get install -y intel-basekit intel-hpckit
+#    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+#    - name: make check
+#      run: |
+#        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check

.github/workflows/nightly.yml

@@ -1,38 +0,0 @@
-name: facebook/zstd/nightly
-on:
-  schedule:
-    - cron: '0 0 * * *'
-  push:
-    branches:
-      - release
-      - dev
-      - '*nightly*'
-permissions: read-all
-jobs:
-  regression-test:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install dependencies
-      run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
-    - name: Regression Test
-      run: |
-        make -C programs zstd
-        make -C tests/regression test
-
-# Longer tests
-  #- make -C tests test-zstd-nolegacy && make clean
-  #- pyenv global 3.4.4; make -C tests versionsTest && make clean
-  #- make zlibwrapper         && make clean
-  #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
-  #- make uasan               && make clean
-  #- make asan32              && make clean
-  #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
-# Valgrind tests
-  #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
-  #- make -C tests valgrindTest && make clean
-# ARM, AArch64, PowerPC, PowerPC64 tests
-  #- make ppctest             && make clean
-  #- make ppc64test           && make clean
-  #- make armtest             && make clean
-  #- make aarch64test         && make clean

.github/workflows/publish-release-artifacts.yml

@@ -5,19 +5,21 @@ on:
     types:
       - published
 
-permissions: read-all
+permissions:
+  contents: read
 
 jobs:
   publish-release-artifacts:
     permissions:
-      contents: write # to fetch code and upload artifacts
+      contents: read # to fetch code (actions/checkout)
+      actions: write # to attach binaries to release artifacts (skx/github-action-publish-binaries)
 
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/')
 
     steps:
       - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
 
       - name: Archive
         env:
@@ -66,7 +68,7 @@ jobs:
           fi
 
       - name: Publish
-        uses: skx/github-action-publish-binaries@b9ca5643b2f1d7371a6cba7f35333f1461bbc703 # tag=release-2.0
+        uses: skx/github-action-publish-binaries@release-2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.github/workflows/release_check.yml

@@ -1,64 +0,0 @@
-name: release_checks
-
-on:
-  push:
-    branches:
-      - release
-  pull_request:
-    branches:
-      - release
-
-permissions: read-all
-
-jobs:
-  verify-manual:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Save current manual
-        run: mv doc/zstd_manual.html doc/zstd_manual_saved.html
-
-      - name: Generate new manual
-        run: make manual
-
-      - name: Compare manuals
-        run: |
-          if ! cmp -s doc/zstd_manual.html doc/zstd_manual_saved.html; then
-            echo "The API manual was not updated before release !"
-            exit 1
-          fi
-
-  verify-man-pages:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y ruby ruby-dev
-          sudo gem install ronn
-
-      - name: Display ronn version
-        run: ronn --version
-
-      - name: Save current man pages
-        run: |
-          mv programs/zstd.1 programs/zstd.1.saved
-          mv programs/zstdgrep.1 programs/zstdgrep.1.saved
-          mv programs/zstdless.1 programs/zstdless.1.saved
-
-      - name: Generate new manual pages
-        run: make -C programs man
-
-      - name: Compare man pages
-        run: |
-          for file in zstd.1 zstdgrep.1 zstdless.1; do
-            if ! cmp -s programs/$file programs/$file.saved; then
-              echo "The man page $file should have been updated."
-              exit 1
-            fi
-          done

.github/workflows/scorecards.yml

@@ -27,12 +27,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # tag=v2.4.1
+        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # tag=v2.1.2
         with:
           results_file: results.sarif
           results_format: sarif
@@ -51,7 +51,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # tag=v4.3.1
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
         with:
           name: SARIF file
           path: results.sarif
@@ -59,6 +59,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # tag=v3.28.9
+        uses: github/codeql-action/upload-sarif@3ebbd71c74ef574dbc558c82f70e52732c8b44fe # tag=v2.2.1
         with:
           sarif_file: results.sarif

.github/workflows/windows-artifacts.yml

@@ -1,81 +0,0 @@
-name: windows-artifacts
-
-on:
-  push:
-    branches: [ test_artifacts, win_artifacts, release ]
-  release:
-    types:
-      - published
-
-permissions: read-all
-
-jobs:
-  windows-artifacts:
-    permissions:
-      contents: write # to fetch code and upload artifacts
-    # see https://ariya.io/2020/07/on-github-actions-with-msys2
-    runs-on: windows-latest
-    # see https://github.com/msys2/setup-msys2
-    strategy:
-      matrix:
-        include:
-          - { msystem: mingw64, env: x86_64, ziparch: win64 }
-          - { msystem: mingw32, env: i686, ziparch: win32 }
-    defaults:
-      run:
-        shell: msys2 {0}
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # tag=v4.2.2
-    - uses: msys2/setup-msys2@40677d36a502eb2cf0fb808cc9dec31bf6152638 # tag=v2.28.0
-      with:
-        msystem: ${{ matrix.msystem }}
-        install: make p7zip git mingw-w64-${{matrix.env}}-gcc
-        update: true
-
-    - name: display versions
-      run: |
-        make -v
-        cc -v
-
-    - name: Building zlib to static link
-      run: |
-        git clone --depth 1 --branch v1.3.1 https://github.com/madler/zlib
-        make -C zlib -f win32/Makefile.gcc libz.a
-
-    - name: Building lz4 to static link
-      run: |
-        git clone --depth 1 --branch v1.10.0 https://github.com/lz4/lz4
-        # ensure both libraries use the same version of libxxhash
-        cp lib/common/xxhash.* lz4/lib
-        CPPFLAGS=-DXXH_NAMESPACE=LZ4_ make -C lz4/lib liblz4.a V=1
-
-    - name: Building zstd programs
-      run: |
-        CPPFLAGS="-I../zlib -I../lz4/lib" LDFLAGS=-static make -j allzstd V=1 HAVE_ZLIB=1 HAVE_LZ4=1 HAVE_LZMA=0 LDLIBS="../zlib/libz.a ../lz4/lib/liblz4.a"
-
-    - name: Create artifacts
-      run: |
-        ./lib/dll/example/build_package.bat || exit 1
-        mv bin/ zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
-
-    - name: Publish zstd-$VERSION-${{matrix.ziparch}}.zip for manual inspection
-      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # tag=v4.3.1
-      with:
-        compression-level: 9  # maximum compression
-        if-no-files-found: error # defaults to `warn`
-        path: ${{ github.workspace }}/zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
-        name: zstd-${{ github.ref_name }}-${{matrix.ziparch}}
-
-    - name: Package artifact for upload
-      run: |
-        7z a -tzip -mx9 "$(cygpath -u '${{ github.workspace }}/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip')" "$(cygpath -u '${{ github.workspace }}/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}')"
-
-    - name: Upload release asset
-      if: github.event_name == 'release'
-      shell: pwsh
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        gh release upload "${{ github.ref_name }}" "$env:GITHUB_WORKSPACE/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip" --clobber
-
-

.gitignore

@@ -12,8 +12,6 @@
 *.so
 *.so.*
 *.dylib
-*.framework
-*.xcframework
 
 # Executables
 /zstd
@@ -29,8 +27,6 @@ tmp*
 dictionary.
 dictionary
 NUL
-cmakebuild/
-install/
 
 # Build artefacts
 contrib/linux-kernel/linux/
@@ -41,15 +37,11 @@ buck-out/
 build-*
 *.gcda
 
-# IDE
-.clang_complete
-compile_flags.txt
-.clang-format
-
 # Other files
 .directory
 _codelite/
 _zstdbench/
+.clang_complete
 *.idea
 *.swp
 .DS_Store

.travis.yml

@@ -0,0 +1,128 @@
+# Travis CI is used to test platforms that github-actions currently doesn't support
+# without either self-hosting or some finnicky work-around. Also, some tests
+# are troublesome to migrate since GH Actions runs tests not in a tty.
+language: c
+
+git:
+  depth: 1
+
+branches:
+  only:
+  - dev
+  - release
+  - master
+  - travisTest
+
+addons:
+  apt:
+    update: true
+
+env:
+  global:
+    - FUZZERTEST=-T1mn
+      ZSTREAM_TESTTIME=-T1mn
+      DECODECORPUS_TESTTIME=-T1mn
+
+matrix:
+  fast_finish: true
+  include:
+    - name: S390X (big endian) + Fuzz test
+      dist: trusty
+      arch: s390x
+      script:
+        - FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
+
+    - name: S390X (big endian) + Fuzz test + no intrinsics
+      dist: trusty
+      arch: s390x
+      script:
+        - MOREFLAGS="-DZSTD_NO_INTRINSICS" FUZZER_FLAGS=--no-big-tests make -C tests fuzztest
+
+    - name: arm64    # ~2.5 mn
+      os: linux
+      arch: arm64
+      script:
+        - make check
+
+    - name: arm64fuzz
+      os: linux
+      arch: arm64
+      script:
+        - make -C tests fuzztest
+
+    # TODO: migrate to GH Actions once newest clang staticanalyze warnings are fixed
+    - name: static analyzer scanbuild    # ~8mn
+      dist: trusty  # note : it's important to pin down a version of static analyzer, since different versions report different false positives
+      script:
+        - make staticAnalyze
+
+    # GH actions can't run this command on OS-X, non-tty issues
+    - name: OS-X make all lib
+      os: osx
+      script:
+        - make -C lib all
+
+    # Introduced to check compat with old toolchains, to prevent e.g. #1872
+    - name: ARM Build Test (on Trusty)
+      dist: trusty
+      script:
+        - make arminstall
+        - make armbuild
+
+    # check release number (release/new tag only)
+    - name: Tag-Specific Test
+      if: tag =~ ^v[0-9]\.[0-9]
+      script:
+        - make -C tests checkTag
+        - tests/checkTag "$TRAVIS_BRANCH"
+
+    - name: PPC64LE + Fuzz test  # ~13mn
+      arch: ppc64le
+      env:
+        - FUZZER_FLAGS=--no-big-tests
+        - MOREFLAGS="-static"
+      script:
+        - cat /proc/cpuinfo
+        - make -C tests fuzztest
+
+    # This test currently fails on GA specifically, for no obvious reason
+    # (it works fine on travisCI, and on local test platforms).
+    - name: Versions Compatibility Test   # ~6mn
+      script:
+        - make -C tests versionsTest
+
+    # meson dedicated test
+    - name: Focal (Meson + clang)    # ~15mn
+      dist: focal
+      language: cpp
+      compiler: clang
+      install:
+        - sudo apt-get install -qq liblz4-dev valgrind tree
+        - |
+          travis_retry curl -o ~/ninja.zip -L 'https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-linux.zip' &&
+          unzip ~/ninja.zip -d ~/.local/bin
+        - |
+          travis_retry curl -o ~/get-pip.py -L 'https://bootstrap.pypa.io/pip/3.6/get-pip.py' &&
+          python3 ~/get-pip.py --user &&
+          pip3 install --user meson
+      script:
+        - |
+          meson setup \
+            --buildtype=debugoptimized \
+            -Db_lundef=false \
+            -Dauto_features=enabled \
+            -Dbin_programs=true \
+            -Dbin_tests=true \
+            -Dbin_contrib=true \
+            -Ddefault_library=both \
+             build/meson builddir
+        - pushd builddir
+        - ninja
+        - meson test --verbose --no-rebuild
+        - DESTDIR=./staging ninja install
+        - tree ./staging
+      after_failure:
+        - cat "$TRAVIS_BUILD_DIR"/builddir/meson-logs/testlog.txt
+
+  allow_failures:
+    - env: ALLOW_FAILURES=true

CHANGELOG

@@ -1,85 +1,3 @@
-V1.5.7 (Feb 2025)
-fix: compression bug in 32-bit mode associated with long-lasting sessions
-api: new method `ZSTD_compressSequencesAndLiterals()` (#4217, #4232)
-api: `ZSTD_getFrameHeader()` works on skippable frames (#4228)
-perf: substantial compression speed improvements (up to +30%) on small data, by @TocarIP (#4144) and @cyan4973 (#4165)
-perf: improved compression speed (~+5%) for dictionary compression at low levels (#4170)
-perf: much faster speed for `--patch-from` at high compression levels (#4276)
-perf: higher `--patch-from` compression ratios, notably at high levels (#4288)
-perf: better speed for binaries on Windows (@pps83) and when compiled with Visual Studio (@MessyHack)
-perf: slight compression ratio improvement thanks to better block boundaries (#4136, #4176, #4178)
-perf: slight compression ratio improvement for `dfast`, aka levels 3 and 4 (#4171)
-perf: runtime bmi2 detection enabled on x86 32-bit mode (#4251)
-cli: multi-threading as default CLI setting, by @daniellerozenblit
-cli: new `--max` command (#4290)
-build: improve `msbuild` version autodetection, support VS2022, by @ManuelBlanc
-build: fix `meson` build by @artem and @Victor-C-Zhang, and on Windows by @bgilbert
-build: compatibility with Apple Framework, by @Treata11
-build: improve icc/icx compatibility, by @josepho0918 and @luau-project
-build: improve compatibility with Android NDK, by Adenilson Cavalcanti
-portability: linux kernel branch, with improved support for Sequence producers (@embg, @gcabiddu, @cyan4973)
-portability: improved qnx compatibility, suggested by @rainbowball
-portability: improved install script for FreeBSD, by @sunpoet
-portability: fixed test suite compatibility with gnu hurd, by @diegonc
-doc: clarify specification, by @elasota
-misc: improved tests/decodecorpus validation tool (#4102), by antmicro
-
-V1.5.6 (Mar 2024)
-api: Promote `ZSTD_c_targetCBlockSize` to Stable API by @felixhandte
-api: new `ZSTD_d_maxBlockSize` experimental parameter, to reduce streaming decompression memory, by @terrelln
-perf: improve performance of param `ZSTD_c_targetCBlockSize`, by @Cyan4973
-perf: improved compression of arrays of integers at high compression, by @Cyan4973
-lib: reduce binary size with selective build-time exclusion, by @felixhandte
-lib: improved huffman speed on small data and linux kernel, by @terrelln
-lib: accept dictionaries with partial literal tables, by @terrelln
-lib: fix CCtx size estimation with external sequence producer, by @embg
-lib: fix corner case decoder behaviors, by @Cyan4973 and @aimuz
-lib: fix zdict prototype mismatch in static_only mode, by @ldv-alt
-lib: fix several bugs in magicless-format decoding, by @embg
-cli: add common compressed file types to `--exclude-compressed`` by @daniellerozenblit
-cli: fix mixing `-c` and `-o` commands with `--rm`, by @Cyan4973
-cli: fix erroneous exclusion of hidden files with `--output-dir-mirror` by @felixhandte
-cli: improved time accuracy on BSD, by @felixhandte
-cli: better errors on argument parsing, by @KapJI
-tests: better compatibility with older versions of `grep`, by @Cyan4973
-tests: lorem ipsum generator as default backup content, by @Cyan4973
-build: cmake improvements by @terrelln, @sighingnow, @gjasny, @JohanMabille, @Saverio976, @gruenich, @teo-tsirpanis
-build: bazel support, by @jondo2010
-build: fix cross-compiling for AArch64 with lld by @jcelerier
-build: fix Apple platform compatibility, by @nidhijaju
-build: fix Visual 2012 and lower compatibility, by @Cyan4973
-build: improve win32 support, by @DimitriPapadopoulos
-build: better C90 compliance for zlibWrapper, by @emaste
-port: make: fat binaries on macos, by @mredig
-port: ARM64EC compatibility for Windows, by @dunhor
-port: QNX support by @klausholstjacobsen
-port: MSYS2 and Cygwin makefile installation and test support, by @QBos07
-port: risc-v support validation in CI, by @Cyan4973
-port: sparc64 support validation in CI, by @Cyan4973
-port: AIX compatibility, by @likema
-port: HP-UX compatibility, by @likema
-doc: Improved specification accuracy, by @elasota
-bug: Fix and deprecate ZSTD_generateSequences (#3981)
-
-v1.5.5 (Apr 2023)
-fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln)
-perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln)
-lib: deprecated bufferless block-level API (#3534) by @terrelln
-cli: mmap large dictionaries to save memory, by @daniellerozenblit
-cli: improve speed of --patch-from mode (~+50%) (#3545) by @daniellerozenblit
-cli: improve i/o speed (~+10%) when processing lots of small files (#3479) by @felixhandte
-cli: zstd no longer crashes when requested to write into write-protected directory (#3541) by @felixhandte
-cli: fix decompression into block device using -o, reported by @georgmu (#3583)
-build: fix zstd CLI compiled with lzma support but not zlib support (#3494) by @Hello71
-build: fix cmake does no longer require 3.18 as minimum version (#3510) by @kou
-build: fix MSVC+ClangCL linking issue (#3569) by @tru
-build: fix zstd-dll, version of zstd CLI that links to the dynamic library (#3496) by @yoniko
-build: fix MSVC warnings (#3495) by @embg
-doc: updated zstd specification to clarify corner cases, by @Cyan4973
-doc: document how to create fat binaries for macos (#3568) by @rickmark
-misc: improve seekable format ingestion speed (~+100%) for very small chunk sizes (#3544) by @Cyan4973
-misc: tests/fullbench can benchmark multiple files (#3516) by @dloidolt
-
 v1.5.4 (Feb 2023)
 perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln)
 perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg)
@@ -161,7 +79,7 @@ build: support for m68k (Motorola 68000's), by @cyan4973
 build: improved AIX support, by @Helflym
 build: improved meson unofficial build, by @eli-schwartz
 cli : custom memory limit when training dictionary (#2925), by @embg
-cli : report advanced parameters information when compressing in very verbose mode (`-vv`), by @Svetlitski-FB
+cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB
 
 v1.5.0  (May 11, 2021)
 api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42)
@@ -228,7 +146,7 @@ api: Add Function to Generate Skippable Frame (#2439, @senhuang42)
 perf: New Algorithms for the Long Distance Matcher (#2483, @mpu)
 perf: Performance Improvements for Long Distance Matcher (#2464, @mpu)
 perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln)
-cli: Fix `--output-dir-mirror` rejection of `..` -containing paths (#2512, @felixhandte)
+cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte)
 cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte)
 cli: Improve Help Message (#2500, @senhuang42)
 tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973)
@@ -515,7 +433,7 @@ misc: added /contrib/docker script by @gyscos
 
 v1.3.3  (Dec 21, 2017)
 perf: faster zstd_opt strategy (levels 16-19)
-fix : bug #944 : multithreading with shared dictionary and large data, reported by @gsliepen
+fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
 cli : fix : content size written in header by default
 cli : fix : improved LZ4 format support, by @felixhandte
 cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file

CONTRIBUTING.md

@@ -60,7 +60,7 @@ Our contribution process works in three main stages:
         * Note: run local tests to ensure that your changes didn't break existing functionality
             * Quick check
             ```
-            make check
+            make shortest
             ```
             * Longer check
             ```
@@ -216,7 +216,7 @@ will typically not be stable enough to obtain reliable benchmark results. If you
 hands on a desktop, this is usually a better scenario.
 
 Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
-do a little more work to ensure that you are in fact measuring the changes you've made and not
+do a little more work to ensure that you are in fact measuring the changes you've made not and
 noise. Here are some things you can do to make your benchmarks more stable:
 
 1. The most simple thing you can do to drastically improve the stability of your benchmark is

Makefile

@@ -85,10 +85,14 @@ test:
 	$(MAKE) -C $(TESTDIR) $@
 	ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
 
+## shortest: same as `make check`
+.PHONY: shortest
+shortest:
+	$(Q)$(MAKE) -C $(TESTDIR) $@
+
 ## check: run basic tests for `zstd` cli
 .PHONY: check
-check:
-	$(Q)$(MAKE) -C $(TESTDIR) $@
+check: shortest
 
 .PHONY: automated_benchmarking
 automated_benchmarking:
@@ -141,13 +145,13 @@ clean:
 	$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
 	$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
 	$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
-	$(Q)$(RM) -r lz4 cmakebuild mesonbuild install
+	$(Q)$(RM) -r lz4
 	@echo Cleaning completed
 
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, Hurd and some BSD targets
 #------------------------------------------------------------------------------
-ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT% CYGWIN_NT% Haiku AIX,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT Haiku AIX))
 
 HOST_OS = POSIX
 
@@ -193,27 +197,18 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
-.PHONY: clangbuild-darwin-fat
-clangbuild-darwin-fat: clean
-	clang -v
-	CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release
-	mv programs/zstd programs/zstd_arm64
-	CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release
-	mv programs/zstd programs/zstd_x64
-	lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd
-
 .PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
 gcc5build: clean
 	gcc-5 -v
-	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
+	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
 
 gcc6build: clean
 	gcc-6 -v
-	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
+	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
 
 gcc7build: clean
 	gcc-7 -v
-	CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
+	CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"
 
 clangbuild: clean
 	clang -v
@@ -237,17 +232,17 @@ ppc64build: clean
 
 .PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
 armfuzz: clean
-	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 aarch64fuzz: clean
 	ld -v
-	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 ppcfuzz: clean
-	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 ppc64fuzz: clean
-	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 .PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
 cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
@@ -256,49 +251,49 @@ cxxtest: clean
 
 gcc5test: clean
 	gcc-5 -v
-	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)"
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
 
 gcc6test: clean
 	gcc-6 -v
-	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)"
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
 
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
 
 aarch64test:
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+	$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
 
 ppctest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
 
 ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
 
 .PHONY: arm-ppc-compilation
 arm-ppc-compilation:
-	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
-	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
-	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)"
-	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
 regressiontest:
 	$(MAKE) -C $(FUZZDIR) regressiontest
 
 uasanregressiontest:
-	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
 
 msanregressiontest:
-	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory -Werror" CXXFLAGS="-O3 -fsanitize=memory -Werror"
+	$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
 
 update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
 update_regressionResults:
-	$(MAKE) -j -C programs zstd
-	$(MAKE) -j -C tests/regression test
-	$(RM) -r $(REGRESS_RESULTS_DIR)
+	$(MAKE) -C programs zstd
+	$(MAKE) -C tests/regression test
+	$(RM) -rf $(REGRESS_RESULTS_DIR)
 	$(MKDIR) $(REGRESS_RESULTS_DIR)
 	./tests/regression/test                         \
         --cache  tests/regression/cache             \
@@ -313,32 +308,31 @@ update_regressionResults:
 # run UBsan with -fsanitize-recover=pointer-overflow
 # this only works with recent compilers such as gcc 8+
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined -Werror $(MOREFLAGS)"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
 
 asan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
 
 asan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $*
 
 msan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason
 
-msan-%:
-	$(MAKE) clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -j -C $(TESTDIR) HAVE_LZMA=0 $*
+msan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
 
 asan32: clean
-	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
+	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
 
 .PHONY: apt-install
 apt-install:
@@ -351,7 +345,7 @@ apt-add-repo:
 	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
 	sudo apt-get update -y -qq
 
-.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install
+.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
 ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
 
@@ -379,53 +373,35 @@ gpp6install: apt-add-repo
 clang38install:
 	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
 
+# Ubuntu 14.04 ships a too-old lz4
+lz4install:
+	[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
+
 endif
 
 
-ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
+
+ifneq (,$(filter MSYS%,$(shell uname)))
 HOST_OS = MSYS
+CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif
 
 #------------------------------------------------------------------------
 # target specific tests
 #------------------------------------------------------------------------
-ifneq (,$(filter MSYS POSIX,$(HOST_OS)))
-
-CMAKE ?= cmake
-CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
-
-ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname')))
-CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
-endif
-
-.PHONY: cmakebuild
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+.PHONY: cmakebuild c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
 cmakebuild:
-	$(CMAKE) --version
-	$(RM) -r cmakebuild install
-	$(MKDIR) cmakebuild install
-	cd cmakebuild; $(CMAKE) -Wdev -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Werror -O0" -DCMAKE_INSTALL_PREFIX=install $(CMAKE_PARAMS) ../build/cmake
-	$(CMAKE) --build cmakebuild --target install -- -j V=1
-	cd cmakebuild; ctest -V -L Medium
+	cmake --version
+	$(RM) -r $(BUILDIR)/cmake/build
+	$(MKDIR) $(BUILDIR)/cmake/build
+	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
+	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
+	$(MAKE) -C $(BUILDIR)/cmake/build install;
+	$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
+	cd $(BUILDIR)/cmake/build; ctest -V -L Medium
 
-MESON ?= meson
-NINJA ?= ninja
-
-.PHONY: mesonbuild
-mesonbuild:
-	$(MESON) setup \
-		--buildtype=debugoptimized \
-		-Db_lundef=false \
-		-Dauto_features=enabled \
-		-Dbin_programs=true \
-		-Dbin_tests=true \
-		-Dbin_contrib=true \
-		-Ddefault_library=both \
-		build/meson mesonbuild
-	$(NINJA) -C mesonbuild/
-	$(MESON) test -C mesonbuild/ --print-errorlogs
-	$(MESON) install -C mesonbuild --destdir staging/
-
-.PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
 c89build: clean
 	$(CC) -v
 	CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd

README.md

@@ -5,7 +5,7 @@ targeting real-time compression scenarios at zlib-level and better compression r
 It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy).
 
 Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available.
-This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) OR [GPLv2](COPYING) licensed **C** library,
+This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library,
 and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files.
 Should your project require another programming language,
 a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages).
@@ -13,12 +13,15 @@ a list of known ports and bindings is provided on [Zstandard homepage](https://f
 **Development branch status:**
 
 [![Build Status][travisDevBadge]][travisLink]
+[![Build status][AppveyorDevBadge]][AppveyorLink]
 [![Build status][CircleDevBadge]][CircleLink]
 [![Build status][CirrusDevBadge]][CirrusLink]
 [![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink]
 
 [travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
 [travisLink]: https://travis-ci.com/facebook/zstd
+[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite"
+[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
 [CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
 [CircleLink]: https://circleci.com/gh/facebook/zstd
 [CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
@@ -29,10 +32,10 @@ a list of known ports and bindings is provided on [Zstandard homepage](https://f
 ## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a desktop featuring a Core i7-9700K CPU @ 4.9GHz
-and running Ubuntu 24.04 (`Linux 6.8.0-53-generic`),
+on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
+with a Core i7-9700K CPU @ 4.9GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 14.2.0,
+compiled with [gcc] 9.3.0,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
@@ -41,23 +44,24 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.5.7 -1**       | 2.896 |   510 MB/s |  1550 MB/s |
-| brotli 1.1.0 -1         | 2.883 |   290 MB/s |   425 MB/s |
-| [zlib] 1.3.1 -1         | 2.743 |   105 MB/s |   390 MB/s |
-| **zstd 1.5.7 --fast=1** | 2.439 |   545 MB/s |  1850 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   520 MB/s |   750 MB/s |
-| **zstd 1.5.7 --fast=4** | 2.146 |   665 MB/s |  2050 MB/s |
-| lzo1x 2.10 -1           | 2.106 |   650 MB/s |   780 MB/s |
-| [lz4] 1.10.0            | 2.101 |   675 MB/s |  3850 MB/s |
-| snappy 1.2.1            | 2.089 |   520 MB/s |  1500 MB/s |
-| lzf 3.6 -1              | 2.077 |   410 MB/s |   820 MB/s |
+| **zstd 1.5.1 -1**       | 2.887 |   530 MB/s |  1700 MB/s |
+| [zlib] 1.2.11 -1        | 2.743 |    95 MB/s |   400 MB/s |
+| brotli 1.0.9 -0         | 2.702 |   395 MB/s |   450 MB/s |
+| **zstd 1.5.1 --fast=1** | 2.437 |   600 MB/s |  2150 MB/s |
+| **zstd 1.5.1 --fast=3** | 2.239 |   670 MB/s |  2250 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   540 MB/s |   760 MB/s |
+| **zstd 1.5.1 --fast=4** | 2.148 |   710 MB/s |  2300 MB/s |
+| lzo1x 2.10 -1           | 2.106 |   660 MB/s |   845 MB/s |
+| [lz4] 1.9.3             | 2.101 |   740 MB/s |  4500 MB/s |
+| lzf 3.6 -1              | 2.077 |   410 MB/s |   830 MB/s |
+| snappy 1.1.9            | 2.073 |   550 MB/s |  1750 MB/s |
 
 [zlib]: https://www.zlib.net/
 [lz4]: https://lz4.github.io/lz4/
 
 The negative compression levels, specified with `--fast=#`,
 offer faster compression and decompression speed
-at the cost of compression ratio.
+at the cost of compression ratio (compared to level 1).
 
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.
@@ -150,18 +154,6 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
 By default, `CMAKE_BUILD_TYPE` is set to `Release`.
 
-#### Support for Fat (Universal2) Output
-
-`zstd` can be built and installed with support for both Apple Silicon (M1/M2) as well as Intel by using CMake's Universal2 support.
-To perform a Fat/Universal2 build and install use the following commands:
-
-```bash
-cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64"
-cd build-cmake-debug
-ninja
-sudo ninja install
-```
-
 ### Meson
 
 A Meson project is provided within [`build/meson`](build/meson). Follow
@@ -184,17 +176,6 @@ You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) depe
 The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
 If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
 
-### Conan
-
-You can install pre-built binaries for zstd or build it from source using [Conan](https://conan.io/). Use the following command:
-
-```bash
-conan install --requires="zstd/[*]" --build=missing
-```
-
-The zstd Conan recipe is kept up to date by Conan maintainers and community contributors.
-If the version is out of date, please [create an issue or pull request](https://github.com/conan-io/conan-center-index) on the ConanCenterIndex repository.
-
 ### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities:
@@ -208,10 +189,6 @@ Going into `build` directory, you will find additional possibilities:
 You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo.
 The output binary will be in `buck-out/gen/programs/`.
 
-### Bazel
-
-You easily can integrate zstd into your Bazel project by using the module hosted on the [Bazel Central Repository](https://registry.bazel.build/modules/zstd).
-
 ## Testing
 
 You can run quick local smoke tests by running `make check`.
@@ -227,7 +204,7 @@ Zstandard is considered safe for production environments.
 
 ## License
 
-Zstandard is dual-licensed under [BSD](LICENSE) OR [GPLv2](COPYING).
+Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
 
 ## Contributing
 

SECURITY.md

@@ -1,15 +0,0 @@
-# Reporting and Fixing Security Issues
-
-Please do not open GitHub issues or pull requests - this makes the problem immediately visible to everyone, including malicious actors. Security issues in this open source project can be safely reported via the Meta Bug Bounty program:
-
-https://www.facebook.com/whitehat
-
-Meta's security team will triage your report and determine whether or not is it eligible for a bounty under our program.
-
-# Receiving Vulnerability Notifications
-
-In the case that a significant security vulnerability is reported to us or discovered by us---without being publicly known---we will, at our discretion, notify high-profile, high-exposure users of Zstandard ahead of our public disclosure of the issue and associated fix.
-
-If you believe your project would benefit from inclusion in this list, please reach out to one of the maintainers.
-
-<!-- Note to maintainers: this list is kept [here](https://fburl.com/wiki/cgc1l62x). -->

appveyor.yml

@@ -0,0 +1,205 @@
+# Following tests are run _only_ on `release` branch
+# and on selected feature branch named `appveyorTest` or `visual*`
+
+-
+  version: 1.0.{build}
+  branches:
+    only:
+    - release
+    - master
+    - /appveyor*/
+    - /visual*/
+  environment:
+    matrix:
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "make allzstd MOREFLAGS=-static"
+      ARTIFACT: "true"
+      BUILD:    "true"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "make allzstd MOREFLAGS=-static"
+      ARTIFACT: "true"
+      BUILD:    "true"
+
+    - COMPILER: "clang-cl"
+      HOST:     "cmake-visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+      CMAKE_GENERATOR: "Visual Studio 15 2017"
+      CMAKE_GENERATOR_PLATFORM: "x64"
+      CMAKE_GENERATOR_TOOLSET: "LLVM"
+      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
+
+  install:
+  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[mingw] (
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+    )
+  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+    )
+
+  build_script:
+  - if [%HOST%]==[mingw] (
+      ( if [%PLATFORM%]==[x64] (
+        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+      ) else if [%PLATFORM%]==[x86] (
+        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+      ) )
+    )
+  - if [%HOST%]==[mingw] if [%BUILD%]==[true] (
+      make -v &&
+      sh -c "%COMPILER% -v" &&
+      ECHO Building zlib to static link &&
+      SET "CC=%COMPILER%" &&
+      sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
+      sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
+      ECHO Building zstd &&
+      SET "CPPFLAGS=-I../../zlib" &&
+      SET "LDFLAGS=../../zlib/libz.a" &&
+      sh -c "%SCRIPT%" &&
+      ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
+          ECHO Creating artifacts &&
+          ECHO %cd% &&
+          lib\dll\example\build_package.bat &&
+          make -C programs DEBUGFLAGS= clean zstd &&
+          cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
+          appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
+          cp zstd.exe ..\bin\zstd.exe &&
+          git clone --depth 1 --branch release https://github.com/facebook/zstd &&
+          cd zstd &&
+          git archive --format=tar release -o zstd-src.tar &&
+          ..\zstd -19 zstd-src.tar &&
+          appveyor PushArtifact zstd-src.tar.zst &&
+          certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
+          appveyor PushArtifact zstd-src.tar.zst.sha256.sig &&
+          cd ..\..\bin\ &&
+          7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
+          appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
+      )
+    )
+  - if [%HOST%]==[cmake-visual] (
+      ECHO *** &&
+      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
+      PUSHD build\cmake &&
+      cmake -DBUILD_TESTING=ON . &&
+      cmake --build . --config %CONFIGURATION% -j4 &&
+      POPD &&
+      ECHO ***
+    )
+
+  test_script:
+  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET "CC=gcc"
+  - SET "CXX=g++"
+  - if [%TEST%]==[cmake] (
+      mkdir build\cmake\build &&
+      cd build\cmake\build &&
+      SET FUZZERTEST=-T2mn &&
+      SET ZSTREAM_TESTTIME=-T2mn &&
+      cmake -G "Visual Studio 14 2015 Win64" .. &&
+      cd ..\..\.. &&
+      make clean
+    )
+
+
+# The following tests are for regular pushes
+# into `dev` or some feature branch
+# There run less tests, for shorter feedback loop
+
+-
+  version: 1.0.{build}
+  environment:
+    matrix:
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
+
+    - COMPILER: "gcc"
+      HOST:     "cygwin"
+      PLATFORM: "x64"
+
+    - COMPILER: "clang-cl"
+      HOST:     "cmake-visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+      CMAKE_GENERATOR: "Visual Studio 15 2017"
+      CMAKE_GENERATOR_PLATFORM: "x64"
+      CMAKE_GENERATOR_TOOLSET: "LLVM"
+      APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
+
+  install:
+  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[cygwin] (
+      ECHO Installing Cygwin Packages &&
+      C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
+        gcc,^
+        cmake,^
+        make
+    )
+  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+    )
+
+  build_script:
+  - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%HOST%]==[cygwin] (
+      set CHERE_INVOKING=yes &&
+      set CC=%COMPILER% &&
+      C:\cygwin64\bin\bash --login -c "
+        set -e;
+        cd build/cmake;
+        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T20s -DZSTD_ZSTREAM_FLAGS=-T20s -DZSTD_FULLBENCH_FLAGS=-i0 .;
+        make VERBOSE=1 -j;
+        ctest -V -L Medium;
+      "
+    )
+  - if [%HOST%]==[cmake-visual] (
+      ECHO *** &&
+      ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
+      PUSHD build\cmake &&
+      cmake -DBUILD_TESTING=ON . &&
+      cmake --build . --config %CONFIGURATION% -j4 &&
+      POPD &&
+      ECHO ***
+    )
+  - if [%HOST%]==[visual] (
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe
+    )
+
+
+  test_script:
+  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET "FUZZERTEST=-T10s"
+  - if [%HOST%]==[mingw] (
+      set "CC=%COMPILER%" &&
+      make clean &&
+      make check
+    )

build/VS2008/zstd/zstd.vcproj

@@ -356,10 +356,6 @@
 				RelativePath="..\..\..\programs\dibio.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\programs\lorem.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\dictBuilder\cover.c"
 				>

build/VS2010/datagen/datagen.vcxproj

@@ -157,8 +157,6 @@
   <ItemGroup>
     <ClCompile Include="..\..\..\programs\util.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\lorem.c" />
-    <ClCompile Include="..\..\..\tests\loremOut.c" />
     <ClCompile Include="..\..\..\tests\datagencli.c" />
   </ItemGroup>
   <ItemGroup>

build/VS2010/fullbench-dll/fullbench-dll.vcxproj

@@ -0,0 +1,189 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>fullbench-dll</RootNamespace>
+    <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
+    <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>MultiByte</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <LinkIncremental>true</LinkIncremental>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <LinkIncremental>false</LinkIncremental>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <RunCodeAnalysis>false</RunCodeAnalysis>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level4</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <EnablePREfast>false</EnablePREfast>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level4</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <TreatWarningAsError>false</TreatWarningAsError>
+      <EnablePREfast>false</EnablePREfast>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+      <AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\lib\common\xxhash.c" />
+    <ClCompile Include="..\..\..\programs\util.c" />
+    <ClCompile Include="..\..\..\programs\timefn.c" />
+    <ClCompile Include="..\..\..\programs\datagen.c" />
+    <ClCompile Include="..\..\..\programs\benchfn.c" />
+    <ClCompile Include="..\..\..\tests\fullbench.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\lib\zstd.h" />
+    <ClInclude Include="..\..\..\programs\datagen.h" />
+    <ClInclude Include="..\..\..\programs\benchfn.h" />
+    <ClInclude Include="..\..\..\programs\util.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\libzstd-dll\libzstd-dll.vcxproj">
+      <Project>{00000000-94d5-4bf9-8a50-7bd9929a0850}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

build/VS2010/fullbench/fullbench.vcxproj

@@ -170,7 +170,6 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
@@ -184,7 +183,6 @@
     <ClCompile Include="..\..\..\programs\util.c" />
     <ClCompile Include="..\..\..\programs\timefn.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\lorem.c" />
     <ClCompile Include="..\..\..\programs\benchfn.c" />
     <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>

build/VS2010/fuzzer/fuzzer.vcxproj

@@ -170,7 +170,6 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />

build/VS2010/libzstd-dll/libzstd-dll.vcxproj

@@ -34,7 +34,6 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -170,6 +169,7 @@
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>

build/VS2010/libzstd/libzstd.vcxproj

@@ -34,7 +34,6 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -163,6 +162,7 @@
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>

build/VS2010/zstd.sln

@@ -7,6 +7,11 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcx
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}"
+	ProjectSection(ProjectDependencies) = postProject
+		{00000000-94D5-4BF9-8A50-7BD9929A0850} = {00000000-94D5-4BF9-8A50-7BD9929A0850}
+	EndProjectSection
+EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libzstd", "libzstd\libzstd.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}"

build/VS2010/zstd/zstd.vcxproj

@@ -35,7 +35,6 @@
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
-    <ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
     <ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -64,7 +63,6 @@
     <ClCompile Include="..\..\..\programs\dibio.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
     <ClCompile Include="..\..\..\programs\fileio_asyncio.c" />
-    <ClCompile Include="..\..\..\programs\lorem.c" />
     <ClCompile Include="..\..\..\programs\zstdcli.c" />
     <ClCompile Include="..\..\..\programs\zstdcli_trace.c" />
   </ItemGroup>
@@ -116,7 +114,6 @@
     <RootNamespace>zstd</RootNamespace>
     <OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
     <IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
-    <InstructionSet>NotSet</InstructionSet>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
@@ -190,7 +187,6 @@
       <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
-      <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -207,7 +203,6 @@
       <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <TreatWarningAsError>true</TreatWarningAsError>
       <EnablePREfast>false</EnablePREfast>
-      <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -227,7 +222,6 @@
       <EnablePREfast>false</EnablePREfast>
       <TreatWarningAsError>false</TreatWarningAsError>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-      <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -250,7 +244,6 @@
       <EnablePREfast>false</EnablePREfast>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
       <AdditionalOptions>/DZSTD_MULTITHREAD %(AdditionalOptions)</AdditionalOptions>
-      <EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>

build/VS_scripts/build.VSPreview.cmd

@@ -1,7 +0,0 @@
-@echo off
-
-rem build 32-bit
-call "%~p0%build.generic.cmd" preview Win32 Release v143
-
-rem build 64-bit
-call "%~p0%build.generic.cmd" preview x64 Release v143

build/VS_scripts/build.generic.cmd

@@ -2,7 +2,7 @@
 
 IF "%1%" == "" GOTO display_help
 
-SETLOCAL ENABLEDELAYEDEXPANSION
+SETLOCAL
 
 SET msbuild_version=%1
 
@@ -19,34 +19,39 @@ GOTO build
 :display_help
 
 echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
-echo   msbuild_version:          VS installed version (latest, VS2012, VS2013, VS2015, VS2017, VS2019, VS2022, ...)
+echo   msbuild_version:          VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
 echo   msbuild_platform:         Platform (x64 or Win32)
 echo   msbuild_configuration:    VS configuration (Release or Debug)
-echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, v143, ...)
+echo   msbuild_toolset:          Platform Toolset (v100, v110, v120, v140, v141, v142, ...)
 
 EXIT /B 1
 
 :build
 
 SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
+SET msbuild_vs2017community="%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe"
+SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017\Professional\MSBuild\15.0\Bin\MSBuild.exe"
+SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
 IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
-IF %msbuild_version% == VS2017 SET vswhere_params=-version [15,16) -products *
-IF %msbuild_version% == VS2017Community SET vswhere_params=-version [15,16) -products Community
-IF %msbuild_version% == VS2017Enterprise SET vswhere_params=-version [15,16) -products Enterprise
-IF %msbuild_version% == VS2017Professional SET vswhere_params=-version [15,16) -products Professional
-IF %msbuild_version% == VS2019 SET vswhere_params=-version [16,17) -products *
-IF %msbuild_version% == VS2022 SET vswhere_params=-version [17,18) -products *
-REM Add the next Visual Studio version here.
-IF %msbuild_version% == latest SET vswhere_params=-latest -products *
-IF %msbuild_version% == preview SET vswhere_params=-prerelease -products *
-
-IF NOT DEFINED vswhere_params GOTO skip_vswhere
-SET vswhere="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
-FOR /F "USEBACKQ TOKENS=*" %%F IN (`%vswhere% !vswhere_params! -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe`) DO (
-	SET msbuild="%%F"
+IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
+IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
+IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
+IF %msbuild_version% == VS2017 (
+	IF EXIST %msbuild_vs2017community% SET msbuild=%msbuild_vs2017community%
+	IF EXIST %msbuild_vs2017professional% SET msbuild=%msbuild_vs2017professional%
+	IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
+)
+
+:: VS2019
+SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
+SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
+SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
+IF %msbuild_version% == VS2019 (
+	IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
+	IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
+	IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
 )
-:skip_vswhere
 
 SET project="%~p0\..\VS2010\zstd.sln"
 

build/cmake/CMakeLists.txt

@@ -7,75 +7,206 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
+
+# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
+# Set and use the newest cmake policies that are validated to work
+set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
+set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
+if("${CMAKE_MAJOR_VERSION}" LESS 3)
+  set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+elseif( "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
+        "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
+    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
+else()
+    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
+endif()
+cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
+
+set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
 
-#-----------------------------------------------------------------------------
-# Setup CMake environment
-#-----------------------------------------------------------------------------
-set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
-
-# Define project paths
 set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
 set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
+# Parse version
+include(GetZstdLibraryVersion)
+GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
 
-#-----------------------------------------------------------------------------
-# Configure CMake policies and version
-#-----------------------------------------------------------------------------
-include(ZstdVersion)
-
-#-----------------------------------------------------------------------------
-# Project declaration
-#-----------------------------------------------------------------------------
+if( CMAKE_MAJOR_VERSION LESS 3 )
+  ## Provide cmake 3+ behavior for older versions of cmake
+  project(zstd)
+  set(PROJECT_VERSION_MAJOR ${zstd_VERSION_MAJOR})
+  set(PROJECT_VERSION_MINOR ${zstd_VERSION_MINOR})
+  set(PROJECT_VERSION_PATCH ${zstd_VERSION_PATCH})
+  set(PROJECT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
+  enable_language(C)   # Main library is in C
+  enable_language(ASM) # And ASM
+  enable_language(CXX) # Testing contributed code also utilizes CXX
+else()
   project(zstd
-    VERSION "${ZSTD_FULL_VERSION}"
-    LANGUAGES C ASM  # Main library is in C and ASM
-    HOMEPAGE_URL "${zstd_HOMEPAGE_URL}"
-    DESCRIPTION "${zstd_DESCRIPTION}"
+    VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
+    LANGUAGES C   # Main library is in C
+              ASM # And ASM
+              CXX # Testing contributed code also utilizes CXX
     )
+endif()
+message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
+set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
+set(zstd_DESCRIPTION  "Zstandard is a real-time compression algorithm, providing high compression ratios.")
 
-#-----------------------------------------------------------------------------
-# Build type configuration
-#-----------------------------------------------------------------------------
+# Set a default build type if none was specified
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   message(STATUS "Setting build type to 'Release' as none was specified.")
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
+  # Set the possible values of build type for cmake-gui
   set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
-#-----------------------------------------------------------------------------
-# Include standard modules
-#-----------------------------------------------------------------------------
 include(GNUInstallDirs)
 
 #-----------------------------------------------------------------------------
-# Display installation information
+# Add extra compilation flags
+#-----------------------------------------------------------------------------
+include(AddZstdCompilationFlags)
+ADD_ZSTD_COMPILATION_FLAGS()
+
+# Always hide XXHash symbols
+add_definitions(-DXXH_NAMESPACE=ZSTD_)
+
+#-----------------------------------------------------------------------------
+# Installation variables
 #-----------------------------------------------------------------------------
 message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
 message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
 
 #-----------------------------------------------------------------------------
-# Configure build options
+# Options
 #-----------------------------------------------------------------------------
-include(ZstdOptions)
+
+# Legacy support
+option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" ON)
+
+if (ZSTD_LEGACY_SUPPORT)
+    message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
+    set(ZSTD_LEGACY_LEVEL 5 CACHE STRING "")
+    add_definitions(-DZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_LEVEL})
+else ()
+    message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
+    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
+endif ()
+
+if (ANDROID)
+    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
+else()
+    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
+endif()
+
+# Multi-threading support
+option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
+
+if (ZSTD_MULTITHREAD_SUPPORT)
+    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
+else ()
+    message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
+endif ()
+
+option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
+option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
+
+# Respect the conventional CMake option for enabling tests if it was specified on the first configure
+if (BUILD_TESTING)
+    set(ZSTD_BUILD_TESTS_default ON)
+else()
+    set(ZSTD_BUILD_TESTS_default OFF)
+endif()
+option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
+if (MSVC)
+    option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
+endif ()
 
 #-----------------------------------------------------------------------------
-# Configure compilation flags
+# External dependencies
 #-----------------------------------------------------------------------------
-include(AddZstdCompilationFlags)
-ADD_ZSTD_COMPILATION_FLAGS(ON ZSTD_ENABLE_CXX ON)
+if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
+    set(THREADS_PREFER_PTHREAD_FLAG ON)
+    find_package(Threads REQUIRED)
+    if(CMAKE_USE_PTHREADS_INIT)
+        set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
+    else()
+        message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
+    endif()
+endif ()
 
 #-----------------------------------------------------------------------------
-# Configure dependencies
+# Add source directories
 #-----------------------------------------------------------------------------
-include(ZstdDependencies)
+add_subdirectory(lib)
+
+option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
+
+if (ZSTD_BUILD_PROGRAMS)
+    if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
+        message(SEND_ERROR "You need to build static library to build zstd CLI")
+    elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
+        message(SEND_ERROR "You need to build shared library to build zstd CLI")
+    endif ()
+
+    add_subdirectory(programs)
+endif ()
+
+if (ZSTD_BUILD_TESTS)
+    enable_testing()
+    if (NOT ZSTD_BUILD_STATIC)
+        message(SEND_ERROR "You need to build static library to build tests")
+    endif ()
+
+    add_subdirectory(tests)
+endif ()
+
+if (ZSTD_BUILD_CONTRIB)
+    add_subdirectory(contrib)
+endif ()
 
 #-----------------------------------------------------------------------------
-# Configure build targets
+# Add clean-all target
 #-----------------------------------------------------------------------------
-include(ZstdBuild)
+add_custom_target(clean-all
+   COMMAND ${CMAKE_BUILD_TOOL} clean
+   COMMAND rm -rf ${CMAKE_BINARY_DIR}/
+)
 
 #-----------------------------------------------------------------------------
-# Configure package generation
+# Generate Package Config files
+#
+# This section is based on the boiler plate code from:
+# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
 #-----------------------------------------------------------------------------
-include(ZstdPackage)
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
+    VERSION ${zstd_VERSION}
+    COMPATIBILITY SameMajorVersion
+    )
+
+# A Package Config file that works from the build directory
+export(EXPORT zstdExports
+    FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
+    NAMESPACE zstd::
+    )
+configure_file(zstdConfig.cmake
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
+    COPYONLY
+    )
+
+# A Package Config file that works from the installation directory
+set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
+install(EXPORT zstdExports
+    FILE zstdTargets.cmake
+    NAMESPACE zstd::
+    DESTINATION ${ConfigPackageLocation}
+    )
+install(FILES
+    zstdConfig.cmake
+    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
+    DESTINATION ${ConfigPackageLocation}
+    )

build/cmake/CMakeModules/AddZstdCompilationFlags.cmake

@@ -1,16 +1,6 @@
-include(CheckCCompilerFlag)
-if(CMAKE_CXX_COMPILER)
 include(CheckCXXCompilerFlag)
-endif()
-
-if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-    set(ZSTD_HAVE_CHECK_LINKER_FLAG true)
-else ()
-    set(ZSTD_HAVE_CHECK_LINKER_FLAG false)
-endif ()
-if (ZSTD_HAVE_CHECK_LINKER_FLAG)
+include(CheckCCompilerFlag)
 include(CheckLinkerFlag)
-endif()
 
 function(EnableCompilerFlag _flag _C _CXX _LD)
     string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}")
@@ -23,27 +13,14 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
             set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
-    if (_CXX AND CMAKE_CXX_COMPILER)
+    if (_CXX)
         CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
         if (CXX_FLAG_${varname})
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
         endif ()
     endif ()
     if (_LD)
-        # We never add a linker flag with CMake < 3.18. We will
-        # implement CHECK_LINKER_FLAG() like feature for CMake < 3.18
-        # or require CMake >= 3.18 when we need to add a required
-        # linker flag in future.
-        #
-        # We also skip linker flags check for MSVC compilers (which includes
-        # clang-cl) since currently check_linker_flag() doesn't give correct
-        # results for this configuration,
-        # see: https://gitlab.kitware.com/cmake/cmake/-/issues/22023
-        if (ZSTD_HAVE_CHECK_LINKER_FLAG AND NOT MSVC)
         CHECK_LINKER_FLAG(C ${_flag} LD_FLAG_${varname})
-        else ()
-            set(LD_FLAG_${varname} false)
-        endif ()
         if (LD_FLAG_${varname})
             set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
             set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
@@ -51,65 +28,49 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
     endif ()
 endfunction()
 
-macro(ADD_ZSTD_COMPILATION_FLAGS _C _CXX _LD)
-    # We set ZSTD_HAS_NOEXECSTACK if we are certain we've set all the required
-    # compiler flags to mark the stack as non-executable.
-    set(ZSTD_HAS_NOEXECSTACK false)
-
+macro(ADD_ZSTD_COMPILATION_FLAGS)
     if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
         # It's possible to select the exact standard used for compilation.
         # It's not necessary, but can be employed for specific purposes.
         # Note that zstd source code is compatible with both C++98 and above
         # and C-gnu90 (c90 + long long + variadic macros ) and above
         # EnableCompilerFlag("-std=c++11" false true) # Set C++ compilation to c++11 standard
-        # EnableCompilerFlag("-std=c99" true false)   # Set C compilation to c99 standard
+        # EnableCompilerFlag("-std=c99" true false)   # Set C compiation to c99 standard
         if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
             # clang-cl normally maps -Wall to -Weverything.
-            EnableCompilerFlag("/clang:-Wall" _C _CXX false)
+            EnableCompilerFlag("/clang:-Wall" true true false)
         else ()
-            EnableCompilerFlag("-Wall" _C _CXX false)
+            EnableCompilerFlag("-Wall" true true false)
         endif ()
-        EnableCompilerFlag("-Wextra" _C _CXX false)
-        EnableCompilerFlag("-Wundef" _C _CXX false)
-        EnableCompilerFlag("-Wshadow" _C _CXX false)
-        EnableCompilerFlag("-Wcast-align" _C _CXX false)
-        EnableCompilerFlag("-Wcast-qual" _C _CXX false)
-        EnableCompilerFlag("-Wstrict-prototypes" _C false false)
+        EnableCompilerFlag("-Wextra" true true false)
+        EnableCompilerFlag("-Wundef" true true false)
+        EnableCompilerFlag("-Wshadow" true true false)
+        EnableCompilerFlag("-Wcast-align" true true false)
+        EnableCompilerFlag("-Wcast-qual" true true false)
+        EnableCompilerFlag("-Wstrict-prototypes" true false false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("-DDEBUGLEVEL=1" _C _CXX false)
+            EnableCompilerFlag("-DDEBUGLEVEL=1" true true false)
         endif ()
         # Add noexecstack flags
         # LDFLAGS
-        EnableCompilerFlag("-Wl,-z,noexecstack" false false _LD)
+        EnableCompilerFlag("-z noexecstack" false false true)
         # CFLAGS & CXXFLAGS
-        EnableCompilerFlag("-Qunused-arguments" _C _CXX false)
-        EnableCompilerFlag("-Wa,--noexecstack" _C _CXX false)
-        # NOTE: Using 3 nested ifs because the variables are sometimes
-        # empty if the condition is false, and sometimes equal to false.
-        # This implicitly converts them to truthy values. There may be
-        # a better way to do this, but this reliably works.
-        if (${LD_FLAG_WL_Z_NOEXECSTACK})
-            if (${C_FLAG_WA_NOEXECSTACK})
-                if (${CXX_FLAG_WA_NOEXECSTACK})
-                    # We've succeeded in marking the stack as non-executable
-                    set(ZSTD_HAS_NOEXECSTACK true)
-                endif()
-            endif()
-        endif()
+        EnableCompilerFlag("-Qunused-arguments" true true false)
+        EnableCompilerFlag("-Wa,--noexecstack" true true false)
     elseif (MSVC) # Add specific compilation flags for Windows Visual
 
         set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
         if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION)
-            EnableCompilerFlag("/MP" _C _CXX false)
+            EnableCompilerFlag("/MP" true true false)
         endif ()
 
         # UNICODE SUPPORT
-        EnableCompilerFlag("/D_UNICODE" _C _CXX false)
-        EnableCompilerFlag("/DUNICODE" _C _CXX false)
+        EnableCompilerFlag("/D_UNICODE" true true false)
+        EnableCompilerFlag("/DUNICODE" true true false)
         # Enable asserts in Debug mode
         if (CMAKE_BUILD_TYPE MATCHES "Debug")
-            EnableCompilerFlag("/DDEBUGLEVEL=1" _C _CXX false)
+            EnableCompilerFlag("/DDEBUGLEVEL=1" true true false)
         endif ()
     endif ()
 

build/cmake/CMakeModules/ZstdBuild.cmake

@@ -1,42 +0,0 @@
-# ################################################################
-# ZSTD Build Targets Configuration
-# ################################################################
-
-# Always build the library first (this defines ZSTD_BUILD_STATIC/SHARED options)
-add_subdirectory(lib)
-
-# Validate build configuration after lib options are defined
-if(ZSTD_BUILD_PROGRAMS)
-    if(NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
-        message(SEND_ERROR "Static library required to build zstd CLI programs")
-    elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
-        message(SEND_ERROR "Shared library required to build zstd CLI programs")
-    endif()
-endif()
-
-if(ZSTD_BUILD_TESTS AND NOT ZSTD_BUILD_STATIC)
-    message(SEND_ERROR "Static library required to build test suite")
-endif()
-
-# Add programs if requested
-if(ZSTD_BUILD_PROGRAMS)
-    add_subdirectory(programs)
-endif()
-
-# Add tests if requested
-if(ZSTD_BUILD_TESTS)
-    enable_testing()
-    add_subdirectory(tests)
-endif()
-
-# Add contrib utilities if requested
-if(ZSTD_BUILD_CONTRIB)
-    add_subdirectory(contrib)
-endif()
-
-# Clean-all target for thorough cleanup
-add_custom_target(clean-all
-    COMMAND ${CMAKE_BUILD_TOOL} clean
-    COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/
-    COMMENT "Performing complete clean including build directory"
-)

build/cmake/CMakeModules/ZstdDependencies.cmake

@@ -1,30 +0,0 @@
-# ################################################################
-# ZSTD Dependencies Configuration
-# ################################################################
-
-# Function to handle HP-UX thread configuration
-function(setup_hpux_threads)
-    find_package(Threads)
-    if(NOT Threads_FOUND)
-        set(CMAKE_USE_PTHREADS_INIT 1 PARENT_SCOPE)
-        set(CMAKE_THREAD_LIBS_INIT -lpthread PARENT_SCOPE)
-        set(CMAKE_HAVE_THREADS_LIBRARY 1 PARENT_SCOPE)
-        set(Threads_FOUND TRUE PARENT_SCOPE)
-    endif()
-endfunction()
-
-# Configure threading support
-if(ZSTD_MULTITHREAD_SUPPORT AND UNIX)
-    if(CMAKE_SYSTEM_NAME MATCHES "HP-UX")
-        setup_hpux_threads()
-    else()
-        set(THREADS_PREFER_PTHREAD_FLAG ON)
-        find_package(Threads REQUIRED)
-    endif()
-    
-    if(CMAKE_USE_PTHREADS_INIT)
-        set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
-    else()
-        message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
-    endif()
-endif()

build/cmake/CMakeModules/ZstdOptions.cmake

@@ -1,68 +0,0 @@
-# ################################################################
-# ZSTD Build Options Configuration
-# ################################################################
-
-# Legacy support configuration
-option(ZSTD_LEGACY_SUPPORT "Enable legacy format support" ON)
-
-if(ZSTD_LEGACY_SUPPORT)
-    message(STATUS "ZSTD_LEGACY_SUPPORT enabled")
-    set(ZSTD_LEGACY_LEVEL 5 CACHE STRING "Legacy support level")
-    add_definitions(-DZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_LEVEL})
-else()
-    message(STATUS "ZSTD_LEGACY_SUPPORT disabled")
-    add_definitions(-DZSTD_LEGACY_SUPPORT=0)
-endif()
-
-# Platform-specific options
-if(APPLE)
-    option(ZSTD_FRAMEWORK "Build as Apple Framework" OFF)
-endif()
-
-# Android-specific configuration
-if(ANDROID)
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
-    # Handle old Android API levels
-    if((NOT ANDROID_PLATFORM_LEVEL) OR (ANDROID_PLATFORM_LEVEL VERSION_LESS 24))
-        message(STATUS "Configuring for old Android API - disabling fseeko/ftello")
-        add_compile_definitions(LIBC_NO_FSEEKO)
-    endif()
-else()
-    set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
-endif()
-
-# Multi-threading support
-option(ZSTD_MULTITHREAD_SUPPORT "Enable multi-threading support" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
-
-if(ZSTD_MULTITHREAD_SUPPORT)
-    message(STATUS "Multi-threading support enabled")
-else()
-    message(STATUS "Multi-threading support disabled")
-endif()
-
-# Build component options
-option(ZSTD_BUILD_PROGRAMS "Build command-line programs" ON)
-option(ZSTD_BUILD_CONTRIB "Build contrib utilities" OFF)
-option(ZSTD_PROGRAMS_LINK_SHARED "Link programs against shared library" OFF)
-
-# Test configuration
-if(BUILD_TESTING)
-    set(ZSTD_BUILD_TESTS_default ON)
-else()
-    set(ZSTD_BUILD_TESTS_default OFF)
-endif()
-option(ZSTD_BUILD_TESTS "Build test suite" ${ZSTD_BUILD_TESTS_default})
-
-# MSVC-specific options
-if(MSVC)
-    option(ZSTD_USE_STATIC_RUNTIME "Link to static runtime libraries" OFF)
-endif()
-
-# C++ support (needed for tests)
-set(ZSTD_ENABLE_CXX ${ZSTD_BUILD_TESTS})
-if(ZSTD_ENABLE_CXX)
-    enable_language(CXX)
-endif()
-
-# Set global definitions
-add_definitions(-DXXH_NAMESPACE=ZSTD_)

build/cmake/CMakeModules/ZstdPackage.cmake

@@ -1,42 +0,0 @@
-# ################################################################
-# ZSTD Package Configuration
-# ################################################################
-
-include(CMakePackageConfigHelpers)
-
-# Generate version file
-write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
-    VERSION ${zstd_VERSION}
-    COMPATIBILITY SameMajorVersion
-)
-
-# Export targets for build directory
-export(EXPORT zstdExports
-    FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
-    NAMESPACE zstd::
-)
-
-# Configure package for installation
-set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
-
-# Install exported targets
-install(EXPORT zstdExports
-    FILE zstdTargets.cmake
-    NAMESPACE zstd::
-    DESTINATION ${ConfigPackageLocation}
-)
-
-# Configure and install package config file
-configure_package_config_file(
-    zstdConfig.cmake.in
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
-    INSTALL_DESTINATION ${ConfigPackageLocation}
-)
-
-# Install config files
-install(FILES
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
-    "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
-    DESTINATION ${ConfigPackageLocation}
-)

build/cmake/CMakeModules/ZstdVersion.cmake

@@ -1,31 +0,0 @@
-# ################################################################
-# ZSTD Version Configuration
-# ################################################################
-
-# Setup CMake policy version
-set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
-set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13")
-
-# Determine appropriate policy version
-if("${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
-   "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
-    set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
-else()
-    set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
-endif()
-
-cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
-
-# Parse version from header file
-include(GetZstdLibraryVersion)
-GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
-
-# Set version variables
-set(ZSTD_SHORT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}")
-set(ZSTD_FULL_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
-
-# Project metadata
-set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
-set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
-
-message(STATUS "ZSTD VERSION: ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")

build/cmake/README.md

@@ -41,48 +41,6 @@ cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=OFF ..
 make
 ```
 
-**Apple Frameworks**
-It's generally recommended to have CMake with versions higher than 3.14 for [iOS-derived platforms](https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#id27).
-```sh
-cmake -S. -B build-cmake -DZSTD_FRAMEWORK=ON -DCMAKE_SYSTEM_NAME=iOS
-```
-Or you can utilize [iOS-CMake](https://github.com/leetal/ios-cmake) toolchain for CMake versions lower than 3.14
-```sh
-cmake -B build -G Xcode -DCMAKE_TOOLCHAIN_FILE=<Path To ios.toolchain.cmake> -DPLATFORM=OS64 -DZSTD_FRAMEWORK=ON
-```
-
-### how to use it with CMake FetchContent
-
-For all options available, you can see it on <https://github.com/facebook/zstd/blob/dev/build/cmake/lib/CMakeLists.txt>
-```cmake
-include(FetchContent)
-
-set(ZSTD_BUILD_STATIC ON)
-set(ZSTD_BUILD_SHARED OFF)
-
-FetchContent_Declare(
-    zstd
-    URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
-    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
-    SOURCE_SUBDIR build/cmake
-)
-
-FetchContent_MakeAvailable(zstd)
-
-target_link_libraries(
-    ${PROJECT_NAME}
-    PRIVATE
-    libzstd_static
-)
-
-# On windows and macos this is needed
-target_include_directories(
-    ${PROJECT_NAME}
-    PRIVATE
-    ${zstd_SOURCE_DIR}/lib
-)
-```
-
 ### referring
 [Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output)
 

build/cmake/contrib/pzstd/CMakeLists.txt

@@ -18,7 +18,6 @@ set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd)
 include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
 
 add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
-target_compile_features(pzstd PRIVATE cxx_std_11)
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG")
 set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow")
 

build/cmake/lib/CMakeLists.txt

@@ -12,70 +12,45 @@ project(libzstd C ASM)
 set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
 option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
 option(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
-option(ZSTD_BUILD_COMPRESSION "BUILD COMPRESSION MODULE" ON)
-option(ZSTD_BUILD_DECOMPRESSION "BUILD DECOMPRESSION MODULE" ON)
-option(ZSTD_BUILD_DICTBUILDER "BUILD DICTBUILDER MODULE" ON)
-option(ZSTD_BUILD_DEPRECATED "BUILD DEPRECATED MODULE" OFF)
-
-set(ZSTDLIB_VISIBLE "" CACHE STRING "Visibility for ZSTDLIB API")
-set(ZSTDERRORLIB_VISIBLE "" CACHE STRING "Visibility for ZSTDERRORLIB_VISIBLE API")
-set(ZDICTLIB_VISIBLE "" CACHE STRING "Visibility for ZDICTLIB_VISIBLE API")
-set(ZSTDLIB_STATIC_API "" CACHE STRING "Visibility for ZSTDLIB_STATIC_API API")
-set(ZDICTLIB_STATIC_API "" CACHE STRING "Visibility for ZDICTLIB_STATIC_API API")
-
-set_property(CACHE ZSTDLIB_VISIBLE PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
-set_property(CACHE ZSTDERRORLIB_VISIBLE PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
-set_property(CACHE ZDICTLIB_VISIBLE PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
-set_property(CACHE ZSTDLIB_STATIC_API PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
-set_property(CACHE ZDICTLIB_STATIC_API PROPERTY STRINGS "" "hidden" "default" "protected" "internal")
 
 if(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
     message(SEND_ERROR "You need to build at least one flavor of libzstd")
 endif()
 
+# Define library directory, where sources and header files are located
+include_directories(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
+
 file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
 file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
-file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
 if (MSVC)
+    file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
     add_compile_options(-DZSTD_DISABLE_ASM)
 else ()
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|AMD64.*|x86_64.*|X86_64.*" AND ${ZSTD_HAS_NOEXECSTACK})
-        set(DecompressSources ${DecompressSources} ${LIBRARY_DIR}/decompress/huf_decompress_amd64.S)
-    else()
-        add_compile_options(-DZSTD_DISABLE_ASM)
-    endif()
+    file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c ${LIBRARY_DIR}/decompress/*.S)
 endif ()
 file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
-file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
 
-file(GLOB PublicHeaders ${LIBRARY_DIR}/*.h)
+set(Sources
+        ${CommonSources}
+        ${CompressSources}
+        ${DecompressSources}
+        ${DictBuilderSources})
+
 file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
 file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
 file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
 file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
-file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
 
-set(Sources ${CommonSources})
-set(Headers ${PublicHeaders} ${CommonHeaders})
-if (ZSTD_BUILD_COMPRESSION)
-    set(Sources ${Sources} ${CompressSources})
-    set(Headers ${Headers} ${CompressHeaders})
-endif()
-if (ZSTD_BUILD_DECOMPRESSION)
-    set(Sources ${Sources} ${DecompressSources})
-    set(Headers ${Headers} ${DecompressHeaders})
-endif()
-if (ZSTD_BUILD_DICTBUILDER)
-    set(Sources ${Sources} ${DictBuilderSources})
-    set(Headers ${Headers} ${DictBuilderHeaders})
-endif()
-if (ZSTD_BUILD_DEPRECATED)
-    set(Sources ${Sources} ${DeprecatedSources})
-    set(Headers ${Headers} ${DeprecatedHeaders})
-endif()
+set(Headers
+        ${LIBRARY_DIR}/zstd.h
+        ${CommonHeaders}
+        ${CompressHeaders}
+        ${DecompressHeaders}
+        ${DictBuilderHeaders})
 
 if (ZSTD_LEGACY_SUPPORT)
     set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
+    include_directories(${LIBRARY_LEGACY_DIR})
 
     set(Sources ${Sources}
             ${LIBRARY_LEGACY_DIR}/zstd_v01.c
@@ -97,91 +72,38 @@ if (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 endif ()
 
-if (MSVC AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
+if (MSVC)
     set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
-else()
-    set(PlatformDependResources)
 endif ()
 
 # Explicitly set the language to C for all files, including ASM files.
 # Our assembly expects to be compiled by a C compiler, and is only enabled for
 # __GNUC__ compatible compilers. Otherwise all the ASM code is disabled by
 # macros.
-if(NOT CMAKE_ASM_COMPILER STREQUAL CMAKE_C_COMPILER)
 set_source_files_properties(${Sources} PROPERTIES LANGUAGE C)
-endif()
 
-macro (add_definition target var)
-    if (NOT ("${${var}}" STREQUAL ""))
-        target_compile_definitions(${target} PUBLIC "${var}=__attribute__((visibility(\"${${var}}\")))")
-    endif ()
-endmacro ()
-
-# Define directories containing the library's public headers
-set(PUBLIC_INCLUDE_DIRS ${LIBRARY_DIR})
-set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} /I \"${LIBRARY_DIR}\"")
 # Split project to static and shared libraries build
 set(library_targets)
 if (ZSTD_BUILD_SHARED)
     add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
-    target_include_directories(libzstd_shared INTERFACE $<BUILD_INTERFACE:${PUBLIC_INCLUDE_DIRS}>)
     list(APPEND library_targets libzstd_shared)
     if (ZSTD_MULTITHREAD_SUPPORT)
-        target_compile_definitions(libzstd_shared PUBLIC ZSTD_MULTITHREAD)
+        set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
         if (UNIX)
             target_link_libraries(libzstd_shared ${THREADS_LIBS})
         endif ()
     endif()
-    add_definition(libzstd_shared ZSTDLIB_VISIBLE)
-    add_definition(libzstd_shared ZSTDERRORLIB_VISIBLE)
-    add_definition(libzstd_shared ZDICTLIB_VISIBLE)
 endif ()
 if (ZSTD_BUILD_STATIC)
     add_library(libzstd_static STATIC ${Sources} ${Headers})
-    target_include_directories(libzstd_static INTERFACE $<BUILD_INTERFACE:${PUBLIC_INCLUDE_DIRS}>)
     list(APPEND library_targets libzstd_static)
     if (ZSTD_MULTITHREAD_SUPPORT)
-        target_compile_definitions(libzstd_static PUBLIC ZSTD_MULTITHREAD)
+        set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
         if (UNIX)
             target_link_libraries(libzstd_static ${THREADS_LIBS})
         endif ()
     endif ()
-    add_definition(libzstd_static ZSTDLIB_VISIBLE)
-    add_definition(libzstd_static ZSTDERRORLIB_VISIBLE)
-    add_definition(libzstd_static ZDICTLIB_VISIBLE)
-    add_definition(libzstd_static ZSTDLIB_STATIC_API)
-    add_definition(libzstd_static ZDICTLIB_STATIC_API)
-endif ()
-if (ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
-    if (NOT BUILD_SHARED_LIBS)
-        message(WARNING "BUILD_SHARED_LIBS is OFF, but ZSTD_BUILD_SHARED is ON and ZSTD_BUILD_STATIC is OFF, which takes precedence, so libzstd is a shared library")
-    endif ()
-    add_library(libzstd INTERFACE)
-    target_link_libraries(libzstd INTERFACE libzstd_shared)
-    list(APPEND library_targets libzstd)
-endif ()
-if (ZSTD_BUILD_STATIC AND NOT ZSTD_BUILD_SHARED)
-    if (BUILD_SHARED_LIBS)
-        message(WARNING "BUILD_SHARED_LIBS is ON, but ZSTD_BUILD_SHARED is OFF and ZSTD_BUILD_STATIC is ON, which takes precedence, is set so libzstd is a static library")
-    endif ()
-    add_library(libzstd INTERFACE)
-    target_link_libraries(libzstd INTERFACE libzstd_static)
-    list(APPEND library_targets libzstd)
-endif ()
-if (ZSTD_BUILD_SHARED AND ZSTD_BUILD_STATIC)
-    # If both ZSTD_BUILD_SHARED and ZSTD_BUILD_STATIC are set, which is the
-    # default, fallback to using BUILD_SHARED_LIBS to determine whether to
-    # set libzstd to static or shared.
-    if (BUILD_SHARED_LIBS)
-        add_library(libzstd INTERFACE)
-        target_link_libraries(libzstd INTERFACE libzstd_shared)
-        list(APPEND library_targets libzstd)
-    else ()
-        add_library(libzstd INTERFACE)
-        target_link_libraries(libzstd INTERFACE libzstd_static)
-        list(APPEND library_targets libzstd)
-    endif ()
 endif ()
 
 # Add specific compile definitions for MSVC project
@@ -207,28 +129,8 @@ if (ZSTD_BUILD_SHARED)
             libzstd_shared
             PROPERTIES
             OUTPUT_NAME zstd
-            VERSION ${ZSTD_FULL_VERSION}
+            VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
             SOVERSION ${zstd_VERSION_MAJOR})
-
-    if (ZSTD_FRAMEWORK)
-        set_target_properties(
-                libzstd_shared
-                PROPERTIES
-                FRAMEWORK TRUE
-                FRAMEWORK_VERSION "${ZSTD_FULL_VERSION}"
-                PRODUCT_BUNDLE_IDENTIFIER "github.com/facebook/zstd"
-                XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
-                PUBLIC_HEADER "${PublicHeaders}"
-                OUTPUT_NAME "zstd"
-                XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
-                XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
-                XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
-                MACOSX_FRAMEWORK_IDENTIFIER "github.com/facebook/zstd"
-                MACOSX_FRAMEWORK_BUNDLE_VERSION "${ZSTD_FULL_VERSION}"
-                MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${ZSTD_SHORT_VERSION}"
-                MACOSX_RPATH TRUE
-                RESOURCE ${PublicHeaders})
-    endif ()
 endif ()
 
 if (ZSTD_BUILD_STATIC)
@@ -237,26 +139,6 @@ if (ZSTD_BUILD_STATIC)
             PROPERTIES
             POSITION_INDEPENDENT_CODE On
             OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
-
-    if (ZSTD_FRAMEWORK)
-        set_target_properties(
-                libzstd_static
-                PROPERTIES
-                FRAMEWORK TRUE
-                FRAMEWORK_VERSION "${ZSTD_FULL_VERSION}"
-                PRODUCT_BUNDLE_IDENTIFIER "github.com/facebook/zstd/${STATIC_LIBRARY_BASE_NAME}"
-                XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
-                PUBLIC_HEADER "${PublicHeaders}"
-                OUTPUT_NAME "${STATIC_LIBRARY_BASE_NAME}"
-                XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
-                XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
-                XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
-                MACOSX_FRAMEWORK_IDENTIFIER "github.com/facebook/zstd/${STATIC_LIBRARY_BASE_NAME}"
-                MACOSX_FRAMEWORK_BUNDLE_VERSION "${ZSTD_FULL_VERSION}"
-                MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${ZSTD_SHORT_VERSION}"
-                MACOSX_RPATH TRUE
-                RESOURCE ${PublicHeaders})
-    endif ()
 endif ()
 
 # pkg-config
@@ -272,7 +154,11 @@ configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzs
 install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 
 # install target
-install(FILES ${PublicHeaders} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
+install(FILES
+    "${LIBRARY_DIR}/zstd.h"
+    "${LIBRARY_DIR}/zdict.h"
+    "${LIBRARY_DIR}/zstd_errors.h"
+    DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
 install(TARGETS ${library_targets}
     EXPORT zstdExports
@@ -281,8 +167,6 @@ install(TARGETS ${library_targets}
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
     BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    FRAMEWORK DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT runtime OPTIONAL
-    PUBLIC_HEADER DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
     )
 
 # uninstall target

build/cmake/programs/CMakeLists.txt

@@ -32,12 +32,7 @@ if (MSVC)
     set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
 endif ()
 
-file(GLOB ZSTD_PROGRAM_SRCS "${PROGRAMS_DIR}/*.c")
-if (MSVC AND ZSTD_PROGRAMS_LINK_SHARED)
-    list(APPEND ZSTD_PROGRAM_SRCS ${LIBRARY_DIR}/common/pool.c ${LIBRARY_DIR}/common/threading.c)
-endif ()
-
-add_executable(zstd ${ZSTD_PROGRAM_SRCS})
+add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PROGRAMS_DIR}/zstdcli_trace.c ${PlatformDependResources})
 target_link_libraries(zstd ${PROGRAMS_ZSTD_LINK_TARGET})
 if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
     target_link_libraries(zstd rt)
@@ -80,9 +75,7 @@ if (UNIX)
         ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1
         DESTINATION "${MAN_INSTALL_DIR}")
 
-    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c
-        ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c
-        ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c)
+    add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/fileio_asyncio.c)
     target_link_libraries(zstd-frugal ${PROGRAMS_ZSTD_LINK_TARGET})
     set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT;ZSTD_NOTRACE")
 endif ()

build/cmake/tests/CMakeLists.txt

@@ -50,18 +50,18 @@ set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs)
 set(TESTS_DIR ${ZSTD_SOURCE_DIR}/tests)
 include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder)
 
-add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${TESTS_DIR}/loremOut.c ${TESTS_DIR}/datagencli.c)
+add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
 target_link_libraries(datagen libzstd_static)
 
 #
 # fullbench
 #
-add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
+add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
 if (NOT MSVC)
     target_compile_options(fullbench PRIVATE "-Wno-deprecated-declarations")
 endif()
 target_link_libraries(fullbench libzstd_static)
-add_test(NAME fullbench COMMAND "$<TARGET_FILE:fullbench>" ${ZSTD_FULLBENCH_FLAGS})
+add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
 
 #
 # fuzzer
@@ -73,7 +73,7 @@ endif()
 target_link_libraries(fuzzer libzstd_static)
 AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
     "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
-add_test(NAME fuzzer COMMAND "$<TARGET_FILE:fuzzer>" ${ZSTD_FUZZER_FLAGS})
+add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
 # Disable the timeout since the run time is too long for the default timeout of
 # 1500 seconds and varies considerably between low-end and high-end CPUs.
 # set_tests_properties(fuzzer PROPERTIES TIMEOUT 0)
@@ -88,7 +88,7 @@ endif()
 target_link_libraries(zstreamtest libzstd_static)
 AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
     "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")
-add_test(NAME zstreamtest COMMAND "$<TARGET_FILE:zstreamtest>" ${ZSTD_ZSTREAM_FLAGS})
+add_test(NAME zstreamtest COMMAND zstreamtest ${ZSTD_ZSTREAM_FLAGS})
 
 #
 # playTests.sh
@@ -110,7 +110,7 @@ endif()
 # Label the "Medium" set of tests (see TESTING.md)
 set_property(TEST fuzzer zstreamtest playTests APPEND PROPERTY LABELS Medium)
 
-add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/lorem.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
+add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
 if (UNIX)
     target_link_libraries(paramgrill libzstd_static m) #m is math library
 else()

build/cmake/zstdConfig.cmake

@@ -0,0 +1 @@
+include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")

build/cmake/zstdConfig.cmake.in

@@ -1,10 +0,0 @@
-@PACKAGE_INIT@
-
-include(CMakeFindDependencyMacro)
-if(@ZSTD_MULTITHREAD_SUPPORT@ AND "@UNIX@")
-  find_dependency(Threads)
-endif()
-
-include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")
-
-check_required_components("zstd")

build/meson/contrib/pzstd/meson.build

@@ -20,6 +20,6 @@ pzstd = executable('pzstd',
   pzstd_sources,
   cpp_args: pzstd_warning_flags,
   include_directories: pzstd_includes,
-  dependencies: [ libzstd_internal_dep, thread_dep ],
+  dependencies: [ libzstd_dep, thread_dep ],
   override_options: ['b_ndebug=true'],
   install: true)

build/meson/lib/meson.build

@@ -30,7 +30,6 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
-  join_paths(zstd_rootdir, 'lib/compress/zstd_preSplit.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
   join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
@@ -125,7 +124,7 @@ libzstd = library('zstd',
   version: zstd_libversion)
 
 libzstd_dep = declare_dependency(link_with: libzstd,
-  include_directories: join_paths(zstd_rootdir,'lib')) # Do not expose private headers
+  include_directories: libzstd_includes)
 
 # we link to both:
 # - the shared library (for public symbols)
@@ -135,8 +134,7 @@ libzstd_dep = declare_dependency(link_with: libzstd,
 # -fvisibility=hidden means those cannot be found
 if get_option('default_library') == 'static'
   libzstd_static = libzstd
-  libzstd_internal_dep = declare_dependency(link_with: libzstd,
-      include_directories: libzstd_includes)
+  libzstd_internal_dep = libzstd_dep
 else
   if get_option('default_library') == 'shared'
     libzstd_static = static_library('zstd_objlib',
@@ -149,13 +147,11 @@ else
   if cc_id == compiler_msvc
     # msvc does not actually support linking to both, but errors out with:
     #   error LNK2005: ZSTD_<foo> already defined in zstd.lib(zstd-1.dll)
-    libzstd_internal_dep = declare_dependency(link_with: libzstd_static,
-      include_directories: libzstd_includes)
+    libzstd_internal_dep = declare_dependency(link_with: libzstd_static)
   else
     libzstd_internal_dep = declare_dependency(link_with: libzstd,
       # the static library must be linked after the shared one
-      dependencies: declare_dependency(link_with: libzstd_static),
-      include_directories: libzstd_includes)
+      dependencies: declare_dependency(link_with: libzstd_static))
   endif
 endif
 

build/meson/meson.build

@@ -10,7 +10,7 @@
 
 project('zstd',
   ['c', 'cpp'],
-  license: 'BSD-3-Clause OR GPL-2.0-only',
+  license: ['BSD', 'GPLv2'],
   default_options : [
     # There shouldn't be any need to force a C standard convention for zstd
     # but in case one would want that anyway, this can be done here.
@@ -65,6 +65,7 @@ zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
 
 # Built-in options
 use_debug = get_option('debug')
+buildtype = get_option('buildtype')
 default_library_type = get_option('default_library')
 
 # Custom options
@@ -87,13 +88,8 @@ feature_lz4 = get_option('lz4')
 # =============================================================================
 
 libm_dep = cc.find_library('m', required: false)
-if host_machine_os == os_windows
-  thread_dep = dependency('', required: false)
-  use_multi_thread = not feature_multi_thread.disabled()
-else
 thread_dep = dependency('threads', required: feature_multi_thread)
 use_multi_thread = thread_dep.found()
-endif
 # Arguments in dependency should be equivalent to those passed to pkg-config
 zlib_dep = dependency('zlib', required: feature_zlib)
 use_zlib = zlib_dep.found()
@@ -115,16 +111,10 @@ if [compiler_gcc, compiler_clang].contains(cc_id)
   if cc_id == compiler_clang
     common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
   endif
-  noexecstack_flags = ['-Wa,--noexecstack' ]
-  noexecstack_link_flags = ['-Wl,-z,noexecstack']
-  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + noexecstack_flags + ['-Wstrict-prototypes'])
-  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags + noexecstack_flags)
+  cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
+  cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
   add_project_arguments(cc_compile_flags, language : 'c')
   add_project_arguments(cxx_compile_flags, language : 'cpp')
-  cc_link_flags = cc.get_supported_link_arguments(noexecstack_link_flags)
-  cxx_link_flags = cxx.get_supported_link_arguments(noexecstack_link_flags)
-  add_project_link_arguments(cc_link_flags, language: 'c')
-  add_project_link_arguments(cxx_link_flags, language: 'cpp')
 elif cc_id == compiler_msvc
   msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
   if use_multi_thread
@@ -142,7 +132,7 @@ endif
 
 subdir('lib')
 
-if bin_programs or bin_tests
+if bin_programs
   subdir('programs')
 endif
 

build/meson/meson_options.txt

@@ -27,7 +27,7 @@ option('bin_contrib', type: 'boolean', value: false,
   description: 'Enable contrib build')
 
 option('multi_thread', type: 'feature', value: 'enabled',
-  description: 'Enable multi-threading when pthread or Windows is detected')
+  description: 'Enable multi-threading when pthread is detected')
 option('zlib', type: 'feature', value: 'auto',
   description: 'Enable zlib support')
 option('lzma', type: 'feature', value: 'auto',

build/meson/programs/meson.build

@@ -18,7 +18,6 @@ zstd_programs_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
   join_paths(zstd_rootdir, 'programs/benchzstd.c'),
   join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/lorem.c'),
   join_paths(zstd_rootdir, 'programs/dibio.c'),
   join_paths(zstd_rootdir, 'programs/zstdcli_trace.c')]
 
@@ -73,14 +72,7 @@ zstd = executable('zstd',
   c_args: zstd_c_args,
   dependencies: zstd_deps,
   export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
-  build_by_default: bin_programs,
-  install: bin_programs)
-
-if not bin_programs
-  # we generate rules to build the programs, but don't install anything
-  # so do not continue to installing scripts and manpages
-  subdir_done()
-endif
+  install: true)
 
 zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
   join_paths(zstd_rootdir, 'programs/timefn.c'),

build/meson/tests/meson.build

@@ -29,7 +29,6 @@ DECODECORPUS_TESTTIME = '-T30'
 test_includes = [ include_directories(join_paths(zstd_rootdir, 'programs')) ]
 
 testcommon_sources = [join_paths(zstd_rootdir, 'programs/datagen.c'),
-  join_paths(zstd_rootdir, 'programs/lorem.c'),
   join_paths(zstd_rootdir, 'programs/util.c'),
   join_paths(zstd_rootdir, 'programs/timefn.c'),
   join_paths(zstd_rootdir, 'programs/benchfn.c'),
@@ -44,8 +43,7 @@ testcommon_dep = declare_dependency(link_with: testcommon,
   dependencies: libzstd_deps,
   include_directories: libzstd_includes)
 
-datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c'),
-  join_paths(zstd_rootdir, 'tests/loremOut.c')]
+datagen_sources = [join_paths(zstd_rootdir, 'tests/datagencli.c')]
 datagen = executable('datagen',
   datagen_sources,
   c_args: [ '-DNDEBUG' ],
@@ -93,7 +91,7 @@ roundTripCrash = executable('roundTripCrash',
 longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
 longmatch = executable('longmatch',
   longmatch_sources,
-  dependencies: [ libzstd_internal_dep ],
+  dependencies: [ libzstd_dep ],
   install: false)
 
 invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
@@ -164,7 +162,7 @@ if host_machine_os != os_windows
       playTests_sh,
       args: opt,
       env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
-      depends: [datagen, zstd],
+      depends: [datagen],
       suite: suite,
       workdir: meson.current_build_dir(),
       timeout: 2800) # Timeout should work on HDD drive

build/single_file_libs/.gitignore

@@ -4,7 +4,6 @@ zstddeclib.c
 zstdenclib.c
 zstd.c
 zstd.h
-zstd_errors.h
 
 # test artifacts
 temp*

build/single_file_libs/build_library_test.sh

@@ -70,7 +70,6 @@ echo "Single file library creation script: PASSED"
 
 # Copy the header to here (for the tests)
 cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h
-cp "$ZSTD_SRC_ROOT/zstd_errors.h" examples/zstd_errors.h
 
 # Compile the generated output
 cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c

build/single_file_libs/zstd-in.c

@@ -69,7 +69,6 @@
 #include "compress/zstd_compress_literals.c"
 #include "compress/zstd_compress_sequences.c"
 #include "compress/zstd_compress_superblock.c"
-#include "compress/zstd_preSplit.c"
 #include "compress/zstd_compress.c"
 #include "compress/zstd_double_fast.c"
 #include "compress/zstd_fast.c"

contrib/docker/Dockerfile

@@ -1,13 +1,13 @@
 # Dockerfile
 # First image to build the binary
-FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a as builder
+FROM alpine as builder
 
 RUN apk --no-cache add make gcc libc-dev
 COPY . /src
 RUN mkdir /pkg && cd /src && make && make DESTDIR=/pkg install
 
 # Second minimal image to only keep the built binary
-FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a
+FROM alpine
 
 # Copy the built files
 COPY --from=builder /pkg /

contrib/externalSequenceProducer/main.c

@@ -27,7 +27,6 @@ do {                                                    \
 } while (0)                                             \
 
 int main(int argc, char *argv[]) {
-    int retn = 0;
     if (argc != 2) {
         printf("Usage: externalSequenceProducer <file>\n");
         return 1;
@@ -97,12 +96,12 @@ int main(int argc, char *argv[]) {
                 break;
             }
         }
-        retn = 1;
+        return 1;
     }
 
     ZSTD_freeCCtx(zc);
     free(src);
     free(dst);
     free(val);
-    return retn;
+    return 0;
 }

contrib/gen_html/Makefile

@@ -40,7 +40,7 @@ gen_html: gen_html.cpp
 
 $(ZSTDMANUAL): gen_html $(ZSTDAPI)
 	echo "Update zstd manual in /doc"
-	./gen_html$(EXT) $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
+	./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
 
 .PHONY: manual
 manual: gen_html $(ZSTDMANUAL)

contrib/gen_html/gen_html.cpp

@@ -211,7 +211,6 @@ int main(int argc, char *argv[]) {
 
     ostream << "<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\">\n<title>" << version << "</title>\n</head>\n<body>" << endl;
     ostream << "<h1>" << version << "</h1>\n";
-    ostream << "Note: the content of this file has been automatically generated by parsing \"zstd.h\" \n";
 
     ostream << "<hr>\n<a name=\"Contents\"></a><h2>Contents</h2>\n<ol>\n";
     for (size_t i=0; i<chapters.size(); i++)

contrib/largeNbDicts/largeNbDicts.c

@@ -739,8 +739,6 @@ static int benchMem(slice_collection_t dstBlocks, slice_collection_t srcBlocks,
     /* BMK_benchTimedFn may not run exactly nbRounds iterations */
     double speedAggregated =
         aggregateData(speedPerRound, roundNb + 1, metricAggregatePref);
-    free(speedPerRound);
-    
     if (metricAggregatePref == fastest)
       DISPLAY("Fastest Speed : %.1f MB/s \n", speedAggregated);
     else
@@ -1027,7 +1025,7 @@ int main (int argc, const char** argv)
     unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
     ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto;
     ZSTD_dictAttachPref_e dictAttachPref = ZSTD_dictDefaultAttach;
-    ZSTD_ParamSwitch_e prefetchCDictTables = ZSTD_ps_auto;
+    ZSTD_paramSwitch_e prefetchCDictTables = ZSTD_ps_auto;
     metricAggregatePref_e metricAggregatePref = fastest;
 
     for (int argNb = 1; argNb < argc ; argNb++) {

contrib/linux-kernel/linux.mk

@@ -26,7 +26,6 @@ zstd_compress-y := \
 		compress/zstd_lazy.o \
 		compress/zstd_ldm.o \
 		compress/zstd_opt.o \
-		compress/zstd_preSplit.o \
 
 zstd_decompress-y := \
 		zstd_decompress_module.o \

contrib/linux-kernel/linux_zstd.h

@@ -77,30 +77,6 @@ int zstd_min_clevel(void);
  */
 int zstd_max_clevel(void);
 
-/**
- * zstd_default_clevel() - default compression level
- *
- * Return: Default compression level.
- */
-int zstd_default_clevel(void);
-
-/**
- * struct zstd_custom_mem - custom memory allocation
- */
-typedef ZSTD_customMem zstd_custom_mem;
-
-/**
- * struct zstd_dict_load_method - Dictionary load method.
- * See zstd_lib.h.
- */
-typedef ZSTD_dictLoadMethod_e zstd_dict_load_method;
-
-/**
- * struct zstd_dict_content_type - Dictionary context type.
- * See zstd_lib.h.
- */
-typedef ZSTD_dictContentType_e zstd_dict_content_type;
-
 /* ======   Parameter Selection   ====== */
 
 /**
@@ -160,32 +136,9 @@ typedef ZSTD_parameters zstd_parameters;
 zstd_parameters zstd_get_params(int level,
 	unsigned long long estimated_src_size);
 
-/**
- * zstd_get_cparams() - returns zstd_compression_parameters for selected level
- * @level:              The compression level
- * @estimated_src_size: The estimated source size to compress or 0
- *                      if unknown.
- * @dict_size:          Dictionary size.
- *
- * Return:              The selected zstd_compression_parameters.
- */
-zstd_compression_parameters zstd_get_cparams(int level,
-	unsigned long long estimated_src_size, size_t dict_size);
+/* ======   Single-pass Compression   ====== */
 
 typedef ZSTD_CCtx zstd_cctx;
-typedef ZSTD_cParameter zstd_cparameter;
-
-/**
- * zstd_cctx_set_param() - sets a compression parameter
- * @cctx:         The context. Must have been initialized with zstd_init_cctx().
- * @param:        The parameter to set.
- * @value:        The value to set the parameter to.
- *
- * Return:        Zero or an error, which can be checked using zstd_is_error().
- */
-size_t zstd_cctx_set_param(zstd_cctx *cctx, zstd_cparameter param, int value);
-
-/* ======   Single-pass Compression   ====== */
 
 /**
  * zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx
@@ -200,20 +153,6 @@ size_t zstd_cctx_set_param(zstd_cctx *cctx, zstd_cparameter param, int value);
  */
 size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);
 
-/**
- * zstd_cctx_workspace_bound_with_ext_seq_prod() - max memory needed to
- * initialize a zstd_cctx when using the block-level external sequence
- * producer API.
- * @parameters: The compression parameters to be used.
- *
- * If multiple compression parameters might be used, the caller must call
- * this function for each set of parameters and use the maximum size.
- *
- * Return:      A lower bound on the size of the workspace that is passed to
- *              zstd_init_cctx().
- */
-size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *parameters);
-
 /**
  * zstd_init_cctx() - initialize a zstd compression context
  * @workspace:      The workspace to emplace the context into. It must outlive
@@ -241,71 +180,6 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size);
 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size, const zstd_parameters *parameters);
 
-/**
- * zstd_create_cctx_advanced() - Create compression context
- * @custom_mem:   Custom allocator.
- *
- * Return:        NULL on error, pointer to compression context otherwise.
- */
-zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem);
-
-/**
- * zstd_free_cctx() - Free compression context
- * @cdict:        Pointer to compression context.
- *
- * Return:        Always 0.
- */
-size_t zstd_free_cctx(zstd_cctx* cctx);
-
-/**
- * struct zstd_cdict - Compression dictionary.
- * See zstd_lib.h.
- */
-typedef ZSTD_CDict zstd_cdict;
-
-/**
- * zstd_create_cdict_byreference() - Create compression dictionary
- * @dict:              Pointer to dictionary buffer.
- * @dict_size:         Size of the dictionary buffer.
- * @dict_load_method:  Dictionary load method.
- * @dict_content_type: Dictionary content type.
- * @custom_mem:        Memory allocator.
- *
- * Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be
- * free before zstd_cdict is destroyed.
- *
- * Return:             NULL on error, pointer to compression dictionary
- *                     otherwise.
- */
-zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
-					  zstd_compression_parameters cparams,
-					  zstd_custom_mem custom_mem);
-
-/**
- * zstd_free_cdict() - Free compression dictionary
- * @cdict:        Pointer to compression dictionary.
- *
- * Return:        Always 0.
- */
-size_t zstd_free_cdict(zstd_cdict* cdict);
-
-/**
- * zstd_compress_using_cdict() - compress src into dst using a dictionary
- * @cctx:         The context. Must have been initialized with zstd_init_cctx().
- * @dst:          The buffer to compress src into.
- * @dst_capacity: The size of the destination buffer. May be any size, but
- *                ZSTD_compressBound(srcSize) is guaranteed to be large enough.
- * @src:          The data to compress.
- * @src_size:     The size of the data to compress.
- * @cdict:        The dictionary to be used.
- *
- * Return:        The compressed size or an error, which can be checked using
- *                zstd_is_error().
- */
-size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
-	size_t dst_capacity, const void *src, size_t src_size,
-	const zstd_cdict *cdict);
-
 /* ======   Single-pass Decompression   ====== */
 
 typedef ZSTD_DCtx zstd_dctx;
@@ -346,71 +220,6 @@ zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size);
 size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size);
 
-/**
- * struct zstd_ddict - Decompression dictionary.
- * See zstd_lib.h.
- */
-typedef ZSTD_DDict zstd_ddict;
-
-/**
- * zstd_create_ddict_byreference() - Create decompression dictionary
- * @dict:              Pointer to dictionary buffer.
- * @dict_size:         Size of the dictionary buffer.
- * @dict_load_method:  Dictionary load method.
- * @dict_content_type: Dictionary content type.
- * @custom_mem:        Memory allocator.
- *
- * Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be
- * free before zstd_ddict is destroyed.
- *
- * Return:             NULL on error, pointer to decompression dictionary
- *                     otherwise.
- */
-zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
-					  zstd_custom_mem custom_mem);
-/**
- * zstd_free_ddict() - Free decompression dictionary
- * @dict:         Pointer to the dictionary.
- *
- * Return:        Always 0.
- */
-size_t zstd_free_ddict(zstd_ddict *ddict);
-
-/**
- * zstd_create_dctx_advanced() - Create decompression context
- * @custom_mem:   Custom allocator.
- *
- * Return:        NULL on error, pointer to decompression context otherwise.
- */
-zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem);
-
-/**
- * zstd_free_dctx() -- Free decompression context
- * @dctx:         Pointer to decompression context.
- * Return:        Always 0.
- */
-size_t zstd_free_dctx(zstd_dctx *dctx);
-
-/**
- * zstd_decompress_using_ddict() - decompress src into dst using a dictionary
- * @dctx:         The decompression context.
- * @dst:          The buffer to decompress src into.
- * @dst_capacity: The size of the destination buffer. Must be at least as large
- *                as the decompressed size. If the caller cannot upper bound the
- *                decompressed size, then it's better to use the streaming API.
- * @src:          The zstd compressed data to decompress. Multiple concatenated
- *                frames and skippable frames are allowed.
- * @src_size:     The exact size of the data to decompress.
- * @ddict:        The dictionary to be used.
- *
- * Return:        The decompressed size or an error, which can be checked using
- *                zstd_is_error().
- */
-size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
-	void *dst, size_t dst_capacity, const void *src, size_t src_size,
-	const zstd_ddict *ddict);
-
-
 /* ======   Streaming Buffers   ====== */
 
 /**
@@ -448,16 +257,6 @@ typedef ZSTD_CStream zstd_cstream;
  */
 size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);
 
-/**
- * zstd_cstream_workspace_bound_with_ext_seq_prod() - memory needed to initialize
- * a zstd_cstream when using the block-level external sequence producer API.
- * @cparams: The compression parameters to be used for compression.
- *
- * Return:   A lower bound on the size of the workspace that is passed to
- *           zstd_init_cstream().
- */
-size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *cparams);
-
 /**
  * zstd_init_cstream() - initialize a zstd streaming compression context
  * @parameters        The zstd parameters to use for compression.
@@ -617,18 +416,6 @@ size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
  */
 size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);
 
-/**
- * zstd_register_sequence_producer() - exposes the zstd library function
- * ZSTD_registerSequenceProducer(). This is used for the block-level external
- * sequence producer API. See upstream zstd.h for detailed documentation.
- */
-typedef ZSTD_sequenceProducer_F zstd_sequence_producer_f;
-void zstd_register_sequence_producer(
-  zstd_cctx *cctx,
-  void* sequence_producer_state,
-  zstd_sequence_producer_f sequence_producer
-);
-
 /**
  * struct zstd_frame_params - zstd frame parameters stored in the frame header
  * @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
@@ -642,7 +429,7 @@ void zstd_register_sequence_producer(
  *
  * See zstd_lib.h.
  */
-typedef ZSTD_FrameHeader zstd_frame_header;
+typedef ZSTD_frameHeader zstd_frame_header;
 
 /**
  * zstd_get_frame_header() - extracts parameters from a zstd or skippable frame
@@ -657,35 +444,4 @@ typedef ZSTD_FrameHeader zstd_frame_header;
 size_t zstd_get_frame_header(zstd_frame_header *params, const void *src,
 	size_t src_size);
 
-/**
- * struct zstd_sequence - a sequence of literals or a match
- *
- * @offset: The offset of the match
- * @litLength: The literal length of the sequence
- * @matchLength: The match length of the sequence
- * @rep: Represents which repeat offset is used
- */
-typedef ZSTD_Sequence zstd_sequence;
-
-/**
- * zstd_compress_sequences_and_literals() - compress an array of zstd_sequence and literals
- *
- * @cctx: The zstd compression context.
- * @dst: The buffer to compress the data into.
- * @dst_capacity: The size of the destination buffer.
- * @in_seqs: The array of zstd_sequence to compress.
- * @in_seqs_size: The number of sequences in in_seqs.
- * @literals: The literals associated to the sequences to be compressed.
- * @lit_size: The size of the literals in the literals buffer.
- * @lit_capacity: The size of the literals buffer.
- * @decompressed_size: The size of the input data
- *
- * Return: The compressed size or an error, which can be checked using
- * 	   zstd_is_error().
- */
-size_t zstd_compress_sequences_and_literals(zstd_cctx *cctx, void* dst, size_t dst_capacity,
-					    const zstd_sequence *in_seqs, size_t in_seqs_size,
-					    const void* literals, size_t lit_size, size_t lit_capacity,
-					    size_t decompressed_size);
-
 #endif  /* LINUX_ZSTD_H */

contrib/linux-kernel/mem.h

@@ -15,7 +15,7 @@
 /*-****************************************
 *  Dependencies
 ******************************************/
-#include <linux/unaligned.h>  /* get_unaligned, put_unaligned* */
+#include <asm/unaligned.h>  /* get_unaligned, put_unaligned* */
 #include <linux/compiler.h>  /* inline */
 #include <linux/swab.h>  /* swab32, swab64 */
 #include <linux/types.h>  /* size_t, ptrdiff_t */
@@ -24,7 +24,6 @@
 /*-****************************************
 *  Compiler specifics
 ******************************************/
-#undef MEM_STATIC /* may be already defined from common/compiler.h */
 #define MEM_STATIC static inline
 
 /*-**************************************************************

contrib/linux-kernel/test/include/linux/xxhash.h

@@ -296,7 +296,7 @@ XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state
  * - xxHash source repository: https://github.com/Cyan4973/xxHash
  */
 
-#include <linux/unaligned.h>
+#include <asm/unaligned.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/module.h>

contrib/linux-kernel/zstd_common_module.c

@@ -24,6 +24,9 @@ EXPORT_SYMBOL_GPL(HUF_readStats_wksp);
 EXPORT_SYMBOL_GPL(ZSTD_isError);
 EXPORT_SYMBOL_GPL(ZSTD_getErrorName);
 EXPORT_SYMBOL_GPL(ZSTD_getErrorCode);
+EXPORT_SYMBOL_GPL(ZSTD_customMalloc);
+EXPORT_SYMBOL_GPL(ZSTD_customCalloc);
+EXPORT_SYMBOL_GPL(ZSTD_customFree);
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Common");

contrib/linux-kernel/zstd_compress_module.c

@@ -16,7 +16,6 @@
 
 #include "common/zstd_deps.h"
 #include "common/zstd_internal.h"
-#include "compress/zstd_compress_internal.h"
 
 #define ZSTD_FORWARD_IF_ERR(ret)            \
 	do {                                \
@@ -67,12 +66,6 @@ int zstd_max_clevel(void)
 }
 EXPORT_SYMBOL(zstd_max_clevel);
 
-int zstd_default_clevel(void)
-{
-	return ZSTD_defaultCLevel();
-}
-EXPORT_SYMBOL(zstd_default_clevel);
-
 size_t zstd_compress_bound(size_t src_size)
 {
 	return ZSTD_compressBound(src_size);
@@ -86,71 +79,12 @@ zstd_parameters zstd_get_params(int level,
 }
 EXPORT_SYMBOL(zstd_get_params);
 
-zstd_compression_parameters zstd_get_cparams(int level,
-	unsigned long long estimated_src_size, size_t dict_size)
-{
-	return ZSTD_getCParams(level, estimated_src_size, dict_size);
-}
-EXPORT_SYMBOL(zstd_get_cparams);
-
-size_t zstd_cctx_set_param(zstd_cctx *cctx, ZSTD_cParameter param, int value)
-{
-	return ZSTD_CCtx_setParameter(cctx, param, value);
-}
-EXPORT_SYMBOL(zstd_cctx_set_param);
-
 size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
 {
 	return ZSTD_estimateCCtxSize_usingCParams(*cparams);
 }
 EXPORT_SYMBOL(zstd_cctx_workspace_bound);
 
-// Used by zstd_cctx_workspace_bound_with_ext_seq_prod()
-static size_t dummy_external_sequence_producer(
-	void *sequenceProducerState,
-	ZSTD_Sequence *outSeqs, size_t outSeqsCapacity,
-	const void *src, size_t srcSize,
-	const void *dict, size_t dictSize,
-	int compressionLevel,
-	size_t windowSize)
-{
-	(void)sequenceProducerState;
-	(void)outSeqs; (void)outSeqsCapacity;
-	(void)src; (void)srcSize;
-	(void)dict; (void)dictSize;
-	(void)compressionLevel;
-	(void)windowSize;
-	return ZSTD_SEQUENCE_PRODUCER_ERROR;
-}
-
-static void init_cctx_params_from_compress_params(
-	ZSTD_CCtx_params *cctx_params,
-	const zstd_compression_parameters *compress_params)
-{
-	ZSTD_parameters zstd_params;
-	memset(&zstd_params, 0, sizeof(zstd_params));
-	zstd_params.cParams = *compress_params;
-	ZSTD_CCtxParams_init_advanced(cctx_params, zstd_params);
-}
-
-size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
-{
-	ZSTD_CCtx_params cctx_params;
-	init_cctx_params_from_compress_params(&cctx_params, compress_params);
-	ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
-	return ZSTD_estimateCCtxSize_usingCCtxParams(&cctx_params);
-}
-EXPORT_SYMBOL(zstd_cctx_workspace_bound_with_ext_seq_prod);
-
-size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
-{
-	ZSTD_CCtx_params cctx_params;
-	init_cctx_params_from_compress_params(&cctx_params, compress_params);
-	ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
-	return ZSTD_estimateCStreamSize_usingCCtxParams(&cctx_params);
-}
-EXPORT_SYMBOL(zstd_cstream_workspace_bound_with_ext_seq_prod);
-
 zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 {
 	if (workspace == NULL)
@@ -159,33 +93,6 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
 }
 EXPORT_SYMBOL(zstd_init_cctx);
 
-zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem)
-{
-	return ZSTD_createCCtx_advanced(custom_mem);
-}
-EXPORT_SYMBOL(zstd_create_cctx_advanced);
-
-size_t zstd_free_cctx(zstd_cctx *cctx)
-{
-	return ZSTD_freeCCtx(cctx);
-}
-EXPORT_SYMBOL(zstd_free_cctx);
-
-zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
-					  zstd_compression_parameters cparams,
-					  zstd_custom_mem custom_mem)
-{
-	return ZSTD_createCDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
-					 ZSTD_dct_auto, cparams, custom_mem);
-}
-EXPORT_SYMBOL(zstd_create_cdict_byreference);
-
-size_t zstd_free_cdict(zstd_cdict *cdict)
-{
-	return ZSTD_freeCDict(cdict);
-}
-EXPORT_SYMBOL(zstd_free_cdict);
-
 size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 	const void *src, size_t src_size, const zstd_parameters *parameters)
 {
@@ -194,15 +101,6 @@ size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
 }
 EXPORT_SYMBOL(zstd_compress_cctx);
 
-size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
-	size_t dst_capacity, const void *src, size_t src_size,
-	const ZSTD_CDict *cdict)
-{
-	return ZSTD_compress_usingCDict(cctx, dst, dst_capacity,
-					src, src_size, cdict);
-}
-EXPORT_SYMBOL(zstd_compress_using_cdict);
-
 size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
 {
 	return ZSTD_estimateCStreamSize_usingCParams(*cparams);
@@ -262,25 +160,5 @@ size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
 }
 EXPORT_SYMBOL(zstd_end_stream);
 
-void zstd_register_sequence_producer(
-  zstd_cctx *cctx,
-  void* sequence_producer_state,
-  zstd_sequence_producer_f sequence_producer
-) {
-	ZSTD_registerSequenceProducer(cctx, sequence_producer_state, sequence_producer);
-}
-EXPORT_SYMBOL(zstd_register_sequence_producer);
-
-size_t zstd_compress_sequences_and_literals(zstd_cctx *cctx, void* dst, size_t dst_capacity,
-					    const zstd_sequence *in_seqs, size_t in_seqs_size,
-					    const void* literals, size_t lit_size, size_t lit_capacity,
-					    size_t decompressed_size)
-{
-	return ZSTD_compressSequencesAndLiterals(cctx, dst, dst_capacity, in_seqs,
-						 in_seqs_size, literals, lit_size,
-						 lit_capacity, decompressed_size);
-}
-EXPORT_SYMBOL(zstd_compress_sequences_and_literals);
-
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Compressor");

contrib/linux-kernel/zstd_decompress_module.c

@@ -44,33 +44,6 @@ size_t zstd_dctx_workspace_bound(void)
 }
 EXPORT_SYMBOL(zstd_dctx_workspace_bound);
 
-zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem)
-{
-	return ZSTD_createDCtx_advanced(custom_mem);
-}
-EXPORT_SYMBOL(zstd_create_dctx_advanced);
-
-size_t zstd_free_dctx(zstd_dctx *dctx)
-{
-	return ZSTD_freeDCtx(dctx);
-}
-EXPORT_SYMBOL(zstd_free_dctx);
-
-zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
-					  zstd_custom_mem custom_mem)
-{
-	return ZSTD_createDDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
-					 ZSTD_dct_auto, custom_mem);
-
-}
-EXPORT_SYMBOL(zstd_create_ddict_byreference);
-
-size_t zstd_free_ddict(zstd_ddict *ddict)
-{
-	return ZSTD_freeDDict(ddict);
-}
-EXPORT_SYMBOL(zstd_free_ddict);
-
 zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size)
 {
 	if (workspace == NULL)
@@ -86,15 +59,6 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
 }
 EXPORT_SYMBOL(zstd_decompress_dctx);
 
-size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
-	void *dst, size_t dst_capacity, const void* src, size_t src_size,
-	const zstd_ddict* ddict)
-{
-	return ZSTD_decompress_usingDDict(dctx, dst, dst_capacity, src,
-					  src_size, ddict);
-}
-EXPORT_SYMBOL(zstd_decompress_using_ddict);
-
 size_t zstd_dstream_workspace_bound(size_t max_window_size)
 {
 	return ZSTD_estimateDStreamSize(max_window_size);
@@ -113,7 +77,7 @@ EXPORT_SYMBOL(zstd_init_dstream);
 
 size_t zstd_reset_dstream(zstd_dstream *dstream)
 {
-	return ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only);
+	return ZSTD_resetDStream(dstream);
 }
 EXPORT_SYMBOL(zstd_reset_dstream);
 

contrib/linux-kernel/zstd_deps.h

@@ -84,7 +84,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 
 #include <linux/kernel.h>
 
-#define assert(x) WARN_ON(!(x))
+#define assert(x) WARN_ON((x))
 
 #endif /* ZSTD_DEPS_ASSERT */
 #endif /* ZSTD_DEPS_NEED_ASSERT */
@@ -115,7 +115,11 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 #ifndef ZSTD_DEPS_STDINT
 #define ZSTD_DEPS_STDINT
 
-/* intptr_t already provided by ZSTD_DEPS_COMMON */
+/*
+ * The Linux Kernel doesn't provide intptr_t, only uintptr_t, which
+ * is an unsigned long.
+ */
+typedef long intptr_t;
 
 #endif /* ZSTD_DEPS_STDINT */
 #endif /* ZSTD_DEPS_NEED_STDINT */

contrib/pzstd/Logging.h

@@ -10,7 +10,6 @@
 
 #include <cstdio>
 #include <mutex>
-#include <chrono>
 
 namespace pzstd {
 

contrib/pzstd/Makefile

@@ -10,7 +10,7 @@
 # Standard variables for installation
 DESTDIR ?=
 PREFIX  ?= /usr/local
-BINDIR  := $(PREFIX)/bin
+BINDIR  := $(DESTDIR)$(PREFIX)/bin
 
 ZSTDDIR = ../../lib
 PROGDIR = ../../programs
@@ -37,13 +37,10 @@ CFLAGS   += -Wno-deprecated-declarations
 PZSTD_INC  = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
 GTEST_INC  = -isystem googletest/googletest/include
 
-# Set the minimum required by gtest
-PZSTD_CXX_STD := -std=c++14
-
 PZSTD_CPPFLAGS  = $(PZSTD_INC)
 PZSTD_CCXXFLAGS =
 PZSTD_CFLAGS    = $(PZSTD_CCXXFLAGS)
-PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS) $(PZSTD_CXX_STD)
+PZSTD_CXXFLAGS  = $(PZSTD_CCXXFLAGS) -std=c++11
 PZSTD_LDFLAGS   =
 EXTRA_FLAGS     =
 ALL_CFLAGS      = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS)   $(PZSTD_CFLAGS)
@@ -109,12 +106,12 @@ check:
 .PHONY: install
 install: PZSTD_CPPFLAGS += -DNDEBUG
 install: pzstd$(EXT)
-	install -d -m 755 $(DESTDIR)$(BINDIR)/
-	install -m 755 pzstd$(EXT) $(DESTDIR)$(BINDIR)/pzstd$(EXT)
+	install -d -m 755 $(BINDIR)/
+	install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT)
 
 .PHONY: uninstall
 uninstall:
-	$(RM) $(DESTDIR)$(BINDIR)/pzstd$(EXT)
+	$(RM) $(BINDIR)/pzstd$(EXT)
 
 # Targets for many different builds
 .PHONY: all

contrib/pzstd/Options.cpp

@@ -322,7 +322,7 @@ Options::Status Options::parse(int argc, const char **argv) {
   g_utilDisplayLevel = verbosity;
   // Remove local input files that are symbolic links
   if (!followLinks) {
-	  localInputFiles.erase(std::remove_if(localInputFiles.begin(), localInputFiles.end(),
+      std::remove_if(localInputFiles.begin(), localInputFiles.end(),
                      [&](const char *path) {
                         bool isLink = UTIL_isLink(path);
                         if (isLink && verbosity >= 2) {
@@ -332,7 +332,7 @@ Options::Status Options::parse(int argc, const char **argv) {
                                     path);
                         }
                         return isLink;
-					 }), localInputFiles.end());
+                    });
   }
 
   // Translate input files/directories into files to (de)compress

contrib/pzstd/Pzstd.cpp

@@ -269,10 +269,7 @@ static void compress(
     std::shared_ptr<BufferWorkQueue> out,
     size_t maxInputSize) {
   auto& errorHolder = state.errorHolder;
-  auto guard = makeScopeGuard([&] {
-    in->finish();
-    out->finish();
-  });
+  auto guard = makeScopeGuard([&] { out->finish(); });
   // Initialize the CCtx
   auto ctx = state.cStreamPool->get();
   if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
@@ -434,10 +431,7 @@ static void decompress(
     std::shared_ptr<BufferWorkQueue> in,
     std::shared_ptr<BufferWorkQueue> out) {
   auto& errorHolder = state.errorHolder;
-  auto guard = makeScopeGuard([&] {
-    in->finish();
-    out->finish();
-  });
+  auto guard = makeScopeGuard([&] { out->finish(); });
   // Initialize the DCtx
   auto ctx = state.dStreamPool->get();
   if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
@@ -584,7 +578,6 @@ std::uint64_t writeFile(
     FILE* outputFd,
     bool decompress) {
   auto& errorHolder = state.errorHolder;
-  auto outsFinishGuard = makeScopeGuard([&outs] { outs.finish(); });
   auto lineClearGuard = makeScopeGuard([&state] {
     state.log.clear(kLogInfo);
   });
@@ -592,7 +585,6 @@ std::uint64_t writeFile(
   std::shared_ptr<BufferWorkQueue> out;
   // Grab the output queue for each decompression job (in order).
   while (outs.pop(out)) {
-    auto outFinishGuard = makeScopeGuard([&out] { out->finish(); });
     if (errorHolder.hasError()) {
       continue;
     }

contrib/pzstd/test/PzstdTest.cpp

@@ -7,7 +7,9 @@
  * in the COPYING file in the root directory of this source tree).
  */
 #include "Pzstd.h"
+extern "C" {
 #include "datagen.h"
+}
 #include "test/RoundTrip.h"
 #include "utils/ScopeGuard.h"
 

contrib/pzstd/test/RoundTripTest.cpp

@@ -6,7 +6,9 @@
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
  */
+extern "C" {
 #include "datagen.h"
+}
 #include "Options.h"
 #include "test/RoundTrip.h"
 #include "utils/ScopeGuard.h"

contrib/pzstd/utils/WorkQueue.h

@@ -115,14 +115,13 @@ class WorkQueue {
   }
 
   /**
-   * Promise that either the reader side or the writer side is done.
-   * If the writer is done, `push()` won't be called again, so once the queue
-   * is empty there will never be any more work. If the reader is done, `pop()`
-   * won't be called again, so further items pushed will just be ignored.
+   * Promise that `push()` won't be called again, so once the queue is empty
+   * there will never any more work.
    */
   void finish() {
     {
       std::lock_guard<std::mutex> lock(mutex_);
+      assert(!done_);
       done_ = true;
     }
     readerCv_.notify_all();

contrib/seekable_format/README.md

@@ -1,42 +0,0 @@
-# Zstandard Seekable Format
-
-The seekable format splits compressed data into a series of independent "frames",
-each compressed individually,
-so that decompression of a section in the middle of an archive
-only requires zstd to decompress at most a frame's worth of extra data,
-instead of the entire archive.
-
-The frames are appended, so that the decompression of the entire payload
-still regenerates the original content, using any compliant zstd decoder.
-
-On top of that, the seekable format generates a jump table,
-which makes it possible to jump directly to the position of the relevant frame
-when requesting only a segment of the data.
-The jump table is simply ignored by zstd decoders unaware of the seekable format.
-
-The format is delivered with an API to create seekable archives
-and to retrieve arbitrary segments inside the archive.
-
-### Maximum Frame Size parameter
-
-When creating a seekable archive, the main parameter is the maximum frame size.
-
-At compression time, user can manually select the boundaries between segments,
-but they don't have to: long segments will be automatically split
-when larger than selected maximum frame size.
-
-Small frame sizes reduce decompression cost when requesting small segments,
-because the decoder will nonetheless have to decompress an entire frame
-to recover just a single byte from it.
-
-A good rule of thumb is to select a maximum frame size roughly equivalent
-to the access pattern when it's known.
-For example, if the application tends to request 4KB blocks,
-then it's a good idea to set a maximum frame size in the vicinity of 4 KB.
-
-But small frame sizes also reduce compression ratio,
-and increase the cost for the jump table,
-so there is a balance to find.
-
-In general, try to avoid really tiny frame sizes (<1 KB),
-which would have a large negative impact on compression ratio.

contrib/seekable_format/examples/Makefile

@@ -11,8 +11,6 @@
 
 ZSTDLIB_PATH = ../../../lib
 ZSTDLIB_NAME = libzstd.a
-# Parallel tools only make sense against multi-threaded libzstd
-ZSTDLIB_TARGET = $(ZSTDLIB_NAME)-mt
 ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
 
 CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
@@ -30,7 +28,7 @@ all: seekable_compression seekable_decompression seekable_decompression_mem \
 	parallel_processing
 
 $(ZSTDLIB):
-	$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_TARGET)
+	make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
 
 seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@

contrib/seekable_format/examples/parallel_compression.c

@@ -23,11 +23,9 @@
 
 #include "xxhash.h"
 
-#define ZSTD_MULTITHREAD 1
-#include "threading.h"
 #include "pool.h"      // use zstd thread pool for demo
 
-#include "../zstd_seekable.h"
+#include "zstd_seekable.h"
 
 static void* malloc_orDie(size_t size)
 {
@@ -74,87 +72,114 @@ static size_t fclose_orDie(FILE* file)
     exit(6);
 }
 
-struct state {
-    FILE* fout;
-    ZSTD_pthread_mutex_t mutex;
-    size_t nextID;
-    struct job* pending;
-    ZSTD_frameLog* frameLog;
-    const int compressionLevel;
-};
+static void fseek_orDie(FILE* file, long int offset, int origin)
+{
+    if (!fseek(file, offset, origin)) {
+        if (!fflush(file)) return;
+    }
+    /* error */
+    perror("fseek");
+    exit(7);
+}
+
+static long int ftell_orDie(FILE* file)
+{
+    long int off = ftell(file);
+    if (off != -1) return off;
+    /* error */
+    perror("ftell");
+    exit(8);
+}
 
 struct job {
-    size_t id;
-    struct job* next;
-    struct state* state;
-
-    void* src;
+    const void* src;
     size_t srcSize;
     void* dst;
     size_t dstSize;
 
     unsigned checksum;
+
+    int compressionLevel;
+    int done;
 };
 
-static void addPending_inmutex(struct state* state, struct job* job)
-{
-    struct job** p = &state->pending;
-    while (*p && (*p)->id < job->id)
-        p = &(*p)->next;
-    job->next = *p;
-    *p = job;
-}
-
-static void flushFrame(struct state* state, struct job* job)
-{
-    fwrite_orDie(job->dst, job->dstSize, state->fout);
-    free(job->dst);
-
-    size_t ret = ZSTD_seekable_logFrame(state->frameLog, (unsigned)job->dstSize, (unsigned)job->srcSize, job->checksum);
-    if (ZSTD_isError(ret)) {
-        fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret));
-        exit(12);
-    }
-}
-
-static void flushPending_inmutex(struct state* state)
-{
-    while (state->pending && state->pending->id == state->nextID) {
-        struct job* p = state->pending;
-        state->pending = p->next;
-        flushFrame(state, p);
-        free(p);
-        state->nextID++;
-    }
-}
-
-static void finishFrame(struct job* job)
-{
-    struct state *state = job->state;
-    ZSTD_pthread_mutex_lock(&state->mutex);
-    addPending_inmutex(state, job);
-    flushPending_inmutex(state);
-    ZSTD_pthread_mutex_unlock(&state->mutex);
-}
-
 static void compressFrame(void* opaque)
 {
     struct job* job = opaque;
 
-    job->checksum = (unsigned)XXH64(job->src, job->srcSize, 0);
+    job->checksum = XXH64(job->src, job->srcSize, 0);
 
-    size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->state->compressionLevel);
+    size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->compressionLevel);
     if (ZSTD_isError(ret)) {
         fprintf(stderr, "ZSTD_compress() error : %s \n", ZSTD_getErrorName(ret));
         exit(20);
     }
+
     job->dstSize = ret;
+    job->done = 1;
+}
 
-    // No longer need
-    free(job->src);
-    job->src = NULL;
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize, int nbThreads)
+{
+    POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
+    if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
 
-    finishFrame(job);
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+
+    if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
+    unsigned dstSize = ZSTD_compressBound(frameSize);
+
+
+    fseek_orDie(fin, 0, SEEK_END);
+    long int length = ftell_orDie(fin);
+    fseek_orDie(fin, 0, SEEK_SET);
+
+    size_t numFrames = (length + frameSize - 1) / frameSize;
+
+    struct job* jobs = malloc_orDie(sizeof(struct job) * numFrames);
+
+    size_t i;
+    for(i = 0; i < numFrames; i++) {
+        void* in = malloc_orDie(frameSize);
+        void* out = malloc_orDie(dstSize);
+
+        size_t inSize = fread_orDie(in, frameSize, fin);
+
+        jobs[i].src = in;
+        jobs[i].srcSize = inSize;
+        jobs[i].dst = out;
+        jobs[i].dstSize = dstSize;
+        jobs[i].compressionLevel = cLevel;
+        jobs[i].done = 0;
+        POOL_add(pool, compressFrame, &jobs[i]);
+    }
+
+    ZSTD_frameLog* fl = ZSTD_seekable_createFrameLog(1);
+    if (fl == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
+    for (i = 0; i < numFrames; i++) {
+        while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
+        fwrite_orDie(jobs[i].dst, jobs[i].dstSize, fout);
+        free((void*)jobs[i].src);
+        free(jobs[i].dst);
+
+        size_t ret = ZSTD_seekable_logFrame(fl, jobs[i].dstSize, jobs[i].srcSize, jobs[i].checksum);
+        if (ZSTD_isError(ret)) { fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret)); }
+    }
+
+    {   unsigned char seekTableBuff[1024];
+        ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
+        while (ZSTD_seekable_writeSeekTable(fl, &out) != 0) {
+            fwrite_orDie(seekTableBuff, out.pos, fout);
+            out.pos = 0;
+        }
+        fwrite_orDie(seekTableBuff, out.pos, fout);
+    }
+
+    ZSTD_seekable_freeFrameLog(fl);
+    free(jobs);
+    fclose_orDie(fout);
+    fclose_orDie(fin);
 }
 
 static const char* createOutFilename_orDie(const char* filename)
@@ -168,72 +193,6 @@ static const char* createOutFilename_orDie(const char* filename)
     return (const char*)outSpace;
 }
 
-static void openInOut_orDie(const char* fname, FILE** fin, FILE** fout) {
-    if (strcmp(fname, "-") == 0) {
-        *fin = stdin;
-        *fout = stdout;
-    } else {
-        *fin = fopen_orDie(fname, "rb");
-        const char* outName = createOutFilename_orDie(fname);
-        *fout = fopen_orDie(outName, "wb");
-    }
-}
-
-static void compressFile_orDie(const char* fname, int cLevel, unsigned frameSize, size_t nbThreads)
-{
-    struct state state = {
-        .nextID = 0,
-        .pending = NULL,
-        .compressionLevel = cLevel,
-    };
-    ZSTD_pthread_mutex_init(&state.mutex, NULL);
-    state.frameLog = ZSTD_seekable_createFrameLog(1);
-    if (state.frameLog == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
-
-    POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
-    if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
-
-    FILE* fin;
-    openInOut_orDie(fname, &fin, &state.fout);
-
-    if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
-    size_t dstSize = ZSTD_compressBound(frameSize);
-
-    for (size_t id = 0; 1; id++) {
-        struct job* job = malloc_orDie(sizeof(struct job));
-        job->id = id;
-        job->next = NULL;
-        job->state = &state;
-        job->src = malloc_orDie(frameSize);
-        job->dst = malloc_orDie(dstSize);
-        job->srcSize = fread_orDie(job->src, frameSize, fin);
-        job->dstSize = dstSize; 
-        POOL_add(pool, compressFrame, job);
-        if (feof(fin))
-            break;
-    }
-
-    POOL_joinJobs(pool);
-    POOL_free(pool);
-    if (state.pending) {
-        fprintf(stderr, "Unexpected leftover output blocks!\n");
-        exit(13);
-    }
-
-    {   unsigned char seekTableBuff[1024];
-        ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
-        while (ZSTD_seekable_writeSeekTable(state.frameLog, &out) != 0) {
-            fwrite_orDie(seekTableBuff, out.pos, state.fout);
-            out.pos = 0;
-        }
-        fwrite_orDie(seekTableBuff, out.pos, state.fout);
-    }
-
-    ZSTD_seekable_freeFrameLog(state.frameLog);
-    fclose_orDie(state.fout);
-    fclose_orDie(fin);
-}
-
 int main(int argc, const char** argv) {
     const char* const exeName = argv[0];
     if (argc!=4) {
@@ -245,9 +204,10 @@ int main(int argc, const char** argv) {
 
     {   const char* const inFileName = argv[1];
         unsigned const frameSize = (unsigned)atoi(argv[2]);
-        size_t const nbThreads = (size_t)atoi(argv[3]);
+        int const nbThreads = atoi(argv[3]);
 
-        compressFile_orDie(inFileName, 5, frameSize, nbThreads);
+        const char* const outFileName = createOutFilename_orDie(inFileName);
+        compressFile_orDie(inFileName, outFileName, 5, frameSize, nbThreads);
     }
 
     return 0;

contrib/seekable_format/examples/parallel_processing.c

@@ -19,7 +19,7 @@
 #define ZSTD_STATIC_LINKING_ONLY
 #include <zstd.h>      // presumes zstd library is installed
 #include <zstd_errors.h>
-#if defined(_WIN32)
+#if defined(WIN32) || defined(_WIN32)
 #  include <windows.h>
 #  define SLEEP(x) Sleep(x)
 #else
@@ -29,7 +29,7 @@
 
 #include "pool.h"      // use zstd thread pool for demo
 
-#include "../zstd_seekable.h"
+#include "zstd_seekable.h"
 
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 
@@ -100,11 +100,13 @@ struct sum_job {
     const char* fname;
     unsigned long long sum;
     unsigned frameNb;
+    int done;
 };
 
 static void sumFrame(void* opaque)
 {
     struct sum_job* job = (struct sum_job*)opaque;
+    job->done = 0;
 
     FILE* const fin = fopen_orDie(job->fname, "rb");
 
@@ -126,6 +128,7 @@ static void sumFrame(void* opaque)
         sum += data[i];
     }
     job->sum = sum;
+    job->done = 1;
 
     fclose(fin);
     ZSTD_seekable_free(seekable);
@@ -150,14 +153,14 @@ static void sumFile_orDie(const char* fname, int nbThreads)
 
     unsigned fnb;
     for (fnb = 0; fnb < numFrames; fnb++) {
-        jobs[fnb] = (struct sum_job){ fname, 0, fnb };
+        jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
         POOL_add(pool, sumFrame, &jobs[fnb]);
     }
-    POOL_joinJobs(pool);
 
     unsigned long long total = 0;
 
     for (fnb = 0; fnb < numFrames; fnb++) {
+        while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
         total += jobs[fnb].sum;
     }
 

contrib/seekable_format/examples/seekable_compression.c

@@ -13,7 +13,7 @@
 #define ZSTD_STATIC_LINKING_ONLY
 #include <zstd.h>      // presumes zstd library is installed
 
-#include "../zstd_seekable.h"
+#include "zstd_seekable.h"
 
 static void* malloc_orDie(size_t size)
 {
@@ -112,23 +112,20 @@ static char* createOutFilename_orDie(const char* filename)
     return (char*)outSpace;
 }
 
-#define CLEVEL_DEFAULT 5
-int main(int argc, const char** argv)
-{
+int main(int argc, const char** argv) {
     const char* const exeName = argv[0];
-    if (argc<3 || argc>4) {
+    if (argc!=3) {
         printf("wrong arguments\n");
         printf("usage:\n");
-        printf("%s FILE FRAME_SIZE [LEVEL] \n", exeName);
+        printf("%s FILE FRAME_SIZE\n", exeName);
         return 1;
     }
 
     {   const char* const inFileName = argv[1];
         unsigned const frameSize = (unsigned)atoi(argv[2]);
-        int const cLevel = (argc==4) ? atoi(argv[3]) : CLEVEL_DEFAULT;
 
         char* const outFileName = createOutFilename_orDie(inFileName);
-        compressFile_orDie(inFileName, outFileName, cLevel, frameSize);
+        compressFile_orDie(inFileName, outFileName, 5, frameSize);
         free(outFileName);
     }
 

contrib/seekable_format/examples/seekable_decompression.c

@@ -16,7 +16,7 @@
 #include <zstd.h>      // presumes zstd library is installed
 #include <zstd_errors.h>
 
-#include "../zstd_seekable.h"
+#include "zstd_seekable.h"
 
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 

contrib/seekable_format/tests/.gitignore

@@ -1,2 +1 @@
 seekable_tests
-data.txt

contrib/seekable_format/tests/Makefile

@@ -24,7 +24,7 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
 .PHONY: default clean test
 default: test
 
-test: seekable_tests parallel_compression_test
+test: seekable_tests
 	./seekable_tests
 
 $(ZSTDLIB):
@@ -32,27 +32,7 @@ $(ZSTDLIB):
 
 seekable_tests : $(SEEKABLE_OBJS)
 
-EXAMPLES_PATH = ../examples
-PARALLEL_COMPRESSION = $(EXAMPLES_PATH)/parallel_compression
-
-DATAGEN_PATH = ../../../tests
-DATAGEN = $(DATAGEN_PATH)/datagen
-
-$(PARALLEL_COMPRESSION):
-	$(MAKE) -C $(EXAMPLES_PATH) parallel_compression
-
-$(DATAGEN):
-	$(MAKE) -C $(DATAGEN_PATH) datagen
-
-data.txt: $(DATAGEN)
-	$(DATAGEN) -g100M > $@
-
-parallel_compression_test: $(PARALLEL_COMPRESSION) data.txt
-	ulimit -Sv 102400; $(PARALLEL_COMPRESSION) data.txt 1048576 2
-
-.PHONY: parallel_compression_test parallel_comp
-
 clean:
 	@$(RM) core *.o tmp* result* *.zst \
-		seekable_tests data.txt
+		seekable_tests
 	@echo Cleaning completed

contrib/seekable_format/tests/seekable_tests.c

@@ -3,55 +3,8 @@
 #include <stdlib.h>  // malloc
 #include <stdio.h>
 #include <assert.h>
-#include <string.h>
 
-#include "../zstd_seekable.h"
-
-
-/* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */
-typedef struct {
-    const void *ptr;
-    size_t size;
-    size_t pos;
-    size_t totalRead;
-} buffWrapperWithTotal_t;
-
-static int readBuffWithTotal(void* opaque, void* buffer, size_t n)
-{
-    buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*)opaque;
-    assert(buff != NULL);
-    if (buff->pos + n > buff->size) return -1;
-    memcpy(buffer, (const char*)buff->ptr + buff->pos, n);
-    buff->pos += n;
-    buff->totalRead += n;
-    return 0;
-}
-
-static int seekBuffWithTotal(void* opaque, long long offset, int origin)
-{
-    buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*) opaque;
-    unsigned long long newOffset;
-    assert(buff != NULL);
-    switch (origin) {
-    case SEEK_SET:
-        assert(offset >= 0);
-        newOffset = (unsigned long long)offset;
-        break;
-    case SEEK_CUR:
-        newOffset = (unsigned long long)((long long)buff->pos + offset);
-        break;
-    case SEEK_END:
-        newOffset = (unsigned long long)((long long)buff->size + offset);
-        break;
-    default:
-        assert(0);  /* not possible */
-    }
-    if (newOffset > buff->size) {
-        return -1;
-    }
-    buff->pos = newOffset;
-    return 0;
-}
+#include "zstd_seekable.h"
 
 /* Basic unit tests for zstd seekable format */
 int main(int argc, const char** argv)
@@ -267,104 +220,6 @@ int main(int argc, const char** argv)
     }
     printf("Success!\n");
 
-
-    printf("Test %u - multiple decompress calls: ", testNb++);
-    {   char const inBuffer[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt";
-        size_t const inSize = sizeof(inBuffer);
-
-        size_t const seekCapacity = 5000;
-        void* const seekBuffer = malloc(seekCapacity);
-        assert(seekBuffer != NULL);
-        size_t seekSize;
-
-        size_t const outCapacity = inSize;
-        char* const outBuffer = malloc(outCapacity);
-        assert(outBuffer != NULL);
-
-        ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream();
-        assert(zscs != NULL);
-
-        /* compress test data with a small frame size to ensure multiple frames in the output */
-        unsigned const maxFrameSize = 40;
-        { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, maxFrameSize);
-          assert(!ZSTD_isError(initStatus));
-        }
-
-        {   ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity };
-            ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize };
-
-            while (inb.pos < inb.size) {
-                size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb);
-                assert(!ZSTD_isError(cStatus));
-            }
-
-            size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb);
-            assert(!ZSTD_isError(endStatus));
-            seekSize = outb.pos;
-        }
-
-        ZSTD_seekable* const stream = ZSTD_seekable_create();
-        assert(stream != NULL);
-        buffWrapperWithTotal_t buffWrapper = {seekBuffer, seekSize, 0, 0};
-        { ZSTD_seekable_customFile srcFile = {&buffWrapper, &readBuffWithTotal, &seekBuffWithTotal};
-          size_t const initStatus = ZSTD_seekable_initAdvanced(stream, srcFile);
-          assert(!ZSTD_isError(initStatus)); }
-
-        /* Perform a series of small reads and seeks (repeatedly read 1 byte and skip 1 byte)
-           and check that we didn't reread input data unnecessarily */
-        size_t pos;
-        for (pos = 0; pos < inSize; pos += 2) {
-            size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos);
-            if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) {
-                free(seekBuffer);
-                free(outBuffer);
-                ZSTD_seekable_freeCStream(zscs);
-                ZSTD_seekable_free(stream);
-                goto _test_error;
-            }
-        }
-        if (buffWrapper.totalRead > seekSize) {
-            /* We read more than the compressed size, meaning there were some rereads.
-               This is unneeded because we only seeked forward. */
-            printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize);
-            free(seekBuffer);
-            free(outBuffer);
-            ZSTD_seekable_freeCStream(zscs);
-            ZSTD_seekable_free(stream);
-            goto _test_error;
-        }
-
-        /* Perform some reads and seeks to ensure correctness */
-        struct {
-            size_t offset;
-            size_t size;
-        } const tests[] = {  /* Assume the frame size is 40 */
-            {20, 40}, /* read partial data from two frames */
-            {60, 10}, /* continue reading from the same offset */
-            {50, 20}, /* seek backward within the same frame */
-            {10, 10}, /* seek backward to a different frame */
-            {25, 10}, /* seek forward within the same frame */
-            {60, 10}, /* seek forward to a different frame */
-        };
-        size_t idx;
-        for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) {
-            size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset);
-            if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) {
-                free(seekBuffer);
-                free(outBuffer);
-                ZSTD_seekable_freeCStream(zscs);
-                ZSTD_seekable_free(stream);
-                goto _test_error;
-            }
-        }
-
-        free(seekBuffer);
-        free(outBuffer);
-        ZSTD_seekable_freeCStream(zscs);
-        ZSTD_seekable_free(stream);
-    }
-    printf("Success!\n");
-
     /* TODO: Add more tests */
     printf("Finished tests\n");
     return 0;

contrib/seekable_format/zstd_seekable.h

@@ -1,13 +1,13 @@
 #ifndef SEEKABLE_H
 #define SEEKABLE_H
 
-#include <stdio.h>
-#include "zstd.h"   /* ZSTDLIB_API */
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
+#include <stdio.h>
+#include "zstd.h"   /* ZSTDLIB_API */
+
 
 #define ZSTD_seekTableFooterSize 9
 
@@ -15,8 +15,8 @@ extern "C" {
 
 #define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U
 
-/* Limit maximum size to avoid potential issues storing the compressed size */
-#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x40000000U
+/* Limit the maximum size to avoid any potential issues storing the compressed size */
+#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x80000000U
 
 /*-****************************************************************************
 *  Seekable Format
@@ -48,18 +48,9 @@ typedef struct ZSTD_seekTable_s ZSTD_seekTable;
 *
 *  Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
 *  for a new compression operation.
-*  - `maxFrameSize` indicates the size at which to automatically start a new
-*            seekable frame.
-*            `maxFrameSize == 0` implies the default maximum size.
-*            Smaller frame sizes allow faster decompression of small segments,
-*            since retrieving a single byte requires decompression of
-*            the full frame where the byte belongs.
-*            In general, size the frames to roughly correspond to
-*            the access granularity (when it's known).
-*            But small sizes also reduce compression ratio.
-*            Avoid really tiny frame sizes (< 1 KB),
-*            that would hurt compression ratio considerably.
-*  - `checksumFlag` indicates whether or not the seek table should include frame
+*  `maxFrameSize` indicates the size at which to automatically start a new
+*  seekable frame.  `maxFrameSize == 0` implies the default maximum size.
+*  `checksumFlag` indicates whether or not the seek table should include frame
 *  checksums on the uncompressed data for verification.
 *  @return : a size hint for input to provide for compression, or an error code
 *            checkable with ZSTD_isError()

contrib/seekable_format/zstdseek_compress.c

@@ -230,8 +230,6 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
     const BYTE* const inBase = (const BYTE*) input->src + input->pos;
     size_t inLen = input->size - input->pos;
 
-    assert(zcs->maxFrameSize < INT_MAX);
-    ZSTD_CCtx_setParameter(zcs->cstream, ZSTD_c_srcSizeHint, (int)zcs->maxFrameSize);
     inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize));
 
     /* if we haven't finished flushing the last frame, don't start writing a new one */

contrib/seekable_format/zstdseek_decompress.c

@@ -77,10 +77,7 @@
 /* ************************************************************
 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
 ***************************************************************/
-#if defined(LIBC_NO_FSEEKO)
-/* Some older libc implementations don't include these functions (e.g. Bionic < 24) */
-#   define LONG_SEEK fseek
-#elif defined(_MSC_VER) && _MSC_VER >= 1400
+#if defined(_MSC_VER) && _MSC_VER >= 1400
 #   define LONG_SEEK _fseeki64
 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
 #   define LONG_SEEK fseeko
@@ -255,8 +252,6 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
 
 ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
 {
-    assert(zs != NULL);
-    if (zs->seekTable.entries == NULL) return NULL;
     ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable));
     if (st==NULL) return NULL;
 
@@ -498,7 +493,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
     size_t srcBytesRead = 0;
     do {
         /* check if we can continue from a previous decompress job */
-        if (targetFrame != zs->curFrame || offset < zs->decompressedOffset) {
+        if (targetFrame != zs->curFrame || offset != zs->decompressedOffset) {
             zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset;
             zs->curFrame = targetFrame;
 

doc/decompressor_errata.md

@@ -6,53 +6,12 @@ Each entry will contain:
 1. The last affected decompressor versions.
 2. The decompressor components affected.
 2. Whether the compressed frame could ever be produced by the reference compressor.
-3. An example frame (hexadecimal string when it can be short enough, link to golden file otherwise)
+3. An example frame.
 4. A description of the bug.
 
 The document is in reverse chronological order, with the bugs that affect the most recent zstd decompressor versions listed first.
 
 
-No sequence using the 2-bytes format
-------------------------------------------------
-
-**Last affected version**: v1.5.5
-
-**Affected decompressor component(s)**: Library & CLI
-
-**Produced by the reference compressor**: No
-
-**Example Frame**: see zstd/tests/golden-decompression/zeroSeq_2B.zst
-
-The zstd decoder incorrectly expects FSE tables when there are 0 sequences present in the block
-if the value 0 is encoded using the 2-bytes format.
-Instead, it should immediately end the sequence section, and move on to next block.
-
-This situation was never generated by the reference compressor,
-because representing 0 sequences with the 2-bytes format is inefficient
-(the 1-byte format is always used in this case).
-
-
-Compressed block with a size of exactly 128 KB
-------------------------------------------------
-
-**Last affected version**: v1.5.2
-
-**Affected decompressor component(s)**: Library & CLI
-
-**Produced by the reference compressor**: No
-
-**Example Frame**: see zstd/tests/golden-decompression/block-128k.zst
-
-The zstd decoder incorrectly rejected blocks of type `Compressed_Block` when their size was exactly 128 KB.
-Note that `128 KB - 1` was accepted, and `128 KB + 1` is forbidden by the spec.
-
-This type of block was never generated by the reference compressor.
-
-These blocks used to be disallowed by the spec up until spec version 0.3.2 when the restriction was lifted by [PR#1689](https://github.com/facebook/zstd/pull/1689).
-
-> A Compressed_Block has the extra restriction that Block_Size is always strictly less than the decompressed size. If this condition cannot be respected, the block must be sent uncompressed instead (Raw_Block).
-
-
 Compressed block with 0 literals and 0 sequences
 ------------------------------------------------
 
@@ -72,7 +31,6 @@ Additionally, these blocks were disallowed by the spec up until spec version 0.3
 
 > A Compressed_Block has the extra restriction that Block_Size is always strictly less than the decompressed size. If this condition cannot be respected, the block must be sent uncompressed instead (Raw_Block).
 
-
 First block is RLE block
 ------------------------
 
@@ -94,7 +52,6 @@ block.
 
 https://github.com/facebook/zstd/blob/8814aa5bfa74f05a86e55e9d508da177a893ceeb/lib/compress/zstd_compress.c#L3527-L3535
 
-
 Tiny FSE Table & Block
 ----------------------
 
@@ -125,24 +82,3 @@ The total `Block_Content` is `5` bytes, and `Last_Table_Offset` is `2`.
 See the compressor workaround code:
 
 https://github.com/facebook/zstd/blob/8814aa5bfa74f05a86e55e9d508da177a893ceeb/lib/compress/zstd_compress.c#L2667-L2682
-
-Magicless format
-----------------------
-
-**Last affected version**: v1.5.5
-
-**Affected decompressor component(s)**: Library
-
-**Produced by the reference compressor**: Yes (example: https://gist.github.com/embg/9940726094f4cf2cef162cffe9319232)
-
-**Example Frame**: `27 b5 2f fd 00 03 19 00 00 66 6f 6f 3f ba c4 59`
-
-v1.5.6 fixes several bugs in which the magicless-format decoder rejects valid frames.
-These include but are not limited to:
-* Valid frames that happen to begin with a legacy magic number (little-endian)
-* Valid frames that happen to begin with a skippable magic number (little-endian)
-
-If you are affected by this issue and cannot update to v1.5.6 or later, there is a
-workaround to recover affected data. Simply prepend the ZSTD magic number
-`0xFD2FB528` (little-endian) to your data and decompress using the standard-format
-decoder.

doc/decompressor_permissive.md

@@ -1,80 +0,0 @@
-Decompressor Permissiveness to Invalid Data
-===========================================
-
-This document describes the behavior of the reference decompressor in cases
-where it accepts formally invalid data instead of reporting an error.
-
-While the reference decompressor *must* decode any compliant frame following
-the specification, its ability to detect erroneous data is on a best effort
-basis: the decoder may accept input data that would be formally invalid,
-when it causes no risk to the decoder, and which detection would cost too much
-complexity or speed regression.
-
-In practice, the vast majority of invalid data are detected, if only because
-many corruption events are dangerous for the decoder process (such as
-requesting an out-of-bound memory access) and many more are easy to check.
-
-This document lists a few known cases where invalid data was formerly accepted
-by the decoder, and what has changed since.
-
-
-Truncated Huffman states
-------------------------
-
-**Last affected version**: v1.5.6
-
-**Produced by the reference compressor**: No
-
-**Example Frame**: `28b5 2ffd 0000 5500 0072 8001 0420 7e1f 02aa 00`
-
-When using FSE-compressed Huffman weights, the compressed weight bitstream
-could contain fewer bits than necessary to decode the initial states.
-
-The reference decompressor up to v1.5.6 will decode truncated or missing
-initial states as zero, which can result in a valid Huffman tree if only
-the second state is truncated.
-
-In newer versions, truncated initial states are reported as a corruption
-error by the decoder.
-
-
-Offset == 0
------------
-
-**Last affected version**: v1.5.5
-
-**Produced by the reference compressor**: No
-
-**Example Frame**: `28b5 2ffd 0000 4500 0008 0002 002f 430b ae`
-
-If a sequence is decoded with `literals_length = 0` and `offset_value = 3`
-while `Repeated_Offset_1 = 1`, the computed offset will be `0`, which is
-invalid.
-
-The reference decompressor up to v1.5.5 processes this case as if the computed
-offset was `1`, including inserting `1` into the repeated offset list.
-This prevents the output buffer from remaining uninitialized, thus denying a
-potential attack vector from an untrusted source.
-However, in the rare case where this scenario would be the outcome of a
-transmission or storage error, the decoder relies on the checksum to detect
-the error.
-
-In newer versions, this case is always detected and reported as a corruption error.
-
-
-Non-zeroes reserved bits
-------------------------
-
-**Last affected version**: v1.5.5
-
-**Produced by the reference compressor**: No
-
-The Sequences section of each block has a header, and one of its elements is a
-byte, which describes the compression mode of each symbol.
-This byte contains 2 reserved bits which must be set to zero.
-
-The reference decompressor up to v1.5.5 just ignores these 2 bits.
-This behavior has no consequence for the rest of the frame decoding process.
-
-In newer versions, the 2 reserved bits are actively checked for value zero,
-and the decoder reports a corruption error if they are not.

doc/educational_decoder/Makefile

@@ -10,7 +10,7 @@
 
 ZSTD ?= zstd   # note: requires zstd installation on local system
 
-UNAME?= $(shell sh -c 'MSYSTEM="MSYS" uname') 
+UNAME?= $(shell uname)
 ifeq ($(UNAME), SunOS)
 DIFF ?= gdiff
 else

doc/educational_decoder/zstd_decompress.c

@@ -997,8 +997,7 @@ static void decompress_sequences(frame_context_t *const ctx,
                                  const size_t num_sequences);
 static sequence_command_t decode_sequence(sequence_states_t *const state,
                                           const u8 *const src,
-                                          i64 *const offset,
-                                          int lastSequence);
+                                          i64 *const offset);
 static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
                                const seq_part_t type, const seq_mode_t mode);
 
@@ -1018,7 +1017,12 @@ static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
     // This is a variable size field using between 1 and 3 bytes. Let's call its
     // first byte byte0."
     u8 header = IO_read_bits(in, 8);
-    if (header < 128) {
+    if (header == 0) {
+        // "There are no sequences. The sequence section stops there.
+        // Regenerated content is defined entirely by literals section."
+        *sequences = NULL;
+        return 0;
+    } else if (header < 128) {
         // "Number_of_Sequences = byte0 . Uses 1 byte."
         num_sequences = header;
     } else if (header < 255) {
@@ -1029,12 +1033,6 @@ static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
         num_sequences = IO_read_bits(in, 16) + 0x7F00;
     }
 
-    if (num_sequences == 0) {
-        // "There are no sequences. The sequence section stops there."
-        *sequences = NULL;
-        return 0;
-    }
-
     *sequences = malloc(num_sequences * sizeof(sequence_command_t));
     if (!*sequences) {
         BAD_ALLOC();
@@ -1116,7 +1114,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
 
     for (size_t i = 0; i < num_sequences; i++) {
         // Decode sequences one by one
-        sequences[i] = decode_sequence(&states, src, &bit_offset, i==num_sequences-1);
+        sequences[i] = decode_sequence(&states, src, &bit_offset);
     }
 
     if (bit_offset != 0) {
@@ -1127,8 +1125,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
 // Decode a single sequence and update the state
 static sequence_command_t decode_sequence(sequence_states_t *const states,
                                           const u8 *const src,
-                                          i64 *const offset,
-                                          int lastSequence) {
+                                          i64 *const offset) {
     // "Each symbol is a code in its own context, which specifies Baseline and
     // Number_of_Bits to add. Codes are FSE compressed, and interleaved with raw
     // additional bits in the same bitstream."
@@ -1163,7 +1160,7 @@ static sequence_command_t decode_sequence(sequence_states_t *const states,
     // Literals_Length_State is updated, followed by Match_Length_State, and
     // then Offset_State."
     // If the stream is complete don't read bits to update state
-    if (!lastSequence) {
+    if (*offset != 0) {
         FSE_update_state(&states->ll_table, &states->ll_state, src, offset);
         FSE_update_state(&states->ml_table, &states->ml_state, src, offset);
         FSE_update_state(&states->of_table, &states->of_state, src, offset);
@@ -1213,7 +1210,7 @@ static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
         break;
     }
     case seq_repeat:
-        // "Repeat_Mode : reuse distribution table from previous compressed
+        // "Repeat_Mode : re-use distribution table from previous compressed
         // block."
         // Nothing to do here, table will be unchanged
         if (!table->symbols) {
@@ -1402,7 +1399,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
 /******* END OUTPUT SIZE COUNTING *********************************************/
 
 /******* DICTIONARY PARSING ***************************************************/
-dictionary_t* create_dictionary(void) {
+dictionary_t* create_dictionary() {
     dictionary_t* const dict = calloc(1, sizeof(dictionary_t));
     if (!dict) {
         BAD_ALLOC();

doc/zstd_compression_format.md

@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.4.4 (2025-03-22)
+0.3.7 (2020-12-09)
 
 
 Introduction
@@ -390,7 +390,7 @@ __`Block_Content`__ and __`Block_Maximum_Size`__
 The size of `Block_Content` is limited by `Block_Maximum_Size`,
 which is the smallest of:
 -  `Window_Size`
--  128 KiB (131.072 bytes)
+-  128 KB
 
 `Block_Maximum_Size` is constant for a given frame.
 This maximum is applicable to both the decompressed size
@@ -470,7 +470,6 @@ This field uses 2 lowest bits of first byte, describing 4 different block types
         repeated `Regenerated_Size` times.
 - `Compressed_Literals_Block` - This is a standard Huffman-compressed block,
         starting with a Huffman tree description.
-        In this mode, there are at least 2 different literals represented in the Huffman tree description.
         See details below.
 - `Treeless_Literals_Block` - This is a Huffman-compressed block,
         using Huffman tree _from previous Huffman-compressed literals block_.
@@ -534,20 +533,15 @@ __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__
 Both `Compressed_Size` and `Regenerated_Size` fields follow __little-endian__ convention.
 Note: `Compressed_Size` __includes__ the size of the Huffman Tree description
 _when_ it is present.
-Note 2: `Compressed_Size` can never be `==0`.
-Even in single-stream scenario, assuming an empty content, it must be `>=1`,
-since it contains at least the final end bit flag.
-In 4-streams scenario, a valid `Compressed_Size` is necessarily `>= 10`
-(6 bytes for the jump table, + 4x1 bytes for the 4 streams).
 
-4 streams is faster than 1 stream in decompression speed,
+4 streams is superior to 1 stream in decompression speed,
 by exploiting instruction level parallelism.
 But it's also more expensive,
 costing on average ~7.3 bytes more than the 1 stream mode, mostly from the jump table.
 
 In general, use the 4 streams mode when there are more literals to decode,
 to favor higher decompression speeds.
-Note that beyond >1KB of literals, the 4 streams mode is compulsory.
+Beyond 1KB, the 4 streams mode is compulsory anyway.
 
 Note that a minimum of 6 bytes is required for the 4 streams mode.
 That's a technical minimum, but it's not recommended to employ the 4 streams mode
@@ -572,7 +566,6 @@ or from a dictionary.
 
 ### `Huffman_Tree_Description`
 This section is only present when `Literals_Block_Type` type is `Compressed_Literals_Block` (`2`).
-The tree describes the weights of all literals symbols that can be present in the literals block, at least 2 and up to 256.
 The format of the Huffman tree description can be found at [Huffman Tree description](#huffman-tree-description).
 The size of `Huffman_Tree_Description` is determined during decoding process,
 it must be used to determine where streams begin.
@@ -582,10 +575,10 @@ it must be used to determine where streams begin.
 ### Jump Table
 The Jump Table is only present when there are 4 Huffman-coded streams.
 
-Reminder : Huffman compressed data consists of either 1 or 4 streams.
+Reminder : Huffman compressed data consists of either 1 or 4 Huffman-coded streams.
 
 If only one stream is present, it is a single bitstream occupying the entire
-remaining portion of the literals block, encoded as described in
+remaining portion of the literals block, encoded as described within
 [Huffman-Coded Streams](#huffman-coded-streams).
 
 If there are four streams, `Literals_Section_Header` only provided
@@ -596,18 +589,17 @@ except for the last stream which may be up to 3 bytes smaller,
 to reach a total decompressed size as specified in `Regenerated_Size`.
 
 The compressed size of each stream is provided explicitly in the Jump Table.
-Jump Table is 6 bytes long, and consists of three 2-byte __little-endian__ fields,
+Jump Table is 6 bytes long, and consist of three 2-byte __little-endian__ fields,
 describing the compressed sizes of the first three streams.
-`Stream4_Size` is computed from `Total_Streams_Size` minus sizes of other streams:
+`Stream4_Size` is computed from total `Total_Streams_Size` minus sizes of other streams.
 
 `Stream4_Size = Total_Streams_Size - 6 - Stream1_Size - Stream2_Size - Stream3_Size`.
 
-`Stream4_Size` is necessarily `>= 1`. Therefore,
-if `Total_Streams_Size < Stream1_Size + Stream2_Size + Stream3_Size + 6 + 1`,
+Note: if `Stream1_Size + Stream2_Size + Stream3_Size > Total_Streams_Size`,
 data is considered corrupted.
 
 Each of these 4 bitstreams is then decoded independently as a Huffman-Coded stream,
-as described in [Huffman-Coded Streams](#huffman-coded-streams)
+as described at [Huffman-Coded Streams](#huffman-coded-streams)
 
 
 Sequences Section
@@ -650,16 +642,14 @@ __`Number_of_Sequences`__
 
 This is a variable size field using between 1 and 3 bytes.
 Let's call its first byte `byte0`.
+- `if (byte0 == 0)` : there are no sequences.
+            The sequence section stops there.
+            Decompressed content is defined entirely as Literals Section content.
+            The FSE tables used in `Repeat_Mode` aren't updated.
 - `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte.
-- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0 - 0x80) << 8) + byte1`. Uses 2 bytes.
-            Note that the 2 bytes format fully overlaps the 1 byte format.
+- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes.
 - `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes.
 
-`if (Number_of_Sequences == 0)` : there are no sequences.
-            The sequence section stops immediately,
-            FSE tables used in `Repeat_Mode` aren't updated.
-            Block's decompressed content is defined solely by the Literals Section content.
-
 __Symbol compression modes__
 
 This is a single byte, defining the compression mode of each symbol type.
@@ -929,10 +919,7 @@ There is an exception though, when current sequence's `literals_length = 0`.
 In this case, repeated offsets are shifted by one,
 so an `offset_value` of 1 means `Repeated_Offset2`,
 an `offset_value` of 2 means `Repeated_Offset3`,
-and an `offset_value` of 3 means `Repeated_Offset1 - 1`.
-
-In the final case, if `Repeated_Offset1 - 1` evaluates to 0, then the
-data is considered corrupted.
+and an `offset_value` of 3 means `Repeated_Offset1 - 1_byte`.
 
 For the first block, the starting offset history is populated with following values :
 `Repeated_Offset1`=1, `Repeated_Offset2`=4, `Repeated_Offset3`=8,
@@ -1038,57 +1025,55 @@ and to compress Huffman headers.
 FSE
 ---
 FSE, short for Finite State Entropy, is an entropy codec based on [ANS].
-FSE encoding/decoding involves a state that is carried over between symbols.
-Decoding must be done in the opposite direction as encoding.
+FSE encoding/decoding involves a state that is carried over between symbols,
+so decoding must be done in the opposite direction as encoding.
 Therefore, all FSE bitstreams are read from end to beginning.
 Note that the order of the bits in the stream is not reversed,
-we just read each multi-bits element in the reverse order they are encoded.
+we just read the elements in the reverse order they are written.
 
 For additional details on FSE, see [Finite State Entropy].
 
 [Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/
 
-FSE decoding is directed by a decoding table with a power of 2 size, each row containing three elements:
+FSE decoding involves a decoding table which has a power of 2 size, and contain three elements:
 `Symbol`, `Num_Bits`, and `Baseline`.
 The `log2` of the table size is its `Accuracy_Log`.
 An FSE state value represents an index in this table.
 
 To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a __little-endian__ value.
-The first symbol in the stream is the `Symbol` indicated in the table for that state.
+The next symbol in the stream is the `Symbol` indicated in the table for that state.
 To obtain the next state value,
 the decoder should consume `Num_Bits` bits from the stream as a __little-endian__ value and add it to `Baseline`.
 
 [ANS]: https://en.wikipedia.org/wiki/Asymmetric_Numeral_Systems
 
 ### FSE Table Description
-To decode an FSE bitstream, it is necessary to build its FSE decoding table.
-The decoding table is derived from a distribution of Probabilities.
-The Zstandard format encodes distributions of Probabilities as follows:
+To decode FSE streams, it is necessary to construct the decoding table.
+The Zstandard format encodes FSE table descriptions as follows:
 
-The distribution of probabilities is described in a bitstream which is read forward,
-in __little-endian__ fashion.
-The amount of bytes consumed from the bitstream to describe the distribution
-is discovered at the end of the decoding process.
+An FSE distribution table describes the probabilities of all symbols
+from `0` to the last present one (included)
+on a normalized scale of `1 << Accuracy_Log` .
+Note that there must be two or more symbols with nonzero probability.
 
-The bitstream starts by reporting on which scale the distribution operates.
+It's a bitstream which is read forward, in __little-endian__ fashion.
+It's not necessary to know bitstream exact size,
+it will be discovered and reported by the decoding process.
+
+The bitstream starts by reporting on which scale it operates.
 Let's `low4Bits` designate the lowest 4 bits of the first byte :
 `Accuracy_Log = low4bits + 5`.
 
-An FSE distribution table describes the probabilities of all symbols
-from `0` to the last present one (included) in natural order.
-The sum of probabilities is normalized to reach a power of 2 total of `1 << Accuracy_Log` .
-There must be two or more symbols with non-zero probabilities.
-
-The number of bits used to decode each probability is variable.
+Then follows each symbol value, from `0` to last present one.
+The number of bits used by each field is variable.
 It depends on :
 
 - Remaining probabilities + 1 :
   __example__ :
   Presuming an `Accuracy_Log` of 8,
-  and presuming 100 probability points have already been distributed,
+  and presuming 100 probabilities points have already been distributed,
   the decoder may read any value from `0` to `256 - 100 + 1 == 157` (inclusive).
-  Therefore, it may read up to `log2sup(157) == 8` bits, where `log2sup(N)`
-  is the smallest integer `T` that satisfies `(1 << T) > N`.
+  Therefore, it must read `log2sup(157) == 8` bits.
 
 - Value decoded : small values use 1 less bit :
   __example__ :
@@ -1099,133 +1084,111 @@ It depends on :
   values from 98 to 157 use 8 bits.
   This is achieved through this scheme :
 
-  | 8-bit field read | Value decoded | Nb of bits consumed |
-  | ---------------- | ------------- | ------------------- |
+  | Value read | Value decoded | Number of bits used |
+  | ---------- | ------------- | ------------------- |
   |   0 -  97  |   0 -  97     |  7                  |
   |  98 - 127  |  98 - 127     |  8                  |
   | 128 - 225  |   0 -  97     |  7                  |
   | 226 - 255  | 128 - 157     |  8                  |
 
-Probability is derived from Value decoded using the following formula:
-`Probality = Value - 1`
+Symbols probabilities are read one by one, in order.
 
-Consequently, a Probability of `0` is described by a Value `1`.
+Probability is obtained from Value decoded by following formula :
+`Proba = value - 1`
 
-A Value `0` is used to signal a special case, named "Probability `-1`".
-It describes a probability which should have been "less than 1".
-Its effect on the decoding table building process is described in the [next section].
-For the purpose of counting total allocated probability points, it counts as one.
+It means value `0` becomes negative probability `-1`.
+`-1` is a special probability, which means "less than 1".
+Its effect on distribution table is described in the [next section].
+For the purpose of calculating total allocated probability points, it counts as one.
 
 [next section]:#from-normalized-distribution-to-decoding-tables
 
-Symbols probabilities are read one by one, in order.
-After each probability is decoded, the total nb of probability points is updated.
-This is used to determine how many bits must be read to decode the probability of next symbol.
-
-When a symbol has a __probability__ of `zero` (decoded from reading a Value `1`),
+When a symbol has a __probability__ of `zero`,
 it is followed by a 2-bits repeat flag.
 This repeat flag tells how many probabilities of zeroes follow the current one.
 It provides a number ranging from 0 to 3.
 If it is a 3, another 2-bits repeat flag follows, and so on.
 
-When the Probability for a symbol makes cumulated total reach `1 << Accuracy_Log`,
-then it's the last symbol, and decoding is complete.
+When last symbol reaches cumulated total of `1 << Accuracy_Log`,
+decoding is complete.
+If the last symbol makes cumulated total go above `1 << Accuracy_Log`,
+distribution is considered corrupted.
 
 Then the decoder can tell how many bytes were used in this process,
 and how many symbols are present.
 The bitstream consumes a round number of bytes.
 Any remaining bit within the last byte is just unused.
 
-If this process results in a non-zero probability for a symbol outside of the
-valid range of symbols that the FSE table is defined for, even if that symbol is
-not used, then the data is considered corrupted.
-For the specific case of offset codes,
-a decoder implementation may reject a frame containing a non-zero probability
-for an offset code larger than the largest offset code supported by the decoder
-implementation.
-
 #### From normalized distribution to decoding tables
 
-The normalized distribution of probabilities is enough
+The distribution of normalized probabilities is enough
 to create a unique decoding table.
-It is generated using the following build rule :
+
+It follows the following build rule :
 
 The table has a size of `Table_Size = 1 << Accuracy_Log`.
-Each row specifies the decoded symbol,
-and instructions to reach the next state (`Number_of_Bits` and `Baseline`).
+Each cell describes the symbol decoded,
+and instructions to get the next state (`Number_of_Bits` and `Baseline`).
 
-Symbols are first scanned in their natural order for "less than 1" probabilities
-(previously decoded from a Value of `0`).
-Symbols with this special probability are being attributed a single row,
+Symbols are scanned in their natural order for "less than 1" probabilities.
+Symbols with this probability are being attributed a single cell,
 starting from the end of the table and retreating.
 These symbols define a full state reset, reading `Accuracy_Log` bits.
 
-Then, all remaining symbols, sorted in natural order, are allocated rows.
-Starting from smallest present symbol, and table position `0`,
-each symbol gets allocated as many rows as its probability.
+Then, all remaining symbols, sorted in natural order, are allocated cells.
+Starting from symbol `0` (if it exists), and table position `0`,
+each symbol gets allocated as many cells as its probability.
+Cell allocation is spread, not linear :
+each successor position follows this rule :
 
-Row allocation is not linear, it follows this order, in modular arithmetic:
 ```
 position += (tableSize>>1) + (tableSize>>3) + 3;
 position &= tableSize-1;
 ```
 
-Using above ordering rule, each symbol gets allocated as many rows as its probability.
-If a position is already occupied by a "less than 1" probability symbol,
-it is simply skipped, and the next position is allocated instead.
-Once enough rows have been allocated for the current symbol,
-the allocation process continues, using the next symbol, in natural order.
-This process guarantees that the table is entirely and exactly filled.
+A position is skipped if already occupied by a "less than 1" probability symbol.
+`position` does not reset between symbols, it simply iterates through
+each position in the table, switching to the next symbol when enough
+states have been allocated to the current one.
 
-Each row specifies a decoded symbol, and is accessed by current state value.
-It also specifies `Number_of_Bits` and `Baseline`, which are required to determine next state value.
+The process guarantees that the table is entirely filled.
+Each cell corresponds to a state value, which contains the symbol being decoded.
 
-To correctly set these fields, it's necessary to sort all occurrences of each symbol in state value order,
-and then attribute N+1 bits to lower rows, and N bits to higher rows,
-following the process described below (using an example):
+To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
+it's first necessary to sort all occurrences of each symbol in state order.
+Lower states will need 1 more bit than higher ones.
+The process is repeated for each symbol.
 
 __Example__ :
-Presuming an `Accuracy_Log` of 7,
-let's imagine a symbol with a Probability of 5:
-it receives 5 rows, corresponding to 5 state values between `0` and `127`.
+Presuming a symbol has a probability of 5,
+it receives 5 cells, corresponding to 5 state values.
+These state values are then sorted in natural order.
 
-In this example, the first state value happens to be `1` (after unspecified previous symbols).
-The next 4 states are then determined using above modular arithmetic rule,
-which specifies to add `64+16+3 = 83` modulo `128` to jump to next position,
-producing the following series: `1`, `84`, `39`, `122`, `77` (modular arithmetic).
-(note: the next symbol will then start at `32`).
+Next power of 2 after 5 is 8.
+Space of probabilities must be divided into 8 equal parts.
+Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
+Divided by 8, each share is 16 large.
 
-These state values are then sorted in natural order,
-resulting in the following series: `1`, `39`, `77`, `84`, `122`.
-
-The next power of 2 after 5 is 8.
-Therefore, the probability space will be divided into 8 equal parts.
-Since the probability space is `1<<7 = 128` large, each share is `128/8 = 16` large.
-
-In order to reach 8 shares, the `8-5 = 3` lowest states will count "double",
+In order to reach 8 shares, 8-5=3 lowest states will count "double",
 doubling their shares (32 in width), hence requiring one more bit.
 
-Baseline is assigned starting from the lowest state using fewer bits,
-continuing in natural state order, looping back at the beginning.
-Each state takes its allocated range from Baseline, sized by its `Number_of_Bits`.
+Baseline is assigned starting from the higher states using fewer bits,
+increasing at each state, then resuming at the first state,
+each state takes its allocated width from Baseline.
 
+| state value      |   1   |  39   |   77   |  84  |  122   |
 | state order      |   0   |   1   |    2   |   3  |    4   |
 | ---------------- | ----- | ----- | ------ | ---- | ------ |
-| state value      |   1   |  39   |   77   |  84  |  122   |
 | width            |  32   |  32   |   32   |  16  |   16   |
 | `Number_of_Bits` |   5   |   5   |    5   |   4  |    4   |
-| allocation order |   3   |   4   |    5   |   1  |    2   |
+| range number     |   2   |   4   |    6   |   0  |    1   |
 | `Baseline`       |  32   |  64   |   96   |   0  |   16   |
 | range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31  |
 
-During decoding, the next state value is determined by using current state value as row number,
-then reading the required `Number_of_Bits` from the bitstream, and adding the specified `Baseline`.
+During decoding, the next state value is determined from current state value,
+by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
 
-Note:
-as a trivial example, it follows that, for a symbol with a Probability of `1`,
-`Baseline` is necessarily `0`, and `Number_of_Bits` is necessarily `Accuracy_Log`.
-
-See [Appendix A] to see the outcome of this process applied to the default distributions.
+See [Appendix A] for the results of this process applied to the default distributions.
 
 [Appendix A]: #appendix-a---decoding-tables-for-predefined-codes
 
@@ -1234,7 +1197,7 @@ Huffman Coding
 --------------
 Zstandard Huffman-coded streams are read backwards,
 similar to the FSE bitstreams.
-Therefore, to find the start of the bitstream, it is required to
+Therefore, to find the start of the bitstream, it is therefore to
 know the offset of the last byte of the Huffman-coded stream.
 
 After writing the last bit containing information, the compressor
@@ -1270,61 +1233,48 @@ This specification limits maximum code length to 11 bits.
 
 #### Representation
 
-All literal symbols from zero (included) to last present one (excluded)
+All literal values from zero (included) to last present one (excluded)
 are represented by `Weight` with values from `0` to `Max_Number_of_Bits`.
 Transformation from `Weight` to `Number_of_Bits` follows this formula :
 ```
 Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
 ```
-When a literal symbol is not present, it receives a `Weight` of 0.
-The least frequent symbol receives a `Weight` of 1.
-If no literal has a `Weight` of 1, then the data is considered corrupted.
-If there are not at least two literals with non-zero `Weight`, then the data
-is considered corrupted.
-The most frequent symbol receives a `Weight` anywhere between 1 and 11 (max).
-The last symbol's `Weight` is deduced from previously retrieved Weights,
-by completing to the nearest power of 2. It's necessarily non 0.
-If it's not possible to reach a clean power of 2 with a single `Weight` value,
-the Huffman Tree Description is considered invalid.
-This final power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
+The last symbol's `Weight` is deduced from previously decoded ones,
+by completing to the nearest power of 2.
+This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
 `Max_Number_of_Bits` must be <= 11,
 otherwise the representation is considered corrupted.
 
 __Example__ :
 Let's presume the following Huffman tree must be described :
 
-|  literal symbol  |  A  |  B  |  C  |  D  |  E  |  F  |
+|  literal value   |  0  |  1  |  2  |  3  |  4  |  5  |
 | ---------------- | --- | --- | --- | --- | --- | --- |
 | `Number_of_Bits` |  1  |  2  |  3  |  0  |  4  |  4  |
 
 The tree depth is 4, since its longest elements uses 4 bits
-(longest elements are the ones with smallest frequency).
-
-All symbols will now receive a `Weight` instead of `Number_of_Bits`.
+(longest elements are the one with smallest frequency).
+Value `5` will not be listed, as it can be determined from values for 0-4,
+nor will values above `5` as they are all 0.
+Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
 Weight formula is :
 ```
 Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0
 ```
-It gives the following series of Weights :
+It gives the following series of weights :
 
-| literal symbol |  A  |  B  |  C  |  D  |  E  |  F  |
-| -------------- | --- | --- | --- | --- | --- | --- |
-|   `Weight`     |  4  |  3  |  2  |  0  |  1  |  1  |
-
-This list will be sent to the decoder, with the following modifications:
-
-- `F` will not be listed, because it can be determined from previous symbols
-- nor will symbols above `F` as they are all 0
-- on the other hand, all symbols before `A`, starting with `\0`, will be listed, with a Weight of 0.
+| literal value |  0  |  1  |  2  |  3  |  4  |
+| ------------- | --- | --- | --- | --- | --- |
+|   `Weight`    |  4  |  3  |  2  |  0  |  1  |
 
 The decoder will do the inverse operation :
-having collected weights of literal symbols from `A` to `E`,
-it knows the last literal, `F`, is present with a non-zero `Weight`.
-The `Weight` of `F` can be determined by advancing to the next power of 2.
+having collected weights of literal symbols from `0` to `4`,
+it knows the last literal, `5`, is present with a non-zero `Weight`.
+The `Weight` of `5` can be determined by advancing to the next power of 2.
 The sum of `2^(Weight-1)` (excluding 0's) is :
 `8 + 4 + 2 + 0 + 1 = 15`.
 Nearest larger power of 2 value is 16.
-Therefore, `Max_Number_of_Bits = log2(16) = 4` and `Weight[F] = log_2(16 - 15) + 1 = 1`.
+Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = 16-15 = 1`.
 
 #### Huffman Tree header
 
@@ -1364,7 +1314,7 @@ sharing a single distribution table.
 To decode an FSE bitstream, it is necessary to know its compressed size.
 Compressed size is provided by `headerByte`.
 It's also necessary to know its _maximum possible_ decompressed size,
-which is `255`, since literal symbols span from `0` to `255`,
+which is `255`, since literal values span from `0` to `255`,
 and last symbol's `Weight` is not represented.
 
 An FSE bitstream starts by a header, describing probabilities distribution.
@@ -1386,13 +1336,6 @@ If updating state after decoding a symbol would require more bits than
 remain in the stream, it is assumed that extra bits are 0.  Then,
 symbols for each of the final states are decoded and the process is complete.
 
-If this process would produce more weights than the maximum number of decoded
-weights (255), then the data is considered corrupted.
-
-If either of the 2 initial states are absent or truncated, then the data is
-considered corrupted.  Consequently, it is not possible to encode fewer than
-2 weights using this mode.
-
 #### Conversion from weights to Huffman prefix codes
 
 All present symbols shall now have a `Weight` value.
@@ -1400,28 +1343,26 @@ It is possible to transform weights into `Number_of_Bits`, using this formula:
 ```
 Number_of_Bits = (Weight>0) ? Max_Number_of_Bits + 1 - Weight : 0
 ```
-In order to determine which prefix code is assigned to each Symbol,
-Symbols are first sorted by `Weight`, then by natural sequential order.
+Symbols are sorted by `Weight`.
+Within same `Weight`, symbols keep natural sequential order.
 Symbols with a `Weight` of zero are removed.
-Then, starting from lowest `Weight` (hence highest `Number_of_Bits`),
-prefix codes are assigned in ascending order.
+Then, starting from lowest `Weight`, prefix codes are distributed in sequential order.
 
 __Example__ :
-Let's assume the following list of weights has been decoded:
+Let's presume the following list of weights has been decoded :
 
-| Literal  |  A  |  B  |  C  |  D  |  E  |  F  |
+| Literal  |  0  |  1  |  2  |  3  |  4  |  5  |
 | -------- | --- | --- | --- | --- | --- | --- |
 | `Weight` |  4  |  3  |  2  |  0  |  1  |  1  |
 
 Sorted by weight and then natural sequential order,
-it gives the following prefix codes distribution:
+it gives the following distribution :
 
-| Literal          |  D  |   E  |   F  |   C  |   B  |   A  |
-| ---------------- | --- | ---- | ---- | ---- | ---- | ---- |
+| Literal          |  3  |  4  |  5  |  2  |  1  |   0  |
+| ---------------- | --- | --- | --- | --- | --- | ---- |
 | `Weight`         |  0  |  1  |  1  |  2  |  3  |   4  |
 | `Number_of_Bits` |  0  |  4  |  4  |  3  |  2  |   1  |
-| prefix code      | N/A | 0000 | 0001 | 001  | 01   | 1    |
-| ascending order  | N/A | 0000 | 0001 | 001x | 01xx | 1xxx |
+| prefix codes     | N/A | 0000| 0001| 001 | 01  |   1  |
 
 ### Huffman-coded Streams
 
@@ -1444,10 +1385,10 @@ it's possible to read the bitstream in a __little-endian__ fashion,
 keeping track of already used bits. Since the bitstream is encoded in reverse
 order, starting from the end read symbols in forward order.
 
-For example, if the literal sequence `ABEF` was encoded using above prefix code,
+For example, if the literal sequence "0145" was encoded using above prefix code,
 it would be encoded (in reverse order) as:
 
-|Symbol  |   F  |   E  |  B | A | Padding |
+|Symbol  |   5  |   4  |  1 | 0 | Padding |
 |--------|------|------|----|---|---------|
 |Encoding|`0000`|`0001`|`01`|`1`| `00001` |
 
@@ -1742,13 +1683,6 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
-- 0.4.4 : minor clarification for block size
-- 0.4.3 : clarifications for Huffman prefix code assignment example
-- 0.4.2 : refactor FSE table construction process, inspired by Donald Pian
-- 0.4.1 : clarifications on a few error scenarios, by Eric Lasota
-- 0.4.0 : fixed imprecise behavior for nbSeq==0, detected by Igor Pavlov
-- 0.3.9 : clarifications for Huffman-compressed literal sizes.
-- 0.3.8 : clarifications for Huffman Blocks and Huffman Tree descriptions.
 - 0.3.7 : clarifications for Repeat_Offsets, matching RFC8878
 - 0.3.6 : clarifications for Dictionary_ID
 - 0.3.5 : clarifications for Block_Maximum_Size

doc/zstd_manual.html

@@ -1,17 +1,16 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.5.7 Manual</title>
+<title>zstd 1.5.4 Manual</title>
 </head>
 <body>
-<h1>zstd 1.5.7 Manual</h1>
-Note: the content of this file has been automatically generated by parsing "zstd.h" 
+<h1>zstd 1.5.4 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
 <li><a href="#Chapter1">Introduction</a></li>
 <li><a href="#Chapter2">Version</a></li>
-<li><a href="#Chapter3">Simple Core API</a></li>
+<li><a href="#Chapter3">Simple API</a></li>
 <li><a href="#Chapter4">Explicit context</a></li>
 <li><a href="#Chapter5">Advanced compression API (Requires v1.4.0+)</a></li>
 <li><a href="#Chapter6">Advanced decompression API (Requires v1.4.0+)</a></li>
@@ -23,15 +22,15 @@ Note: the content of this file has been automatically generated by parsing "zstd
 <li><a href="#Chapter12">Dictionary helper functions</a></li>
 <li><a href="#Chapter13">Advanced dictionary and prefix API (Requires v1.4.0+)</a></li>
 <li><a href="#Chapter14">experimental API (static linking only)</a></li>
-<li><a href="#Chapter15">Frame header and size functions</a></li>
+<li><a href="#Chapter15">Frame size functions</a></li>
 <li><a href="#Chapter16">Memory management</a></li>
 <li><a href="#Chapter17">Advanced compression functions</a></li>
 <li><a href="#Chapter18">Advanced decompression functions</a></li>
 <li><a href="#Chapter19">Advanced streaming functions</a></li>
-<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions (DEPRECATED)</a></li>
+<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li>
 <li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li>
 <li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li>
-<li><a href="#Chapter23">Block level API (DEPRECATED)</a></li>
+<li><a href="#Chapter23">Block level API</a></li>
 </ol>
 <hr>
 <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -75,14 +74,13 @@ Note: the content of this file has been automatically generated by parsing "zstd
 </b><p>  Return runtime library version, like "1.4.5". Requires v1.3.0+. 
 </p></pre><BR>
 
-<a name="Chapter3"></a><h2>Simple Core API</h2><pre></pre>
+<a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
 
 <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
                 const void* src, size_t srcSize,
                       int compressionLevel);
 </b><p>  Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
-  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
-        enough space to successfully compress the data.
+  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
   @return : compressed size written into `dst` (<= `dstCapacity),
             or an error code if it fails (which can be tested using ZSTD_isError()). 
 </p></pre><BR>
@@ -90,16 +88,12 @@ Note: the content of this file has been automatically generated by parsing "zstd
 <pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                   const void* src, size_t compressedSize);
 </b><p>  `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
-  Multiple compressed frames can be decompressed at once with this method.
-  The result will be the concatenation of all decompressed frames, back to back.
   `dstCapacity` is an upper bound of originalSize to regenerate.
-  First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
-  If maximum upper bound isn't known, prefer using streaming mode to decompress data.
+  If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
   @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
             or an errorCode if it fails (which can be tested using ZSTD_isError()). 
 </p></pre><BR>
 
-<h3>Decompression helper functions</h3><pre></pre><b><pre></pre></b><BR>
 <pre><b>#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)
 unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
@@ -110,8 +104,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
             - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
    note 1 : a 0 return value means the frame is valid but "empty".
-           When invoking this method on a skippable frame, it will return 0.
-  note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
+   note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
             When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
             In which case, it's necessary to use streaming mode to decompress data.
             Optionally, application can rely on some implicit limit,
@@ -129,8 +122,9 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
 </p></pre><BR>
 
 <pre><b>ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-</b><p>  This function is now obsolete, in favor of ZSTD_getFrameContentSize().
+</b><p>  NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
   Both functions work the same way, but ZSTD_getDecompressedSize() blends
   "empty", "unknown" and "error" results to the same return value (0),
   while ZSTD_getFrameContentSize() gives them separate return values.
@@ -143,42 +137,34 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
  @return : the compressed size of the first frame starting at `src`,
            suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
         or an error code if input is invalid 
-  Note 1: this method is called _find*() because it's not enough to read the header,
-          it may have to scan through the frame's content, to reach its end.
-  Note 2: this method also works with Skippable Frames. In which case,
-          it returns the size of the complete skippable frame,
-          which is always equal to its content size + 8 bytes for headers. 
 </p></pre><BR>
 
-<h3>Compression helper functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
+<h3>Helper functions</h3><pre></pre><b><pre></b>/* ZSTD_compressBound() :<b>
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
 #define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
 size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
-</b><p> maximum compressed size in worst case single-pass scenario.
- When invoking `ZSTD_compress()`, or any other one-pass compression function,
- it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
- as it eliminates one potential failure scenario,
- aka not enough room in dst buffer to write the compressed frame.
- Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
-        In which case, ZSTD_compressBound() will return an error code
-        which can be tested using ZSTD_isError().
-
- ZSTD_COMPRESSBOUND() :
- same as ZSTD_compressBound(), but as a macro.
- It can be used to produce constants, which can be useful for static allocation,
- for example to size a static array on stack.
- Will produce constant value 0 if srcSize is too large.
- 
-</p></pre><BR>
-
-<h3>Error helper functions</h3><pre></pre><b><pre></b>/* ZSTD_isError() :<b>
+</b>/* ZSTD_isError() :<b>
  * Most ZSTD_* functions returning a size_t value can be tested for error,
  * using ZSTD_isError().
  * @return 1 if error, 0 otherwise
  */
-unsigned     ZSTD_isError(size_t result);      </b>/*!< tells if a `size_t` function result is an error code */<b>
-ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); </b>/* convert a result into an error code, which can be compared to error enum list */<b>
-const char*  ZSTD_getErrorName(size_t result); </b>/*!< provides readable string from a function result */<b>
+unsigned    ZSTD_isError(size_t code);          </b>/*!< tells if a `size_t` function result is an error code */<b>
+const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable string from an error code */<b>
 int         ZSTD_minCLevel(void);               </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
 int         ZSTD_maxCLevel(void);               </b>/*!< maximum compression level available */<b>
 int         ZSTD_defaultCLevel(void);           </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */<b>
@@ -186,34 +172,34 @@ int          ZSTD_defaultCLevel(void);         </b>/*!< default compression leve
 <a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
 
 <h3>Compression context</h3><pre>  When compressing many times,
-  it is recommended to allocate a compression context just once,
-  and reuse it for each successive compression operation.
-  This will make the workload easier for system's memory.
+  it is recommended to allocate a context just once,
+  and re-use it for each successive compression operation.
+  This will make workload friendlier for system's memory.
   Note : re-using context is just a speed / resource optimization.
          It doesn't change the compression ratio, which remains identical.
-  Note 2: For parallel execution in multi-threaded environments,
-         use one different context per thread .
+  Note 2 : In multi-threaded environments,
+         use one different context per thread for parallel execution.
  
 </pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
 ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* compatible with NULL pointer */<b>
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);  </b>/* accept NULL pointer */<b>
 </pre></b><BR>
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
                          void* dst, size_t dstCapacity,
                    const void* src, size_t srcSize,
                          int compressionLevel);
 </b><p>  Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
-  Important : in order to mirror `ZSTD_compress()` behavior,
-  this function compresses at the requested compression level,
-  __ignoring any other advanced parameter__ .
+  Important : in order to behave similarly to `ZSTD_compress()`,
+  this function compresses at requested compression level,
+  __ignoring any other parameter__ .
   If any advanced parameter was set using the advanced API,
-  they will all be reset. Only @compressionLevel remains.
+  they will all be reset. Only `compressionLevel` remains.
  
 </p></pre><BR>
 
 <h3>Decompression context</h3><pre>  When decompressing many times,
   it is recommended to allocate a context only once,
-  and reuse it for each successive compression operation.
+  and re-use it for each successive compression operation.
   This will make workload friendlier for system's memory.
   Use one context per thread for parallel execution. 
 </pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
@@ -225,7 +211,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                      const void* src, size_t srcSize);
 </b><p>  Same as ZSTD_decompress(),
   requires an allocated ZSTD_DCtx.
-  Compatible with sticky parameters (see below).
+  Compatible with sticky parameters.
  
 </p></pre><BR>
 
@@ -309,19 +295,6 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                               * The higher the value of selected strategy, the more complex it is,
                               * resulting in stronger and slower compression.
                               * Special: value 0 means "use default strategy". */
-
-    ZSTD_c_targetCBlockSize=130, </b>/* v1.5.6+<b>
-                                  * Attempts to fit compressed block size into approximately targetCBlockSize.
-                                  * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
-                                  * Note that it's not a guarantee, just a convergence target (default:0).
-                                  * No target when targetCBlockSize == 0.
-                                  * This is helpful in low bandwidth streaming environments to improve end-to-end latency,
-                                  * when a client can make use of partial documents (a prominent example being Chrome).
-                                  * Note: this parameter is stable since v1.5.6.
-                                  * It was present as an experimental parameter in earlier versions,
-                                  * but it's not recommended using it with earlier library versions
-                                  * due to massive performance regressions.
-                                  */
     </b>/* LDM mode parameters */<b>
     ZSTD_c_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
                                      * This parameter is designed to improve compression ratio
@@ -401,14 +374,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_c_forceMaxWindow
      * ZSTD_c_forceAttachDict
      * ZSTD_c_literalCompressionMode
+     * ZSTD_c_targetCBlockSize
      * ZSTD_c_srcSizeHint
      * ZSTD_c_enableDedicatedDictSearch
      * ZSTD_c_stableInBuffer
      * ZSTD_c_stableOutBuffer
      * ZSTD_c_blockDelimiters
      * ZSTD_c_validateSequences
-     * ZSTD_c_blockSplitterLevel
-     * ZSTD_c_splitAfterSequences
+     * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
      * ZSTD_c_prefetchCDictTables
      * ZSTD_c_enableSeqProducerFallback
@@ -422,7 +395,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_c_experimentalParam3=1000,
      ZSTD_c_experimentalParam4=1001,
      ZSTD_c_experimentalParam5=1002,
-     </b>/* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */<b>
+     ZSTD_c_experimentalParam6=1003,
      ZSTD_c_experimentalParam7=1004,
      ZSTD_c_experimentalParam8=1005,
      ZSTD_c_experimentalParam9=1006,
@@ -435,8 +408,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_c_experimentalParam16=1013,
      ZSTD_c_experimentalParam17=1014,
      ZSTD_c_experimentalParam18=1015,
-     ZSTD_c_experimentalParam19=1016,
-     ZSTD_c_experimentalParam20=1017
+     ZSTD_c_experimentalParam19=1016
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>typedef struct {
@@ -510,13 +482,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
                        void* dst, size_t dstCapacity,
                  const void* src, size_t srcSize);
 </b><p>  Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
-  (note that this entry point doesn't even expose a compression level parameter).
   ZSTD_compress2() always starts a new frame.
   Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
   - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
   - The function is always blocking, returns when compression is completed.
-  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
-        enough space to successfully compress the data, though it is possible it fails for other reasons.
+  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
  @return : compressed size written into `dst` (<= `dstCapacity),
            or an error code if it fails (which can be tested using ZSTD_isError()).
  
@@ -541,7 +511,6 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
      * ZSTD_d_disableHuffmanAssembly
-     * ZSTD_d_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
@@ -549,8 +518,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
      ZSTD_d_experimentalParam4=1003,
-     ZSTD_d_experimentalParam5=1004,
-     ZSTD_d_experimentalParam6=1005
+     ZSTD_d_experimentalParam5=1004
 
 } ZSTD_dParameter;
 </b></pre><BR>
@@ -598,14 +566,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);  </b>/* accept NULL pointer */<b>
   A ZSTD_CStream object is required to track streaming operation.
   Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
   ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-  It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
+  It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
 
   For parallel execution, use one separate ZSTD_CStream per thread.
 
   note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
 
   Parameters are sticky : when starting a new compression on the same context,
-  it will reuse the same sticky parameters as previous compression session.
+  it will re-use the same sticky parameters as previous compression session.
   When in doubt, it's recommended to fully initialize the context before usage.
   Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
   ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
@@ -696,11 +664,6 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);  </b>/* accept NULL pointer */<b>
             only ZSTD_e_end or ZSTD_e_flush operations are allowed.
             Before starting a new compression job, or changing compression parameters,
             it is required to fully flush internal buffers.
-  - note: if an operation ends with an error, it may leave @cctx in an undefined state.
-          Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state.
-          In order to be re-employed after an error, a state must be reset,
-          which can be done explicitly (ZSTD_CCtx_reset()),
-          or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx())
  
 </p></pre><BR>
 
@@ -733,7 +696,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 <a name="Chapter9"></a><h2>Streaming decompression - HowTo</h2><pre>
   A ZSTD_DStream object is required to track streaming operations.
   Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-  ZSTD_DStream objects can be re-employed multiple times.
+  ZSTD_DStream objects can be re-used multiple times.
 
   Use ZSTD_initDStream() to start a new decompression operation.
  @return : recommended first input size
@@ -743,21 +706,16 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
   The function will update both `pos` fields.
   If `input.pos < input.size`, some input has not been consumed.
   It's up to the caller to present again remaining data.
-
   The function tries to flush all data decoded immediately, respecting output buffer size.
   If `output.pos < output.size`, decoder has flushed everything it could.
-
-  However, when `output.pos == output.size`, it's more difficult to know.
-  If @return > 0, the frame is not complete, meaning
-  either there is still some data left to flush within internal buffers,
-  or there is more input to read to complete the frame (or both).
+  But if `output.pos == output.size`, there might be some data left within internal buffers.,
   In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
   Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
  @return : 0 when a frame is completely decoded and fully flushed,
         or an error code, which can be tested using ZSTD_isError(),
         or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                 the return value is a suggested next input size (just a hint for better latency)
-                                that will never request more than the remaining content of the compressed frame.
+                                that will never request more than the remaining frame size.
  
 <BR></pre>
 
@@ -783,22 +741,15 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
  Function will update both input and output `pos` fields exposing current state via these fields:
  - `input.pos < input.size`, some input remaining and caller should provide remaining input
    on the next call.
- - `output.pos < output.size`, decoder flushed internal output buffer.
- - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
-   check ZSTD_decompressStream() @return value,
-   if > 0, invoke it again to flush remaining data to output.
+ - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+   call ZSTD_decompressStream() again to flush remaining data to output.
  Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
 
  @return : 0 when a frame is completely decoded and fully flushed,
            or an error code, which can be tested using ZSTD_isError(),
            or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
  
- Note: when an operation returns with an error code, the @zds state may be left in undefined state.
-       It's UB to invoke `ZSTD_decompressStream()` on such a state.
-       In order to re-use such a state, it must be first reset,
-       which can be done explicitly (`ZSTD_DCtx_reset()`),
-       or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`)
- 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
@@ -915,11 +866,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
 
 <a name="Chapter13"></a><h2>Advanced dictionary and prefix API (Requires v1.4.0+)</h2><pre>
  This API allows dictionaries to be used with ZSTD_compress2(),
- ZSTD_compressStream2(), and ZSTD_decompressDCtx().
- Dictionaries are sticky, they remain valid when same context is reused,
- they only reset when the context is reset
- with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
- In contrast, Prefixes are single-use.
+ ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
+ only reset with the context is reset with ZSTD_reset_parameters or
+ ZSTD_reset_session_and_parameters. Prefixes are single-use.
 <BR></pre>
 
 <pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
@@ -940,10 +889,6 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
            In such a case, dictionary buffer must outlive its users.
   Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
            to precisely select how dictionary content must be interpreted. 
-  Note 5 : This method does not benefit from LDM (long distance mode).
-           If you want to employ LDM on some large dictionary content,
-           prefer employing ZSTD_CCtx_refPrefix() described below.
- 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
@@ -967,7 +912,6 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);  </b>/* accept NULL pointer */<b>
   Decompression will need same prefix to properly regenerate data.
   Compressing with a prefix is similar in outcome as performing a diff and compressing it,
   but performs much faster, especially during decompression (compression speed is tunable with compression level).
-  This method is compatible with LDM (long distance mode).
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
   Note 1 : Prefix buffer is referenced. It **must** outlive compression.
@@ -1088,7 +1032,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
                                *
                                * Note: This field is optional. ZSTD_generateSequences() will calculate the value of
                                * 'rep', but repeat offsets do not necessarily need to be calculated from an external
-                               * sequence provider perspective. For example, ZSTD_compressSequences() does not
+                               * sequence provider's perspective. For example, ZSTD_compressSequences() does not
                                * use this 'rep' field at all (as of now).
                                */
 } ZSTD_Sequence;
@@ -1193,16 +1137,16 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 } ZSTD_literalCompressionMode_e;
 </b></pre><BR>
 <pre><b>typedef enum {
-  </b>/* Note: This enum controls features which are conditionally beneficial.<b>
-   * Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto),
-   * but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature.
+  </b>/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final<b>
+   * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
+   * or ZSTD_ps_disable allow for a force enable/disable the feature.
    */
   ZSTD_ps_auto = 0,         </b>/* Let the library automatically determine whether the feature shall be enabled */<b>
   ZSTD_ps_enable = 1,       </b>/* Force-enable the feature */<b>
   ZSTD_ps_disable = 2       </b>/* Do not use the feature */<b>
-} ZSTD_ParamSwitch_e;
+} ZSTD_paramSwitch_e;
 </b></pre><BR>
-<a name="Chapter15"></a><h2>Frame header and size functions</h2><pre></pre>
+<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>
 
 <pre><b>ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
 </b><p>  `src` should point to the start of a series of ZSTD encoded and/or skippable frames
@@ -1243,36 +1187,11 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
-</b><p>  srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX.
+</b><p>  srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
  @return : size of the Frame Header,
            or an error code (if srcSize is too small) 
 </p></pre><BR>
 
-<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e;
-</b></pre><BR>
-<pre><b>typedef struct {
-    unsigned long long frameContentSize; </b>/* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */<b>
-    unsigned long long windowSize;       </b>/* can be very large, up to <= frameContentSize */<b>
-    unsigned blockSizeMax;
-    ZSTD_FrameType_e frameType;          </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
-    unsigned headerSize;
-    unsigned dictID;                     </b>/* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */<b>
-    unsigned checksumFlag;
-    unsigned _reserved1;
-    unsigned _reserved2;
-} ZSTD_FrameHeader;
-</b></pre><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize);
-</b>/*! ZSTD_getFrameHeader_advanced() :<b>
- *  same as ZSTD_getFrameHeader(),
- *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
-</b><p>  decode Frame Header into `zfhPtr`, or requires larger `srcSize`.
- @return : 0 => header is complete, `zfhPtr` is correctly filled,
-          >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled,
-           or an error code, which can be tested using ZSTD_isError() 
-</p></pre><BR>
-
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
 </b><p> Zstd supports in-place decompression, where the input and output buffers overlap.
  In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
@@ -1319,9 +1238,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr,
 </p></pre><BR>
 
 <pre><b>typedef enum {
-  ZSTD_sf_noBlockDelimiters = 0,         </b>/* ZSTD_Sequence[] has no block delimiters, just sequences */<b>
-  ZSTD_sf_explicitBlockDelimiters = 1    </b>/* ZSTD_Sequence[] contains explicit block delimiters */<b>
-} ZSTD_SequenceFormat_e;
+  ZSTD_sf_noBlockDelimiters = 0,         </b>/* Representation of ZSTD_Sequence has no block delimiters, sequences only */<b>
+  ZSTD_sf_explicitBlockDelimiters = 1    </b>/* Representation of ZSTD_Sequence contains explicit block delimiters */<b>
+} ZSTD_sequenceFormat_e;
 </b></pre><BR>
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
 </b><p> `srcSize` : size of the input buffer
@@ -1332,37 +1251,19 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr,
  
 </p></pre><BR>
 
-<pre><b>ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
-ZSTDLIB_STATIC_API size_t
-ZSTD_generateSequences(ZSTD_CCtx* zc,
-           ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
-           const void* src, size_t srcSize);
-</b><p> WARNING: This function is meant for debugging and informational purposes ONLY!
- Its implementation is flawed, and it will be deleted in a future version.
- It is not guaranteed to succeed, as there are several cases where it will give
- up and fail. You should NOT use this function in production code.
-
- This function is deprecated, and will be removed in a future version.
-
- Generate sequences using ZSTD_compress2(), given a source buffer.
-
- @param zc The compression context to be used for ZSTD_compress2(). Set any
-           compression parameters you need on this context.
- @param outSeqs The output sequences buffer of size @p outSeqsSize
- @param outSeqsCapacity The size of the output sequences buffer.
-                    ZSTD_sequenceBound(srcSize) is an upper bound on the number
-                    of sequences that can be generated.
- @param src The source buffer to generate sequences from of size @p srcSize.
- @param srcSize The size of the source buffer.
+<pre><b></b><p> Generate sequences using ZSTD_compress2(), given a source buffer.
 
  Each block will end with a dummy sequence
  with offset == 0, matchLength == 0, and litLength == length of last literals.
  litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
  simply acts as a block delimiter.
 
- @returns The number of sequences generated, necessarily less than
-          ZSTD_sequenceBound(srcSize), or an error code that can be checked
-          with ZSTD_isError().
+ @zc can be used to insert custom compression params.
+ This function invokes ZSTD_compress2().
+
+ The output of this function can be fed into ZSTD_compressSequences() with CCtx
+ setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
+ @return : number of sequences generated
  
 </p></pre><BR>
 
@@ -1380,14 +1281,13 @@ ZSTD_generateSequences(ZSTD_CCtx* zc,
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t
-ZSTD_compressSequences(ZSTD_CCtx* cctx,
-           void* dst, size_t dstCapacity,
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
             const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
             const void* src, size_t srcSize);
 </b><p> Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
  @src contains the entire input (not just the literals).
  If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
- If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.).
+ If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
  The entire source is compressed into a single frame.
 
  The compression behavior changes based on cctx params. In particular:
@@ -1396,17 +1296,11 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx,
     the block size derived from the cctx, and sequences may be split. This is the default setting.
 
     If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
-    valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
+    block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
 
-    When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes
-    using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit
-    can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation.
-    By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10).
-    ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction.
-
-    If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined
-    behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for
-    specifics regarding offset/matchlength requirements) and then bail out and return an error.
+    If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
+    behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
+    specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
 
     In addition to the two adjustable experimental params, there are other important cctx params.
     - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
@@ -1414,47 +1308,21 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx,
     - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
       is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
 
- Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused.
- Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly,
-         and cannot emit an RLE block that disagrees with the repcode history.
- @return : final compressed size, or a ZSTD error code.
- 
-</p></pre><BR>
-
-<pre><b>ZSTDLIB_STATIC_API size_t
-ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
-                      void* dst, size_t dstCapacity,
-                const ZSTD_Sequence* inSeqs, size_t nbSequences,
-                const void* literals, size_t litSize, size_t litBufCapacity,
-                size_t decompressedSize);
-</b><p> This is a variant of ZSTD_compressSequences() which,
- instead of receiving (src,srcSize) as input parameter, receives (literals,litSize),
- aka all the literals, already extracted and laid out into a single continuous buffer.
- This can be useful if the process generating the sequences also happens to generate the buffer of literals,
- thus skipping an extraction + caching stage.
- It's a speed optimization, useful when the right conditions are met,
- but it also features the following limitations:
- - Only supports explicit delimiter mode
- - Currently does not support Sequences validation (so input Sequences are trusted)
- - Not compatible with frame checksum, which must be disabled
- - If any block is incompressible, will fail and return an error
- - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error.
- - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals.
-   @litBufCapacity must be at least 8 bytes larger than @litSize.
- - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error.
+ Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
+ Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
+         and cannot emit an RLE block that disagrees with the repcode history
  @return : final compressed size, or a ZSTD error code.
  
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
-                                 const void* src, size_t srcSize,
-                                       unsigned magicVariant);
+                                const void* src, size_t srcSize, unsigned magicVariant);
 </b><p> Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
 
  Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
  ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
- As such, the parameter magicVariant controls the exact skippable frame magic number variant used,
- so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+ As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
 
  Returns an error if destination buffer is not large enough, if the source size is not representable
  with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
@@ -1463,14 +1331,13 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
-                                      unsigned* magicVariant,
+<pre><b>size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
                                 const void* src, size_t srcSize);
-</b><p> Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer.
+</b><p> Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
 
- The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written,
- i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.
- This can be NULL if the caller is not interested in the magicVariant.
+ The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
+ i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
+ in the magicVariant.
 
  Returns an error if destination buffer is not large enough, or if the frame is not skippable.
 
@@ -1478,36 +1345,32 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
+<pre><b>unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
 </b><p>  Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
  
 </p></pre><BR>
 
 <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
 </b><p>  These functions make it possible to estimate memory usage
   of a future {D,C}Ctx, before its creation.
-  This is useful in combination with ZSTD_initStatic(),
-  which makes it possible to employ a static buffer for ZSTD_CCtx* state.
 
   ZSTD_estimateCCtxSize() will provide a memory budget large enough
-  to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2()
-  associated with any compression level up to max specified one.
+  for any compression level up to selected one.
+  Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
+         does not include space for a window buffer.
+         Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
   The estimate will assume the input may be arbitrarily large,
   which is the worst case.
 
-  Note that the size estimation is specific for one-shot compression,
-  it is not valid for streaming (see ZSTD_estimateCStreamSize*())
-  nor other potential ways of using a ZSTD_CCtx* state.
-
   When srcSize can be bound by a known and rather "small" value,
-  this knowledge can be used to provide a tighter budget estimation
-  because the ZSTD_CCtx* state will need less memory for small inputs.
-  This tighter estimation can be provided by employing more advanced functions
+  this fact can be used to provide a tighter estimation
+  because the CCtx compression context will need less memory.
+  This tighter estimation can be provided by more advanced functions
   ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
   and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
   Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
@@ -1515,31 +1378,32 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
   Note : only single-threaded compression is supported.
   ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
 
+  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+  Size estimates assume that no external sequence producer is registered.
+ 
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel);
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
-ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize);
+ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
-</b><p>  ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression
-  using any compression level up to the max specified one.
-  It will also consider src size to be arbitrarily "large", which is a worst case scenario.
+</b><p>  ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
+  It will also consider src size to be arbitrarily "large", which is worst case.
   If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
   ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
   ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
   Note : CStream size estimation is only correct for single-threaded compression.
-  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
-  Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
-  Size estimates assume that no external sequence producer is registered.
-
-  ZSTD_DStream memory budget depends on frame's window Size.
+  ZSTD_DStream memory budget depends on window Size.
   This information can be passed manually, using ZSTD_estimateDStreamSize,
   or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
-  Any frame requesting a window size larger than max specified one will be rejected.
   Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
          an internal ?Dict will be created, which additional size is not estimated here.
          In this case, get total size by adding ZSTD_estimate?DictSize
+  Note 2 : only single-threaded compression is supported.
+  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+  Size estimates assume that no external sequence producer is registered.
  
 </p></pre><BR>
 
@@ -1585,14 +1449,13 @@ static
 #ifdef __GNUC__
 __attribute__((__unused__))
 #endif
+ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this constant defers to stdlib's functions */<b>
 </b><p>  These prototypes make it possible to pass your own allocation/free functions.
   ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
   All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
  
 </p></pre><BR>
 
-<pre><b>ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL };  </b>/**< this constant defers to stdlib's functions */<b>
-</b></pre><BR>
 <pre><b>typedef struct POOL_ctx_s ZSTD_threadPool;
 ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
 ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool);  </b>/* accept NULL pointer */<b>
@@ -1642,24 +1505,10 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPo
 </p></pre><BR>
 
 <pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
-</b><p>  Set all parameters provided within @p cparams into the working @p cctx.
+</b><p>  Set all parameters provided within @cparams into the working @cctx.
   Note : if modifying parameters during compression (MT mode only),
          note that changes to the .windowLog parameter will be ignored.
- @return 0 on success, or an error code (can be checked with ZSTD_isError()).
-         On failure, no parameters are updated.
- 
-</p></pre><BR>
-
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
-</b><p>  Set all parameters provided within @p fparams into the working @p cctx.
- @return 0 on success, or an error code (can be checked with ZSTD_isError()).
- 
-</p></pre><BR>
-
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
-</b><p>  Set all parameters provided within @p params into the working @p cctx.
- @return 0 on success, or an error code (can be checked with ZSTD_isError()).
- 
+ @return 0 on success, or an error code (can be checked with ZSTD_isError()) 
 </p></pre><BR>
 
 <pre><b>ZSTD_DEPRECATED("use ZSTD_compress2")
@@ -1905,9 +1754,12 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
         const void* dict, size_t dictSize,
               ZSTD_parameters params,
               unsigned long long pledgedSrcSize);
-</b><p> This function is DEPRECATED, and is equivalent to:
+</b><p> This function is DEPRECATED, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
-     ZSTD_CCtx_setParams(zcs, params);
+     // Pseudocode: Set each zstd parameter and leave the rest as-is.
+     for ((param, value) : params) {
+         ZSTD_CCtx_setParameter(zcs, param, value);
+     }
      ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
      ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
 
@@ -1936,9 +1788,12 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                    const ZSTD_CDict* cdict,
                          ZSTD_frameParameters fParams,
                          unsigned long long pledgedSrcSize);
-</b><p>   This function is DEPRECATED, and is equivalent to:
+</b><p>   This function is DEPRECATED, and is approximately equivalent to:
      ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
-     ZSTD_CCtx_setFParams(zcs, fParams);
+     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
+     for ((fParam, value) : fParams) {
+         ZSTD_CCtx_setParameter(zcs, fParam, value);
+     }
      ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
      ZSTD_CCtx_refCDict(zcs, cdict);
 
@@ -1960,7 +1815,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
        explicitly specified.
 
   start a new frame, using same parameters from previous frame.
-  This is typically useful to skip dictionary loading stage, since it will reuse it in-place.
+  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
   Note that zcs must be init at least once before using ZSTD_resetCStream().
   If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
   If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
@@ -2021,69 +1876,21 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 </b><p>
      ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 
- reuse decompression parameters from previous init; saves dictionary loading
+ re-use decompression parameters from previous init; saves dictionary loading
  
 </p></pre><BR>
 
-<pre><b>ZSTDLIB_STATIC_API void
-ZSTD_registerSequenceProducer(
-  ZSTD_CCtx* cctx,
-  void* sequenceProducerState,
-  ZSTD_sequenceProducer_F sequenceProducer
-);
-</b><p> Instruct zstd to use a block-level external sequence producer function.
-
- The sequenceProducerState must be initialized by the caller, and the caller is
- responsible for managing its lifetime. This parameter is sticky across
- compressions. It will remain set until the user explicitly resets compression
- parameters.
-
- Sequence producer registration is considered to be an "advanced parameter",
- part of the "advanced API". This means it will only have an effect on compression
- APIs which respect advanced parameters, such as compress2() and compressStream2().
- Older compression APIs such as compressCCtx(), which predate the introduction of
- "advanced parameters", will ignore any external sequence producer setting.
-
- The sequence producer can be "cleared" by registering a NULL function pointer. This
- removes all limitations described above in the "LIMITATIONS" section of the API docs.
-
- The user is strongly encouraged to read the full API documentation (above) before
- calling this function. 
-</p></pre><BR>
-
-<pre><b>ZSTDLIB_STATIC_API void
-ZSTD_CCtxParams_registerSequenceProducer(
-  ZSTD_CCtx_params* params,
-  void* sequenceProducerState,
-  ZSTD_sequenceProducer_F sequenceProducer
-);
-</b><p> Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params.
- This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(),
- which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx().
-
- If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx()
- is required, then this function is for you. Otherwise, you probably don't need it.
-
- See tests/zstreamtest.c for example usage. 
-</p></pre><BR>
-
-<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions (DEPRECATED)</h2><pre>
-  This API is deprecated, and will be removed in a future version.
-  It allows streaming (de)compression with user allocated buffers.
-  However, it is hard to use, and not as well tested as the rest of
-  our API.
-
-  Please use the normal streaming API instead: ZSTD_compressStream2,
-  and ZSTD_decompressStream.
-  If there is functionality that you need, but it doesn't provide,
-  please open an issue on our GitHub.
+<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
+  This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
+  But it's also a complex one, with several restrictions, documented below.
+  Prefer normal streaming API for an easier experience.
  
 <BR></pre>
 
 <a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
-  ZSTD_CCtx object can be reused multiple times within successive compression operations.
+  ZSTD_CCtx object can be re-used multiple times within successive compression operations.
 
   Start by initializing a context.
   Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
@@ -2104,14 +1911,11 @@ ZSTD_CCtxParams_registerSequenceProducer(
   It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
   Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
 
-  `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again.
+  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
 <BR></pre>
 
-<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
-ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
-ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
+<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
 </pre></b><BR>
 <pre><b>size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
@@ -2121,7 +1925,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
 <a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
-  A ZSTD_DCtx object can be reused multiple times.
+  A ZSTD_DCtx object can be re-used multiple times.
 
   First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
@@ -2131,7 +1935,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
            >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
-  It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame,
+  It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
   such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
   Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
   As a consequence, check that values remain within valid application range.
@@ -2189,25 +1993,36 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
   For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
 <BR></pre>
 
-<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
-<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
-</b></pre><BR>
+<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+typedef struct {
+    unsigned long long frameContentSize; </b>/* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */<b>
+    unsigned long long windowSize;       </b>/* can be very large, up to <= frameContentSize */<b>
+    unsigned blockSizeMax;
+    ZSTD_frameType_e frameType;          </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
+    unsigned headerSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
+} ZSTD_frameHeader;
+</pre></b><BR>
+<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   </b>/**< doesn't consume input */<b>
+</b>/*! ZSTD_getFrameHeader_advanced() :<b>
+ *  same as ZSTD_getFrameHeader(),
+ *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
+</b><p>  decode Frame Header, or requires larger `srcSize`.
+ @return : 0, `zfhPtr` is correctly filled,
+          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+           or an error code, which can be tested using ZSTD_isError() 
+</p></pre><BR>
+
 <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 </b></pre><BR>
-<a name="Chapter23"></a><h2>Block level API (DEPRECATED)</h2><pre></pre>
+<a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
 
-<pre><b></b><p>    You can get the frame header down to 2 bytes by setting:
-      - ZSTD_c_format = ZSTD_f_zstd1_magicless
-      - ZSTD_c_contentSizeFlag = 0
-      - ZSTD_c_checksumFlag = 0
-      - ZSTD_c_dictIDFlag = 0
-
-    This API is not as well tested as our normal API, so we recommend not using it.
-    We will be removing it in a future version. If the normal API doesn't provide
-    the functionality you need, please open a GitHub issue.
-
-    Block functions produce and decode raw zstd blocks, without frame metadata.
-    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
+<pre><b></b><p>    Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
 
     A few rules to respect :
@@ -2231,14 +2046,36 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
         Use ZSTD_insertBlock() for such a case.
 </p></pre><BR>
 
-<h3>Raw zstd block functions</h3><pre></pre><b><pre>ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
-ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
-ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
+<h3>Raw zstd block functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
 </pre></b><BR>
+<pre><b>ZSTDLIB_STATIC_API void
+ZSTD_registerSequenceProducer(
+  ZSTD_CCtx* cctx,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F* sequenceProducer
+);
+</b><p> Instruct zstd to use a block-level external sequence producer function.
+
+ The sequenceProducerState must be initialized by the caller, and the caller is
+ responsible for managing its lifetime. This parameter is sticky across
+ compressions. It will remain set until the user explicitly resets compression
+ parameters.
+
+ Sequence producer registration is considered to be an "advanced parameter",
+ part of the "advanced API". This means it will only have an effect on compression
+ APIs which respect advanced parameters, such as compress2() and compressStream2().
+ Older compression APIs such as compressCCtx(), which predate the introduction of
+ "advanced parameters", will ignore any external sequence producer setting.
+
+ The sequence producer can be "cleared" by registering a NULL function pointer. This
+ removes all limitations described above in the "LIMITATIONS" section of the API docs.
+
+ The user is strongly encouraged to read the full API documentation (above) before
+ calling this function. 
+</p></pre><BR>
+
 </html>
 </body>

examples/streaming_compression.c

@@ -42,13 +42,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
      */
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
     CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
-    if (nbThreads > 1) {
-        size_t const r = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
-        if (ZSTD_isError(r)) {
-            fprintf (stderr, "Note: the linked libzstd library doesn't support multithreading. "
-                             "Reverting to single-thread mode. \n");
-        }
-    }
+    ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
 
     /* This loop read from the input file, compresses that entire chunk,
      * and writes all output produced to the output file.
@@ -123,7 +117,7 @@ int main(int argc, const char** argv)
     }
 
     int cLevel = 1;
-    int nbThreads = 1;
+    int nbThreads = 4;
 
     if (argc >= 3) {
       cLevel = atoi (argv[2]);

lib/Makefile

@@ -8,9 +8,6 @@
 # You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
-# default target (when running `make` with no argument)
-lib-release:
-
 # Modules
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
@@ -57,14 +54,13 @@ VERSION := $(ZSTD_VERSION)
 # Note: by default, the static library is built single-threaded and dynamic library is built
 # multi-threaded. It is possible to force multi or single threaded builds by appending
 # -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
-
+.PHONY: default
+default: lib-release
 
 CPPFLAGS_DYNLIB  += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
 LDFLAGS_DYNLIB   += -pthread
-CPPFLAGS_STATICLIB +=                  # static library build defaults to single-threaded
+CPPFLAGS_STATLIB +=                    # static library build defaults to single-threaded
 
-# pkg-config Libs.private points to LDFLAGS_DYNLIB
-PCLIB := $(LDFLAGS_DYNLIB)
 
 ifeq ($(findstring GCC,$(CCVER)),GCC)
 decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
@@ -73,15 +69,13 @@ endif
 
 # macOS linker doesn't support -soname, and use different extension
 # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
-UNAME_TARGET_SYSTEM ?= $(UNAME)
-
-ifeq ($(UNAME_TARGET_SYSTEM), Darwin)
+ifeq ($(UNAME), Darwin)
   SHARED_EXT = dylib
   SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
   SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
   SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
-  ifeq ($(UNAME_TARGET_SYSTEM), AIX)
+  ifeq ($(UNAME), AIX)
     SONAME_FLAGS =
   else
     SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
@@ -97,7 +91,7 @@ all: lib
 
 
 .PHONY: libzstd.a  # must be run every time
-libzstd.a: CPPFLAGS += $(CPPFLAGS_STATICLIB)
+libzstd.a: CPPFLAGS += $(CPPFLAGS_STATLIB)
 
 SET_CACHE_DIRECTORY = \
    +$(MAKE) --no-print-directory $@ \
@@ -115,19 +109,19 @@ libzstd.a:
 else
 # BUILD_DIR is defined
 
-ZSTD_STATICLIB_DIR := $(BUILD_DIR)/static
-ZSTD_STATICLIB := $(ZSTD_STATICLIB_DIR)/libzstd.a
-ZSTD_STATICLIB_OBJ := $(addprefix $(ZSTD_STATICLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
-$(ZSTD_STATICLIB): ARFLAGS = rcs
-$(ZSTD_STATICLIB): | $(ZSTD_STATICLIB_DIR)
-$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ)
+ZSTD_STATLIB_DIR := $(BUILD_DIR)/static
+ZSTD_STATLIB := $(ZSTD_STATLIB_DIR)/libzstd.a
+ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
+$(ZSTD_STATLIB): ARFLAGS = rcs
+$(ZSTD_STATLIB): | $(ZSTD_STATLIB_DIR)
+$(ZSTD_STATLIB): $(ZSTD_STATLIB_OBJ)
   # Check for multithread flag at target execution time
 	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
     @echo compiling multi-threaded static library $(LIBVER),\
     @echo compiling single-threaded static library $(LIBVER))
 	$(AR) $(ARFLAGS) $@ $^
 
-libzstd.a: $(ZSTD_STATICLIB)
+libzstd.a: $(ZSTD_STATLIB)
 	cp -f $< $@
 
 endif
@@ -166,7 +160,7 @@ $(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
 	$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
     @echo compiling multi-threaded dynamic library $(LIBVER),\
     @echo compiling single-threaded dynamic library $(LIBVER))
-	$(CC) $(FLAGS) $^ $(SONAME_FLAGS) -o $@
+	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
 	ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
 	ln -sf $@ libzstd.$(SHARED_EXT)
@@ -188,17 +182,14 @@ lib : libzstd.a libzstd
 # make does not consider implicit pattern rule for .PHONY target
 
 %-mt : CPPFLAGS_DYNLIB  := -DZSTD_MULTITHREAD
-%-mt : CPPFLAGS_STATICLIB := -DZSTD_MULTITHREAD
+%-mt : CPPFLAGS_STATLIB := -DZSTD_MULTITHREAD
 %-mt : LDFLAGS_DYNLIB   := -pthread
-%-mt : PCLIB :=
-%-mt : PCMTLIB := $(LDFLAGS_DYNLIB)
 %-mt : %
 	@echo multi-threaded build completed
 
 %-nomt : CPPFLAGS_DYNLIB  :=
 %-nomt : LDFLAGS_DYNLIB   :=
-%-nomt : CPPFLAGS_STATICLIB :=
-%-nomt : PCLIB :=
+%-nomt : CPPFLAGS_STATLIB :=
 %-nomt : %
 	@echo single-threaded build completed
 
@@ -209,53 +200,43 @@ lib : libzstd.a libzstd
 
 # Generate .h dependencies automatically
 
-# -MMD: compiler generates dependency information as a side-effect of compilation, without system headers
-# -MP: adds phony target for each dependency other than main file.
-DEPFLAGS = -MMD -MP
+DEPFLAGS = -MT $@ -MMD -MP -MF
 
-# ensure that ZSTD_DYNLIB_DIR exists prior to generating %.o
-$(ZSTD_DYNLIB_DIR)/%.o : %.c | $(ZSTD_DYNLIB_DIR)
+$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR)
 	@echo CC $@
-	$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
+	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
 
-$(ZSTD_STATICLIB_DIR)/%.o : %.c | $(ZSTD_STATICLIB_DIR)
+$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
 	@echo CC $@
-	$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
+	$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
 
 $(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
 	@echo AS $@
 	$(COMPILE.S) $(OUTPUT_OPTION) $<
 
-$(ZSTD_STATICLIB_DIR)/%.o : %.S | $(ZSTD_STATICLIB_DIR)
+$(ZSTD_STATLIB_DIR)/%.o : %.S | $(ZSTD_STATLIB_DIR)
 	@echo AS $@
 	$(COMPILE.S) $(OUTPUT_OPTION) $<
 
-MKDIR ?= mkdir -p
-$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATICLIB_DIR):
-	$(MKDIR) $@
+MKDIR ?= mkdir
+$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
+	$(MKDIR) -p $@
 
-DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATICLIB_OBJ:.o=.d)
+DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d)
 $(DEPFILES):
 
-# The leading '-' means: do not fail is include fails (ex: directory does not exist yet)
--include $(wildcard $(DEPFILES))
+include $(wildcard $(DEPFILES))
 
 
-# Special case : build library in single-thread mode _and_ without zstdmt_compress.c
-# Note : we still need threading.c and pool.c for the dictionary builder,
-# but they will correctly behave single-threaded.
-ZSTDMT_FILES = zstdmt_compress.c
-ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(notdir $(ZSTD_FILES)))
+# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
+ZSTDMT_FILES = compress/zstdmt_compress.c
+ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
 libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
 libzstd-nomt: LDFLAGS += -shared
 libzstd-nomt: $(ZSTD_NOMT_FILES)
 	@echo compiling single-thread dynamic library $(LIBVER)
 	@echo files : $(ZSTD_NOMT_FILES)
-	@if echo "$(ZSTD_NOMT_FILES)" | tr ' ' '\n' | $(GREP) -q zstdmt; then \
-        echo "Error: Found zstdmt in list."; \
-        exit 1; \
-    fi
-	$(CC) $(FLAGS) $^ $(SONAME_FLAGS) -o $@
+	$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 
 .PHONY: clean
 clean:
@@ -268,7 +249,7 @@ clean:
 #-----------------------------------------------------------------------------
 # make install is validated only for below listed environments
 #-----------------------------------------------------------------------------
-ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT%,$(UNAME)))
+ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
 
 lib: libzstd.pc
 
@@ -299,21 +280,13 @@ PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
 # to PREFIX, rather than as a resolved value.
 PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})
 
-
-ifneq ($(MT),)
-  PCLIB :=
-  PCMTLIB := $(LDFLAGS_DYNLIB)
-else
-  PCLIB := $(LDFLAGS_DYNLIB)
-endif
-
-ifneq (,$(filter FreeBSD NetBSD DragonFly,$(UNAME)))
+ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
   PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
   PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif
 
-ifneq (,$(filter SunOS,$(UNAME)))
+ifneq (,$(filter $(UNAME),SunOS))
   INSTALL ?= ginstall
 else
   INSTALL ?= install
@@ -323,10 +296,6 @@ INSTALL_PROGRAM ?= $(INSTALL)
 INSTALL_DATA    ?= $(INSTALL) -m 644
 
 
-# pkg-config library define.
-# For static single-threaded library declare -pthread in Libs.private
-# For static multi-threaded library declare -pthread in Libs and Cflags
-.PHONY: libzstd.pc
 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
 	@sed \
@@ -335,8 +304,7 @@ libzstd.pc: libzstd.pc.in
 	        -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
 	        -e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
 	        -e 's|@VERSION@|$(VERSION)|' \
-	        -e 's|@LIBS_MT@|$(PCMTLIB)|' \
-	        -e 's|@LIBS_PRIVATE@|$(PCLIB)|' \
+	        -e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \
 	        $< >$@
 
 .PHONY: install

lib/README.md

@@ -27,16 +27,12 @@ Enabling multithreading requires 2 conditions :
 
 For convenience, we provide a build target to generate multi and single threaded libraries:
 - Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
-  Note that the `.pc` generated on calling `make lib-mt` will already include the require Libs and Cflags.
 - Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
 - By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.
 
 When linking a POSIX program with a multithreaded version of `libzstd`,
 note that it's necessary to invoke the `-pthread` flag during link stage.
 
-The `.pc` generated from `make install` or `make install-pc` always assume a single-threaded static library
-is compiled. To correctly generate a `.pc` for the multi-threaded static library, set `MT=1` as ENV variable.
-
 Multithreading capabilities are exposed
 via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
 
@@ -92,7 +88,7 @@ The file structure is designed to make this selection manually achievable for an
         For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
 
 - While invoking `make libzstd`, it's possible to define build macros
-        `ZSTD_LIB_COMPRESSION`, `ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
+        `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
         and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the
         corresponding features. This will also disable compilation of all
         dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable
@@ -123,15 +119,6 @@ The file structure is designed to make this selection manually achievable for an
   binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
   `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`).
 
-  On the compressor side, Zstd's compression levels map to several internal
-  strategies. In environments where the higher compression levels aren't used,
-  it is possible to exclude all but the fastest strategy with
-  `ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1`. (Note that this will change
-  the behavior of the default compression level.) Or if you want to retain the
-  default compressor as well, you can set
-  `ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1`, at the cost of an additional
-  ~20KB or so.
-
   For squeezing the last ounce of size out, you can also define
   `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
   which removes the error messages that are otherwise returned by
@@ -149,13 +136,6 @@ The file structure is designed to make this selection manually achievable for an
   will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
   the shared library, which is now hidden by default.
 
-- The build macro `STATIC_BMI2` can be set to 1 to force usage of `bmi2` instructions.
-  It is generally not necessary to set this build macro,
-  because `STATIC_BMI2` will be automatically set to 1
-  on detecting the presence of the corresponding instruction set in the compilation target.
-  It's nonetheless available as an optional manual toggle for better control,
-  and can also be used to forcefully disable `bmi2` instructions by setting it to 0.
-
 - The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
   which can detect at runtime the presence of BMI2 instructions, and use them only if present.
   These instructions contribute to better performance, notably on the decoder side.
@@ -189,15 +169,6 @@ The file structure is designed to make this selection manually achievable for an
   `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
   with the old macro names.
 
-- The C compiler macro `HUF_DISABLE_FAST_DECODE` disables the newer Huffman fast C
-  and assembly decoding loops. You may want to use this macro if these loops are
-  slower on your platform.
-
-- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant,
-  which is useful when autodetection fails, for example with older versions of `musl`.
-  For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
-  Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
-
 #### Windows : using MinGW+MSYS to create DLL
 
 DLL can be created using MinGW+MSYS with the `make libzstd` command.

lib/common/allocations.h

@@ -1,55 +0,0 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under both the BSD-style license (found in the
- * LICENSE file in the root directory of this source tree) and the GPLv2 (found
- * in the COPYING file in the root directory of this source tree).
- * You may select, at your option, one of the above-listed licenses.
- */
-
-/* This file provides custom allocation primitives
- */
-
-#define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
-
-#include "compiler.h" /* MEM_STATIC */
-#define ZSTD_STATIC_LINKING_ONLY
-#include "../zstd.h" /* ZSTD_customMem */
-
-#ifndef ZSTD_ALLOCATIONS_H
-#define ZSTD_ALLOCATIONS_H
-
-/* custom memory allocation functions */
-
-MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
-{
-    if (customMem.customAlloc)
-        return customMem.customAlloc(customMem.opaque, size);
-    return ZSTD_malloc(size);
-}
-
-MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
-{
-    if (customMem.customAlloc) {
-        /* calloc implemented as malloc+memset;
-         * not as efficient as calloc, but next best guess for custom malloc */
-        void* const ptr = customMem.customAlloc(customMem.opaque, size);
-        ZSTD_memset(ptr, 0, size);
-        return ptr;
-    }
-    return ZSTD_calloc(1, size);
-}
-
-MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
-{
-    if (ptr!=NULL) {
-        if (customMem.customFree)
-            customMem.customFree(customMem.opaque, ptr);
-        else
-            ZSTD_free(ptr);
-    }
-}
-
-#endif /* ZSTD_ALLOCATIONS_H */

lib/common/bits.h

@@ -17,11 +17,11 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
 {
     assert(val != 0);
     {
-        static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
+        static const int DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
                                                 30, 22, 20, 15, 25, 17, 4, 8,
                                                 31, 27, 13, 23, 21, 19, 16, 7,
                                                 26, 12, 18, 6, 11, 5, 10, 9};
-        return DeBruijnBytePos[((U32) ((val & (0-val)) * 0x077CB531U)) >> 27];
+        return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
     }
 }
 
@@ -29,28 +29,26 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
 {
     assert(val != 0);
 #   if defined(_MSC_VER)
-#  if STATIC_BMI2
-    return (unsigned)_tzcnt_u32(val);
+#       if STATIC_BMI2 == 1
+            return _tzcnt_u32(val);
 #       else
             if (val != 0) {
                 unsigned long r;
                 _BitScanForward(&r, val);
                 return (unsigned)r;
             } else {
-        __assume(0); /* Should not reach this code path */
+                /* Should not reach this code path */
+                __assume(0);
             }
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4)
         return (unsigned)__builtin_ctz(val);
-#elif defined(__ICCARM__)
-    return (unsigned)__builtin_ctz(val);
 #   else
         return ZSTD_countTrailingZeros32_fallback(val);
 #   endif
 }
 
-MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
-{
+MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
     assert(val != 0);
     {
         static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
@@ -70,21 +68,20 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
 {
     assert(val != 0);
 #   if defined(_MSC_VER)
-#  if STATIC_BMI2
-    return (unsigned)_lzcnt_u32(val);
+#       if STATIC_BMI2 == 1
+            return _lzcnt_u32(val);
 #       else
             if (val != 0) {
                 unsigned long r;
                 _BitScanReverse(&r, val);
                 return (unsigned)(31 - r);
             } else {
-        __assume(0); /* Should not reach this code path */
+                /* Should not reach this code path */
+                __assume(0);
             }
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4)
         return (unsigned)__builtin_clz(val);
-#elif defined(__ICCARM__)
-    return (unsigned)__builtin_clz(val);
 #   else
         return ZSTD_countLeadingZeros32_fallback(val);
 #   endif
@@ -94,21 +91,20 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
 {
     assert(val != 0);
 #   if defined(_MSC_VER) && defined(_WIN64)
-#  if STATIC_BMI2
-    return (unsigned)_tzcnt_u64(val);
+#       if STATIC_BMI2 == 1
+            return _tzcnt_u64(val);
 #       else
             if (val != 0) {
                 unsigned long r;
                 _BitScanForward64(&r, val);
                 return (unsigned)r;
             } else {
-        __assume(0); /* Should not reach this code path */
+                /* Should not reach this code path */
+                __assume(0);
             }
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
         return (unsigned)__builtin_ctzll(val);
-#elif defined(__ICCARM__)
-    return (unsigned)__builtin_ctzll(val);
 #   else
         {
             U32 mostSignificantWord = (U32)(val >> 32);
@@ -126,21 +122,20 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
 {
     assert(val != 0);
 #   if defined(_MSC_VER) && defined(_WIN64)
-#  if STATIC_BMI2
-    return (unsigned)_lzcnt_u64(val);
+#       if STATIC_BMI2 == 1
+            return _lzcnt_u64(val);
 #       else
             if (val != 0) {
                 unsigned long r;
                 _BitScanReverse64(&r, val);
                 return (unsigned)(63 - r);
             } else {
-        __assume(0); /* Should not reach this code path */
+                /* Should not reach this code path */
+                __assume(0);
             }
 #       endif
 #   elif defined(__GNUC__) && (__GNUC__ >= 4)
         return (unsigned)(__builtin_clzll(val));
-#elif defined(__ICCARM__)
-    return (unsigned)(__builtin_clzll(val));
 #   else
         {
             U32 mostSignificantWord = (U32)(val >> 32);
@@ -177,29 +172,4 @@ MEM_STATIC unsigned ZSTD_highbit32(U32 val)   /* compress, dictBuilder, decodeCo
     return 31 - ZSTD_countLeadingZeros32(val);
 }
 
-/* ZSTD_rotateRight_*():
- * Rotates a bitfield to the right by "count" bits.
- * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
- */
-MEM_STATIC
-U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
-    assert(count < 64);
-    count &= 0x3F; /* for fickle pattern recognition */
-    return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
-}
-
-MEM_STATIC
-U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
-    assert(count < 32);
-    count &= 0x1F; /* for fickle pattern recognition */
-    return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
-}
-
-MEM_STATIC
-U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
-    assert(count < 16);
-    count &= 0x0F; /* for fickle pattern recognition */
-    return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
-}
-
 #endif /* ZSTD_BITS_H */

lib/common/bitstream.h

@@ -14,6 +14,9 @@
 #ifndef BITSTREAM_H_MODULE
 #define BITSTREAM_H_MODULE
 
+#if defined (__cplusplus)
+extern "C" {
+#endif
 /*
 *  This API consists of small unitary functions, which must be inlined for best performance.
 *  Since link-time-optimization is not available for all compilers,
@@ -29,6 +32,7 @@
 #include "error_private.h"  /* error codes and messages */
 #include "bits.h"           /* ZSTD_highbit32 */
 
+
 /*=========================================
 *  Target specific
 =========================================*/
@@ -48,13 +52,12 @@
 /*-******************************************
 *  bitStream encoding API (write forward)
 ********************************************/
-typedef size_t BitContainerType;
 /* bitStream can mix input from multiple sources.
  * A critical property of these streams is that they encode and decode in **reverse** direction.
  * So the first bit sequence you add will be the last to be read, like a LIFO stack.
  */
 typedef struct {
-    BitContainerType bitContainer;
+    size_t bitContainer;
     unsigned bitPos;
     char*  startPtr;
     char*  ptr;
@@ -62,7 +65,7 @@ typedef struct {
 } BIT_CStream_t;
 
 MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
-MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
+MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
 MEM_STATIC void   BIT_flushBits(BIT_CStream_t* bitC);
 MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 
@@ -71,7 +74,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 *  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
 *
 *  bits are first added to a local register.
-*  Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
+*  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
 *  Writing data into memory is an explicit operation, performed by the flushBits function.
 *  Hence keep track how many bits are potentially stored into local register to avoid register overflow.
 *  After a flushBits, a maximum of 7 bits might still be stored into local register.
@@ -88,28 +91,28 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 *  bitStream decoding API (read backward)
 **********************************************/
 typedef struct {
-    BitContainerType bitContainer;
+    size_t   bitContainer;
     unsigned bitsConsumed;
     const char* ptr;
     const char* start;
     const char* limitPtr;
 } BIT_DStream_t;
 
-typedef enum { BIT_DStream_unfinished = 0,  /* fully refilled */
-               BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */
-               BIT_DStream_completed = 2,   /* bitstream entirely consumed, bit-exact */
-               BIT_DStream_overflow = 3     /* user requested more bits than present in bitstream */
-    } BIT_DStream_status;  /* result of BIT_reloadDStream() */
+typedef enum { BIT_DStream_unfinished = 0,
+               BIT_DStream_endOfBuffer = 1,
+               BIT_DStream_completed = 2,
+               BIT_DStream_overflow = 3 } BIT_DStream_status;  /* result of BIT_reloadDStream() */
+               /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
 
 MEM_STATIC size_t   BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
-MEM_STATIC BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
+MEM_STATIC size_t   BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
 MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
 
 
 /* Start by invoking BIT_initDStream().
 *  A chunk of the bitStream is then stored into a local register.
-*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType).
+*  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
 *  You can then retrieve bitFields stored into the local register, **in reverse order**.
 *  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
 *  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
@@ -121,7 +124,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
 /*-****************************************
 *  unsafe API
 ******************************************/
-MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
+MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
 /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
 
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
@@ -159,15 +162,10 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
     return 0;
 }
 
-FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits)
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 {
-#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS)
-#  if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__)
+#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
     return  _bzhi_u64(bitContainer, nbBits);
-#  else
-    DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32));
-    return _bzhi_u32(bitContainer, nbBits);
-#  endif
 #else
     assert(nbBits < BIT_MASK_SIZE);
     return bitContainer & BIT_mask[nbBits];
@@ -178,7 +176,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitCont
  *  can add up to 31 bits into `bitC`.
  *  Note : does not check for register overflow ! */
 MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
-                            BitContainerType value, unsigned nbBits)
+                            size_t value, unsigned nbBits)
 {
     DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
     assert(nbBits < BIT_MASK_SIZE);
@@ -191,7 +189,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
  *  works only if `value` is _clean_,
  *  meaning all high bits above nbBits are 0 */
 MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
-                                BitContainerType value, unsigned nbBits)
+                                size_t value, unsigned nbBits)
 {
     assert((value>>nbBits) == 0);
     assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
@@ -238,7 +236,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
     BIT_addBitsFast(bitC, 1, 1);   /* endMark */
     BIT_flushBits(bitC);
     if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
-    return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
+    return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
 }
 
 
@@ -269,22 +267,22 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->bitContainer = *(const BYTE*)(bitD->start);
         switch(srcSize)
         {
-        case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
+        case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
                 ZSTD_FALLTHROUGH;
 
-        case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
+        case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
                 ZSTD_FALLTHROUGH;
 
-        case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
+        case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
                 ZSTD_FALLTHROUGH;
 
-        case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24;
+        case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
                 ZSTD_FALLTHROUGH;
 
-        case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16;
+        case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
                 ZSTD_FALLTHROUGH;
 
-        case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) <<  8;
+        case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
                 ZSTD_FALLTHROUGH;
 
         default: break;
@@ -299,12 +297,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
     return srcSize;
 }
 
-FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
 {
     return bitContainer >> start;
 }
 
-FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
     U32 const regMask = sizeof(bitContainer)*8 - 1;
     /* if start > regMask, bitstream is corrupted, and result is undefined */
@@ -314,7 +312,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitCon
      * such cpus old (pre-Haswell, 2013) and their performance is not of that
      * importance.
      */
-#if defined(__x86_64__) || defined(_M_X64)
+#if defined(__x86_64__) || defined(_M_X86)
     return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
 #else
     return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
@@ -327,7 +325,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitCon
  *  On 32-bits, maxNbBits==24.
  *  On 64-bits, maxNbBits==56.
  * @return : value extracted */
-FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t*  bitD, U32 nbBits)
+MEM_STATIC  FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t*  bitD, U32 nbBits)
 {
     /* arbitrate between double-shift and shift+mask */
 #if 1
@@ -343,14 +341,14 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t*  bitD,
 
 /*! BIT_lookBitsFast() :
  *  unsafe version; only works if nbBits >= 1 */
-MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
 {
     U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
     assert(nbBits >= 1);
     return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
 }
 
-FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
 {
     bitD->bitsConsumed += nbBits;
 }
@@ -359,38 +357,23 @@ FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
  * @return : extracted value. */
-FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
 {
-    BitContainerType const value = BIT_lookBits(bitD, nbBits);
+    size_t const value = BIT_lookBits(bitD, nbBits);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
 /*! BIT_readBitsFast() :
  *  unsafe version; only works if nbBits >= 1 */
-MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
-    BitContainerType const value = BIT_lookBitsFast(bitD, nbBits);
+    size_t const value = BIT_lookBitsFast(bitD, nbBits);
     assert(nbBits >= 1);
     BIT_skipBits(bitD, nbBits);
     return value;
 }
 
-/*! BIT_reloadDStream_internal() :
- *  Simple variant of BIT_reloadDStream(), with two conditions:
- *  1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
- *  2. look window is valid after shifted down : bitD->ptr >= bitD->start
- */
-MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD)
-{
-    assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
-    bitD->ptr -= bitD->bitsConsumed >> 3;
-    assert(bitD->ptr >= bitD->start);
-    bitD->bitsConsumed &= 7;
-    bitD->bitContainer = MEM_readLEST(bitD->ptr);
-    return BIT_DStream_unfinished;
-}
-
 /*! BIT_reloadDStreamFast() :
  *  Similar to BIT_reloadDStream(), but with two differences:
  *  1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
@@ -401,35 +384,31 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
 {
     if (UNLIKELY(bitD->ptr < bitD->limitPtr))
         return BIT_DStream_overflow;
-    return BIT_reloadDStream_internal(bitD);
+    assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
+    bitD->ptr -= bitD->bitsConsumed >> 3;
+    bitD->bitsConsumed &= 7;
+    bitD->bitContainer = MEM_readLEST(bitD->ptr);
+    return BIT_DStream_unfinished;
 }
 
 /*! BIT_reloadDStream() :
  *  Refill `bitD` from buffer previously set in BIT_initDStream() .
- *  This function is safe, it guarantees it will not never beyond src buffer.
+ *  This function is safe, it guarantees it will not read beyond src buffer.
  * @return : status of `BIT_DStream_t` internal register.
  *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
-FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
-    /* note : once in overflow mode, a bitstream remains in this mode until it's reset */
-    if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) {
-        static const BitContainerType zeroFilled = 0;
-        bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */
-        /* overflow detected, erroneous scenario or end of stream: no update */
+    if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */
         return BIT_DStream_overflow;
-    }
-
-    assert(bitD->ptr >= bitD->start);
 
     if (bitD->ptr >= bitD->limitPtr) {
-        return BIT_reloadDStream_internal(bitD);
+        return BIT_reloadDStreamFast(bitD);
     }
     if (bitD->ptr == bitD->start) {
-        /* reached end of bitStream => no update */
         if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
         return BIT_DStream_completed;
     }
-    /* start < ptr < limitPtr => cautious update */
+    /* start < ptr < limitPtr */
     {   U32 nbBytes = bitD->bitsConsumed >> 3;
         BIT_DStream_status result = BIT_DStream_unfinished;
         if (bitD->ptr - nbBytes < bitD->start) {
@@ -451,4 +430,8 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
     return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
 }
 
+#if defined (__cplusplus)
+}
+#endif
+
 #endif /* BITSTREAM_H_MODULE */