CI failure fixes

2025-10-04 00:02:33 -04:00 · 2022-10-11 13:12:19 -07:00 · 2022-10-11 13:12:19 -07:00 · 8888a2ddcc
commit 8888a2ddcc
parent a70ca2bd7d
48 changed files with 944 additions and 235 deletions
--- a/.github/workflows/dev-long-tests.yml
+++ b/.github/workflows/dev-long-tests.yml
@ -13,7 +13,7 @@ jobs:
  make-all:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: make all
      run: make all

@ -24,7 +24,7 @@ jobs:
      DEVNULLRIGHTS: 1
      READFROMBLOCKDEVICE: 1
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: make test
      run: make test

@ -32,29 +32,36 @@ jobs:
  make-test-osx:
    runs-on: macos-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: OS-X test
      run: make test # make -c lib all doesn't work because of the fact that it's not a tty

  no-intrinsics-fuzztest:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: no intrinsics fuzztest
      run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest

  tsan-zstreamtest:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: thread sanitizer zstreamtest
      run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream

+  ubsan-zstreamtest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: undefined behavior sanitizer zstreamtest
+      run: CC=clang make uasan-test-zstream
+
  # lasts ~15mn
  tsan-fuzztest:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: thread sanitizer fuzztest
      run: CC=clang make tsan-fuzztest

@ -62,17 +69,24 @@ jobs:
  gcc-8-asan-ubsan-testzstd:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: gcc-8 + ASan + UBSan + Test Zstd
      run: |
        sudo apt-get -qqq update
        make gcc8install
        CC=gcc-8 make -j uasan-test-zstd </dev/null V=1

+  clang-asan-ubsan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: clang + ASan + UBSan + Test Zstd
+      run: CC=clang make -j uasan-test-zstd </dev/null V=1
+
  gcc-asan-ubsan-testzstd-32bit:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: ASan + UBSan + Test Zstd, 32bit mode
      run: |
        sudo apt-get -qqq update
@ -86,41 +100,65 @@ jobs:
  gcc-8-asan-ubsan-fuzz:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: gcc-8 + ASan + UBSan + Fuzz Test
      run: |
        sudo apt-get -qqq update
        make gcc8install
        CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest

+  clang-asan-ubsan-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: clang + ASan + UBSan + Fuzz Test
+      run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+
  gcc-asan-ubsan-fuzz32:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: ASan + UBSan + Fuzz Test 32bit
      run: |
        sudo apt-get -qqq update
        make libc6install
        CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest

+  clang-asan-ubsan-fuzz32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: clang + ASan + UBSan + Fuzz Test 32bit
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+
  asan-ubsan-regression:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: ASan + UBSan + Regression Test
      run: make -j uasanregressiontest

+  clang-ubsan-regression:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: clang + ASan + UBSan + Regression Test
+      run: CC=clang make -j uasanregressiontest
+
  msan-regression:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: MSan + Regression Test
      run: make -j msanregressiontest

  clang-msan-fuzz:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: clang + MSan + Fuzz Test
      run: |
        sudo apt-get -qqq update
@ -131,7 +169,7 @@ jobs:
  clang-msan-testzstd:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: clang + MSan + Test Zstd
      run: |
        sudo apt-get update
@ -141,7 +179,7 @@ jobs:
  armfuzz:
      runs-on: ubuntu-latest
      steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
      - name: Qemu ARM emulation + Fuzz Test
        run: |
          sudo apt-get -qqq update
@ -151,13 +189,13 @@ jobs:
  valgrind-fuzz-test:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: valgrind + fuzz test stack mode    # ~ 7mn
      shell: 'script -q -e -c "bash {0}"'
      run: |
        sudo apt-get -qqq update
        make valgrindinstall
-        make -C tests valgrindTest
+        make -C tests test-valgrind
        make clean
        make -C tests test-fuzzer-stackmode

@ -171,7 +209,7 @@ jobs:
          { compiler: gcc, platform: x64, action: test, script: ""},
        ]
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Mingw long test
      run: |
        $env:PATH_ORIGINAL = $env:PATH
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@ -14,21 +14,21 @@ jobs:
  linux-kernel:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: linux kernel, library + build + test
      run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"

  benchmarking:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: make benchmarking
      run: make benchmarking

  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: make check on 32-bit
      run: |
        sudo apt update
@ -38,7 +38,7 @@ jobs:
  check-x32:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
      env:
        CHECK_CONSTRAINED_MEM: true
@ -50,7 +50,7 @@ jobs:
  gcc-7-libzstd:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: gcc-7 + libzstdmt compilation
      run: |
        sudo apt-get -qqq update
@ -65,7 +65,7 @@ jobs:
  cmake-build-and-test-check:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: cmake build and test check
      run: |
        FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
@ -76,7 +76,7 @@ jobs:
  cpp-gnu90-c99-compatibility:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: C++, gnu90 and c99 compatibility
      run: |
        make cxxtest
@ -90,7 +90,7 @@ jobs:
  mingw-cross-compilation:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: mingw cross-compilation
      run: |
        # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@ -101,7 +101,7 @@ jobs:
  armbuild:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: ARM Build Test
      run: |
        sudo apt-get -qqq update
@ -121,18 +121,18 @@ jobs:
  zlib-wrapper:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: zlib wrapper test
      run: |
        sudo apt-get -qqq update
        make valgrindinstall
        make -C zlibWrapper test
-        make -C zlibWrapper valgrindTest
+        make -C zlibWrapper test-valgrind

  lz4-threadpool-libs:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: LZ4, thread pool, and libs build testslib wrapper test
      run: |
        make lz4install
@ -146,7 +146,7 @@ jobs:
  gcc-make-tests-32bit:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Make all, 32bit mode
      run: |
        sudo apt-get -qqq update
@ -156,7 +156,7 @@ jobs:
  gcc-8-make:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
      - name: gcc-8 build
        run: |
          sudo apt-get -qqq update
@ -166,7 +166,7 @@ jobs:
  implicit-fall-through:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
      - name: -Wimplicit-fallthrough build
        run: |
          make clean
@ -185,7 +185,7 @@ jobs:
            flags: "-A Win32"
          - generator: "MinGW Makefiles"
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Add MSBuild to PATH
      uses: microsoft/setup-msbuild@v1.0.2
    - name: Build
@ -204,7 +204,7 @@ jobs:
        platform: [x64, Win32]
        configuration: [Debug, Release]
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Add MSBuild to PATH
      uses: microsoft/setup-msbuild@v1.0.2
    - name: Build
@ -223,7 +223,7 @@ jobs:
 #        platform: [x64, Win32]
 #        configuration: [Debug, Release]
 #    steps:
-#    - uses: actions/checkout@v2
+#    - uses: actions/checkout@v3
 #    - name: Add MSBuild to PATH
 #      uses: microsoft/setup-msbuild@v1.0.2
 #    - name: Build
@ -235,7 +235,7 @@ jobs:
  minimal-decompressor-macros:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: minimal decompressor macros
      run: |
        make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@ -250,7 +250,7 @@ jobs:
  dynamic-bmi2:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: dynamic bmi2 tests
      run: |
        make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@ -262,7 +262,7 @@ jobs:
  test-variants:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: make all variants & validate
      run: |
        make -j -C programs allVariants MOREFLAGS=-O0
@ -288,7 +288,7 @@ jobs:
      XCC: ${{ matrix.xcc }}
      XEMU: ${{ matrix.xemu }}
    steps:
-    - uses: actions/checkout@v2 # https://github.com/actions/checkout
+    - uses: actions/checkout@v3
    - name: apt update & install
      run: |
        sudo apt-get update
@ -341,7 +341,7 @@ jobs:
          { compiler: clang, platform: x64, script: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd V=1"},
        ]
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Mingw short test
      run: |
        ECHO "Building ${{matrix.compiler}} ${{matrix.platform}}"
@ -374,7 +374,7 @@ jobs:
        platform: [x64, Win32]
        configuration: [Release]
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Add MSBuild to PATH
      uses: microsoft/setup-msbuild@v1.0.2
    - name: Build and run tests
@ -394,7 +394,7 @@ jobs:
  intel-cet-compatibility:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
    - name: Build Zstd
      run: |
        make -j zstd V=1
@ -410,6 +410,22 @@ jobs:
      run: |
        sde-external-9.0.0-2021-11-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3

+  pkg-config:
+    runs-on: ubuntu-latest
+    container:
+      image: debian:testing
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          apt -y update
+          apt -y install --no-install-recommends gcc libc6-dev make pkg-config
+      - name: Build and install
+        run: make -C lib install
+      - name: Test pkg-config
+        run: |
+          cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd)
+          ./simple LICENSE


 # This test currently fails on Github Actions specifically.
@ -419,7 +435,7 @@ jobs:
 #  versions-compatibility:
 #    runs-on: ubuntu-latest
 #    steps:
-#    - uses: actions/checkout@v2
+#    - uses: actions/checkout@v3
 #    - name: Versions Compatibility Test
 #      run: |
 #        make -C tests versionsTest
@ -443,7 +459,7 @@ jobs:
 #        sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
 #        sudo apt-get update
 #        sudo apt-get install -y intel-basekit intel-hpckit
-#    - uses: actions/checkout@v2
+#    - uses: actions/checkout@v3
 #    - name: make check
 #      run: |
 #        make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
--- a/.github/workflows/publish-release-artifacts.yml
+++ b/.github/workflows/publish-release-artifacts.yml
@ -5,14 +5,21 @@ on:
    types:
      - published

+permissions:
+  contents: read
+
 jobs:
  publish-release-artifacts:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+      actions: write # to attach binaries to release artifacts (skx/github-action-publish-binaries)
+
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/')

    steps:
      - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3

      - name: Archive
        env:
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@ -0,0 +1,64 @@
+name: Scorecards supply-chain security
+on:
+  # Only the default branch is supported.
+  branch_protection_rule:
+  schedule:
+    - cron: '22 21 * * 2'
+  push:
+    # TODO: Add release branch when supported?
+    branches: [ "dev" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecards analysis
+    if: github.repository == 'facebook/zstd'
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Used to receive a badge.
+      id-token: write
+      # Needs for private repositories.
+      contents: read
+      actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # tag=v3.0.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) Read-only PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecards on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_READ_TOKEN }}
+
+          # Publish the results for public repositories to enable scorecard badges. For more details, see
+          # https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories, `publish_results` will automatically be set to `false`, regardless
+          # of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # tag=v3.0.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # tag=v1.0.26
+        with:
+          sarif_file: results.sarif
--- a/2
+++ b/2
@ -157,7 +157,7 @@ MKDIR ?= mkdir -p

 HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 EGREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
+ifeq ($(HAVE_COLORNEVER), 1)
 EGREP_OPTIONS += --color=never
 endif
 EGREP = egrep $(EGREP_OPTIONS)
--- a/TESTING.md
+++ b/TESTING.md
@ -22,7 +22,7 @@ They consist of the following tests:
  - `tests/playTests.sh --test-large-data`
  - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
 - `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
+- Valgrind Test (`make -C tests test-valgrind`) (testing CLI and fuzzer under `valgrind`)
 - Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64

 Long Tests
--- a/build/cmake/CMakeModules/JoinPaths.cmake
+++ b/build/cmake/CMakeModules/JoinPaths.cmake
@ -0,0 +1,23 @@
+# This module provides function for joining paths
+# known from most languages
+#
+# SPDX-License-Identifier: (MIT OR CC0-1.0)
+# Copyright 2020 Jan Tojnar
+# https://github.com/jtojnar/cmake-snips
+#
+# Modelled after Python’s os.path.join
+# https://docs.python.org/3.7/library/os.path.html#os.path.join
+# Windows not supported
+function(join_paths joined_path first_path_segment)
+    set(temp_path "${first_path_segment}")
+    foreach(current_segment IN LISTS ARGN)
+        if(NOT ("${current_segment}" STREQUAL ""))
+            if(IS_ABSOLUTE "${current_segment}")
+                set(temp_path "${current_segment}")
+            else()
+                set(temp_path "${temp_path}/${current_segment}")
+            endif()
+        endif()
+    endforeach()
+    set(${joined_path} "${temp_path}" PARENT_SCOPE)
+endfunction()
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@ -135,30 +135,17 @@ if (ZSTD_BUILD_STATIC)
            OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
 endif ()

-if (UNIX OR MINGW)
-    # pkg-config
-    set(PREFIX "${CMAKE_INSTALL_PREFIX}")
-    set(EXEC_PREFIX "\${prefix}")
-    set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
-    set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
-    set(VERSION "${zstd_VERSION}")
+# pkg-config
+include(JoinPaths) # can be replaced by cmake_path(APPEND) in CMake 3.20
+set(PREFIX "${CMAKE_INSTALL_PREFIX}")
+set(EXEC_PREFIX "\${prefix}")
+join_paths(LIBDIR "\${exec_prefix}" "${CMAKE_INSTALL_LIBDIR}")
+join_paths(INCLUDEDIR "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
+set(LIBS_PRIVATE "${THREADS_LIBS}")
+set(VERSION "${zstd_VERSION}")

-    string(LENGTH "${PREFIX}" PREFIX_LENGTH)
-    string(SUBSTRING "${LIBDIR}" 0 ${PREFIX_LENGTH} LIBDIR_PREFIX)
-    string(SUBSTRING "${LIBDIR}" ${PREFIX_LENGTH} -1 LIBDIR_SUFFIX)
-    string(SUBSTRING "${INCLUDEDIR}" 0 ${PREFIX_LENGTH} INCLUDEDIR_PREFIX)
-    string(SUBSTRING "${INCLUDEDIR}" ${PREFIX_LENGTH} -1 INCLUDEDIR_SUFFIX)
-
-    if ("${INCLUDEDIR_PREFIX}" STREQUAL "${PREFIX}")
-        set(INCLUDEDIR "\${prefix}${INCLUDEDIR_SUFFIX}")
-    endif()
-    if ("${LIBDIR_PREFIX}" STREQUAL "${PREFIX}")
-        set(LIBDIR "\${exec_prefix}${LIBDIR_SUFFIX}")
-    endif()
-
-    configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
-endif ()
+configure_file("${LIBRARY_DIR}/libzstd.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" @ONLY)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")

 # install target
 install(FILES
--- a/build/single_file_libs/combine.sh
+++ b/build/single_file_libs/combine.sh
@ -2,11 +2,7 @@

 # Tool to bundle multiple C/C++ source files, inlining any includes.
 # 
-# Note: this POSIX-compliant script is many times slower than the original bash
-# implementation (due to the grep calls) but it runs and works everywhere.
-# 
 # TODO: ROOTS, FOUND, etc., as arrays (since they fail on paths with spaces)
-# TODO: revert to Bash-only regex (the grep ones being too slow)
 # 
 # Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)

@ -52,13 +48,54 @@ test_deps() {
  fi
 }

-# Tests if list $1 has item $2 (returning zero on a match)
-list_has_item() {
-  if echo "$1" | grep -Eq "(^|\s*)$2(\$|\s*)"; then
+# Test if glob pattern $1 matches subject $2 (see fnmatch(3))
+fnmatch() {
+  case "$2" in
+  $1)
    return 0
-  else
-    return 1
+    ;;
+  esac
+  return 1
+}
+
+# Test if line $1 is local include directive
+is_include_line() {
+  fnmatch "*#*include*" "$1" || return 1
+  printf "%s\n" "$1" | grep -Eq '^\s*#\s*include\s*".+"'
+}
+
+# Test if line $1 is pragma once directive
+is_pragma_once_line() {
+  fnmatch "*#*pragma*once*" "$1" || return 1
+  printf "%s\n" "$1" | grep -Eq '^\s*#\s*pragma\s*once\s*'
+}
+
+# Tests if list $1 has item $2 (returning zero on a match)
+# (originally used grep -Eq "(^|\s*)$2(\$|\s*))
+readonly list_FS="$IFS"
+list_has_item() {
+  # Re: escaping glob pattern special characters in item string:
+  #
+  # bash (tested 3.2.57, 5.1.4), dash (tested 0.5.10.2), NetBSD /bin/sh
+  # (tested 8.2), and Solaris /bin/sh (tested 11.4) require escaping
+  # backslashes in a bracket expression despite POSIX specifying that
+  # backslash loses significance in a bracket expression.
+  #
+  # Conversely, neither FreeBSD /bin/sh (tested 12.2) nor OpenBSD /bin/sh
+  # (tested 7.1) obey backslash-escaping in case statement patterns even
+  # outside bracket expressions, so escape special characters using bracket
+  # expressions.
+  #
+  # Solaris /bin/sh (tested 11.4) requires vertical bar (|) to be escaped.
+  #
+  # All accommodations should behave as expected under strict POSIX semantics.
+  if fnmatch "*[\\*?[|]*" "$2"; then
+    set -- "$1" "$(printf '%s\n' "$2" | sed -e 's/[*?[|]/[&]/g; s/[\]/[\\&]/g')"
  fi
+  for item_P in "*[$list_FS]$2[$list_FS]*" "*[$list_FS]$2" "$2[$list_FS]*" "$2"; do
+    fnmatch "${item_P}" "$1" && return 0
+  done
+  return 1
 }

 # Adds a new line with the supplied arguments to $DESTN (or stdout)
@ -124,7 +161,7 @@ add_file() {
    # Read the file
    local line=
    while IFS= read -r line; do
-      if echo "$line" | grep -Eq '^\s*#\s*include\s*".+"'; then
+      if is_include_line "$line"; then
        # We have an include directive so strip the (first) file
        local inc=$(echo "$line" | grep -Eo '".*"' | sed -E 's/"([^"]+)"/\1/' | head -1)
        local res_inc="$(resolve_include "$srcdir" "$inc")"
@ -155,7 +192,7 @@ add_file() {
        # Skip any 'pragma once' directives, otherwise write the source line
        local write=$PONCE
        if [ $write -eq 0 ]; then
-          if echo "$line" | grep -Eqv '^\s*#\s*pragma\s*once\s*'; then
+          if ! is_pragma_once_line "$line"; then
            write=1
          fi
        fi
--- a/contrib/match_finders/zstd_edist.c
+++ b/contrib/match_finders/zstd_edist.c
@ -26,7 +26,7 @@
 *  Constants
 ***************************************/

-/* Just a sential for the entires of the diagonal matrix */
+/* Just a sential for the entries of the diagonal matrix */
 #define ZSTD_EDIST_DIAG_MAX (S32)(1 << 30)

 /* How large should a snake be to be considered a 'big' snake. 
@ -57,10 +57,10 @@ typedef struct {
    const BYTE* src;
    size_t dictSize;
    size_t srcSize;
-    S32* forwardDiag;            /* Entires of the forward diagonal stored here */ 
-    S32* backwardDiag;           /* Entires of the backward diagonal stored here.
+    S32* forwardDiag;            /* Entries of the forward diagonal stored here */
+    S32* backwardDiag;           /* Entries of the backward diagonal stored here.
                                  *   Note: this buffer and the 'forwardDiag' buffer 
-                                  *   are contiguous. See the ZSTD_eDist_genSequences */ 
+                                  *   are contiguous. See the ZSTD_eDist_genSequences */
    ZSTD_eDist_match* matches;   /* Accumulate matches of length 1 in this buffer. 
                                  *   In a subsequence post-processing step, we combine 
                                  *   contiguous matches. */
@ -460,7 +460,7 @@ static size_t ZSTD_eDist_convertMatchesToSequences(ZSTD_Sequence* sequences,
 }

 /*-*************************************
-*  Interal utils
+*  Internal utils
 ***************************************/

 static size_t ZSTD_eDist_hamingDist(const BYTE* const a,
--- a/contrib/seqBench/Makefile
+++ b/contrib/seqBench/Makefile
@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS  ?= -O3 -g
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: seqBench
+
+all : seqBench
+
+seqBench: util.o timefn.o benchfn.o datagen.o xxhash.o seqBench.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) seqBench
--- a/contrib/seqBench/seqBench.c
+++ b/contrib/seqBench/seqBench.c
@ -0,0 +1,53 @@
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+int main(int argc, char *argv[]) {
+    ZSTD_CCtx* zc = ZSTD_createCCtx();
+
+    if (argc != 2) {
+        printf("Usage: seqBench <file>\n"); // TODO provide the block delim option here
+        return 1;
+    }
+
+    FILE *f = fopen(argv[1], "rb");
+    fseek(f, 0, SEEK_END);
+    long inBufSize = ftell(f);
+    fseek(f, 0, SEEK_SET);
+
+    char *inBuf = malloc(inBufSize + 1);
+    fread(inBuf, inBufSize, 1, f);
+    fclose(f);
+
+    size_t seqsSize = ZSTD_sequenceBound(inBufSize);
+    ZSTD_Sequence *seqs = (ZSTD_Sequence*)malloc(seqsSize * sizeof(ZSTD_Sequence));
+    char *outBuf = malloc(ZSTD_compressBound(inBufSize));
+
+    ZSTD_generateSequences(zc, seqs, seqsSize, inBuf, inBufSize);
+    ZSTD_CCtx_setParameter(zc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
+    size_t outBufSize = ZSTD_compressSequences(zc, outBuf, inBufSize, seqs, seqsSize, inBuf, inBufSize);
+    if (ZSTD_isError(outBufSize)) {
+        printf("ERROR: %lu\n", outBufSize);
+        return 1;
+    }
+
+    char *validationBuf = malloc(inBufSize);
+    ZSTD_decompress(validationBuf, inBufSize, outBuf, outBufSize);
+
+    if (memcmp(inBuf, validationBuf, inBufSize) == 0) {
+        printf("Compression and decompression were successful!\n");
+    } else {
+        printf("ERROR: input and validation buffers don't match!\n");
+        for (int i = 0; i < inBufSize; i++) {
+            if (inBuf[i] != validationBuf[i]) {
+                printf("First bad index: %d\n", i);
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
--- a/examples/common.h
+++ b/examples/common.h
@ -21,6 +21,17 @@
 #include <sys/stat.h>  // stat
 #include <zstd.h>

+
+/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
+#if defined(__GNUC__)
+#  define UNUSED_ATTR __attribute__((unused))
+#else
+#  define UNUSED_ATTR
+#endif
+
+#define HEADER_FUNCTION static UNUSED_ATTR
+
+
 /*
 * Define the returned error code from utility functions.
 */
@ -68,7 +79,7 @@ typedef enum {
 *
 * @return The size of a given file path.
 */
-static size_t fsize_orDie(const char *filename)
+HEADER_FUNCTION size_t fsize_orDie(const char *filename)
 {
    struct stat st;
    if (stat(filename, &st) != 0) {
@ -96,7 +107,7 @@ static size_t fsize_orDie(const char *filename)
 * @return If successful this function will return a FILE pointer to an
 * opened file otherwise it sends an error to stderr and exits.
 */
-static FILE* fopen_orDie(const char *filename, const char *instruction)
+HEADER_FUNCTION FILE* fopen_orDie(const char *filename, const char *instruction)
 {
    FILE* const inFile = fopen(filename, instruction);
    if (inFile) return inFile;
@ -108,7 +119,7 @@ static FILE* fopen_orDie(const char *filename, const char *instruction)
 /*! fclose_orDie() :
 * Close an opened file using given FILE pointer.
 */
-static void fclose_orDie(FILE* file)
+HEADER_FUNCTION void fclose_orDie(FILE* file)
 {
    if (!fclose(file)) { return; };
    /* error */
@ -123,7 +134,7 @@ static void fclose_orDie(FILE* file)
 *
 * @return The number of bytes read.
 */
-static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+HEADER_FUNCTION size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
 {
    size_t const readSize = fread(buffer, 1, sizeToRead, file);
    if (readSize == sizeToRead) return readSize;   /* good */
@ -143,7 +154,7 @@ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
 *
 * @return The number of bytes written.
 */
-static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+HEADER_FUNCTION size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
 {
    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
@ -159,7 +170,7 @@ static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
 * cated memory.  If there is an error, this function will send that
 * error to stderr and exit.
 */
-static void* malloc_orDie(size_t size)
+HEADER_FUNCTION void* malloc_orDie(size_t size)
 {
    void* const buff = malloc(size);
    if (buff) return buff;
@ -177,7 +188,7 @@ static void* malloc_orDie(size_t size)
 * @return If successful this function will load file into buffer and
 * return file size, otherwise it will printout an error to stderr and exit.
 */
-static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
+HEADER_FUNCTION size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
 {
    size_t const fileSize = fsize_orDie(fileName);
    CHECK(fileSize <= bufferSize, "File too large!");
@ -201,7 +212,8 @@ static size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSi
 * @return If successful this function will return buffer and bufferSize(=fileSize),
 * otherwise it will printout an error to stderr and exit.
 */
-static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
+HEADER_FUNCTION void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize)
+{
    size_t const fileSize = fsize_orDie(fileName);
    *bufferSize = fileSize;
    void* const buffer = malloc_orDie(*bufferSize);
@ -217,7 +229,7 @@ static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
 * Note: This function will send an error to stderr and exit if it
 * cannot write to a given file.
 */
-static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+HEADER_FUNCTION void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
 {
    FILE* const oFile = fopen_orDie(fileName, "wb");
    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
--- a/examples/dictionary_compression.c
+++ b/examples/dictionary_compression.c
@ -6,7 +6,17 @@
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
- */
+**/
+
+/* This example deals with Dictionary compression,
+ * its counterpart is `examples/dictionary_decompression.c` .
+ * These examples presume that a dictionary already exists.
+ * The main method to create a dictionary is `zstd --train`,
+ * look at the CLI documentation for details.
+ * Another possible method is to employ dictionary training API,
+ * published in `lib/zdict.h` .
+**/
+
 #include <stdio.h>     // printf
 #include <stdlib.h>    // free
 #include <string.h>    // memset, strcat
@ -14,7 +24,7 @@
 #include "common.h"    // Helper functions, CHECK(), and CHECK_ZSTD()

 /* createDict() :
-   `dictFileName` is supposed to have been created using `zstd --train` */
+** `dictFileName` is supposed already created using `zstd --train` */
 static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 {
    size_t dictSize;
--- a/lib/Makefile
+++ b/lib/Makefile
@ -298,7 +298,7 @@ INSTALL_DATA    ?= $(INSTALL) -m 644

 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
-	@sed $(SED_ERE_OPT) \
+	@sed \
 	        -e 's|@PREFIX@|$(PREFIX)|' \
 	        -e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \
 	        -e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@ -173,6 +173,11 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* ****************************************
 *  HUF detailed API
 * ****************************************/
+#define HUF_OPTIMAL_DEPTH_THRESHOLD 1
+typedef enum {
+   HUF_depth_fast, /** Use heuristic to find the table depth**/
+   HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/
+ } HUF_depth_mode;

 /*! HUF_compress() does the following:
 *  1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
@ -185,7 +190,8 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 *  For example, it's possible to compress several blocks using the same 'CTable',
 *  or to save and regenerate 'CTable' using external methods.
 */
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
+unsigned HUF_minTableLog(size_t srcSize, unsigned maxSymbolValue);
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode);
 size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);   /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
 size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
 size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
@ -199,6 +205,7 @@ typedef enum {
   HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
   HUF_repeat_valid  /**< Can use the previous table and it is assumed to be valid */
 } HUF_repeat;
+
 /** HUF_compress4X_repeat() :
 *  Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
 *  If it uses hufTable it does not modify hufTable or repeat.
@ -209,7 +216,8 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned tableLog,
                       void* workSpace, size_t wkspSize,    /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
+                       unsigned suspectUncompressible, HUF_depth_mode depthMode);

 /** HUF_buildCTable_wksp() :
 *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@ -315,7 +323,8 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned tableLog,
                       void* workSpace, size_t wkspSize,   /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
+                       unsigned suspectUncompressible, HUF_depth_mode depthMode);

 size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 #ifndef HUF_FORCE_DECOMPRESS_X1
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@ -1222,15 +1222,6 @@ static size_t HUF_compressCTable_internal(
    return (size_t)(op-ostart);
 }

-
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
-{
-    unsigned tableLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
-    assert(tableLog <= HUF_TABLELOG_MAX);
-
-    return tableLog;
-}
-
 typedef struct {
    unsigned count[HUF_SYMBOLVALUE_MAX + 1];
    HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)];
@ -1244,6 +1235,51 @@ typedef struct {
 #define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096
 #define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10  /* Must be >= 2 */

+unsigned HUF_minTableLog(size_t srcSize, unsigned maxSymbolValue)
+{
+    U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1;
+    U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2;
+    U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
+    if (minBits < FSE_MIN_TABLELOG) minBits = FSE_MIN_TABLELOG;
+    assert(srcSize > 1); /* Not supported, RLE should be used instead */
+    return minBits;
+}
+
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
+{
+    unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+
+    if (depthMode == HUF_depth_optimal) { /** Test valid depths and return optimal **/
+        BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+        size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
+        size_t optSize = ((size_t) ~0);
+        unsigned huffLog;
+        size_t maxBits, hSize, newSize;
+
+        if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog;
+
+        for (huffLog = HUF_minTableLog(srcSize, maxSymbolValue); huffLog <= maxTableLog; huffLog++) {
+            maxBits = HUF_buildCTable_wksp(table, count,
+                                            maxSymbolValue, huffLog,
+                                            workSpace, wkspSize);
+            if (ERR_isError(maxBits)) continue;
+
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits,
+                                              workSpace, wkspSize);
+            if (ERR_isError(hSize)) continue;
+
+            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+
+            if (newSize < optSize) {
+                optSize = newSize;
+                optLog = huffLog;
+            }
+        }
+    }
+    assert(optLog <= HUF_TABLELOG_MAX);
+    return optLog;
+}
+
 /* HUF_compress_internal() :
 * `workSpace_align4` must be aligned on 4-bytes boundaries,
 * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */
@ -1254,7 +1290,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
                       HUF_nbStreams_e nbStreams,
                       void* workSpace, size_t wkspSize,
                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
-                 const int bmi2, unsigned suspectUncompressible)
+                 const int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
 {
    HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
    BYTE* const ostart = (BYTE*)dst;
@ -1318,7 +1354,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
    }

    /* Build Huffman Tree */
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, depthMode);
    {   size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
                                            maxSymbolValue, huffLog,
                                            &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
@ -1367,7 +1403,7 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
    return HUF_compress_internal(dst, dstSize, src, srcSize,
                                 maxSymbolValue, huffLog, HUF_singleStream,
                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0);
+                                 NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
 }

 size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
@ -1375,13 +1411,13 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                      unsigned maxSymbolValue, unsigned huffLog,
                      void* workSpace, size_t wkspSize,
                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
-                      int bmi2, unsigned suspectUncompressible)
+                      int bmi2, unsigned suspectUncompressible, HUF_depth_mode depthMode)
 {
    DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
    return HUF_compress_internal(dst, dstSize, src, srcSize,
                                 maxSymbolValue, huffLog, HUF_singleStream,
                                 workSpace, wkspSize, hufTable,
-                                 repeat, preferRepeat, bmi2, suspectUncompressible);
+                                 repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
 }

 /* HUF_compress4X_repeat():
@ -1396,7 +1432,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
    return HUF_compress_internal(dst, dstSize, src, srcSize,
                                 maxSymbolValue, huffLog, HUF_fourStreams,
                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0);
+                                 NULL, NULL, 0, 0 /*bmi2*/, 0, HUF_depth_fast);
 }

 /* HUF_compress4X_repeat():
@ -1407,13 +1443,14 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                      const void* src, size_t srcSize,
                      unsigned maxSymbolValue, unsigned huffLog,
                      void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2,
+                      unsigned suspectUncompressible, HUF_depth_mode depthMode)
 {
    DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
    return HUF_compress_internal(dst, dstSize, src, srcSize,
                                 maxSymbolValue, huffLog, HUF_fourStreams,
                                 workSpace, wkspSize,
-                                 hufTable, repeat, preferRepeat, bmi2, suspectUncompressible);
+                                 hufTable, repeat, preferRepeat, bmi2, suspectUncompressible, depthMode);
 }

 #ifndef ZSTD_NO_UNUSED_FUNCTIONS
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@ -2653,6 +2653,8 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
        /* Base suspicion of uncompressibility on ratio of literals to sequences */
        unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
        size_t const litSize = (size_t)(seqStorePtr->lit - literals);
+
+        HUF_depth_mode depthMode = cctxParams->cParams.strategy > HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
        size_t const cSize = ZSTD_compressLiterals(
                                    &prevEntropy->huf, &nextEntropy->huf,
                                    cctxParams->cParams.strategy,
@ -2660,7 +2662,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
                                    op, dstCapacity,
                                    literals, litSize,
                                    entropyWorkspace, entropyWkspSize,
-                                    bmi2, suspectUncompressible);
+                                    bmi2, suspectUncompressible, depthMode);
        FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
        assert(cSize <= dstCapacity);
        op += cSize;
@ -3003,6 +3005,10 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
    zc->seqCollector.seqIndex += seqStoreSeqSize;
 }

+size_t ZSTD_sequenceBound(size_t srcSize) {
+    return (srcSize / ZSTD_MINMATCH_MIN) + 1;
+}
+
 size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
                              size_t outSeqsSize, const void* src, size_t srcSize)
 {
@ -3103,7 +3109,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
                                                  ZSTD_hufCTables_t* nextHuf,
                                                  ZSTD_hufCTablesMetadata_t* hufMetadata,
                                                  const int literalsCompressionIsDisabled,
-                                                  void* workspace, size_t wkspSize)
+                                                  void* workspace, size_t wkspSize, HUF_depth_mode depthMode)
 {
    BYTE* const wkspStart = (BYTE*)workspace;
    BYTE* const wkspEnd = wkspStart + wkspSize;
@ -3160,7 +3166,7 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi

    /* Build Huffman Tree */
    ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, depthMode);
    assert(huffLog <= LitHufLog);
    {   size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
                                                    maxSymbolValue, huffLog,
@ -3264,12 +3270,15 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
                                   void* workspace, size_t wkspSize)
 {
    size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
+    HUF_depth_mode depthMode = cctxParams->cParams.strategy > HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_depth_optimal : HUF_depth_fast;
+
    entropyMetadata->hufMetadata.hufDesSize =
        ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
                                            &prevEntropy->huf, &nextEntropy->huf,
                                            &entropyMetadata->hufMetadata,
                                            ZSTD_literalsCompressionIsDisabled(cctxParams),
-                                            workspace, wkspSize);
+                                            workspace, wkspSize, depthMode);
+
    FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
    entropyMetadata->fseMetadata.fseTablesSize =
        ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
--- a/lib/compress/zstd_compress_literals.c
+++ b/lib/compress/zstd_compress_literals.c
@ -99,7 +99,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
                        const void* src, size_t srcSize,
                              void* entropyWorkspace, size_t entropyWorkspaceSize,
                        const int bmi2,
-                        unsigned suspectUncompressible)
+                        unsigned suspectUncompressible, HUF_depth_mode depthMode)
 {
    size_t const minGain = ZSTD_minGain(srcSize, strategy);
    size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
@ -128,7 +128,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
    RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
    {   HUF_repeat repeat = prevHuf->repeatMode;
        int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
-        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned);
+        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned, HUF_depth_mode);
        huf_compress_f huf_compress;
        if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
        huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
@ -138,7 +138,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
                                entropyWorkspace, entropyWorkspaceSize,
                                (HUF_CElt*)nextHuf->CTable,
                                &repeat, preferRepeat,
-                                bmi2, suspectUncompressible);
+                                bmi2, suspectUncompressible, depthMode);
        if (repeat != HUF_repeat_none) {
            /* reused the existing table */
            DEBUGLOG(5, "Reusing previous huffman table");
--- a/lib/compress/zstd_compress_literals.h
+++ b/lib/compress/zstd_compress_literals.h
@ -26,6 +26,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
                        const void* src, size_t srcSize,
                              void* entropyWorkspace, size_t entropyWorkspaceSize,
                        const int bmi2,
-                        unsigned suspectUncompressible);
+                        unsigned suspectUncompressible, HUF_depth_mode depthMode);

 #endif /* ZSTD_COMPRESS_LITERALS_H */
--- a/lib/compress/zstd_lazy.c
+++ b/lib/compress/zstd_lazy.c
@ -692,7 +692,8 @@ size_t ZSTD_HcFindBestMatch(
        if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
            const BYTE* const match = base + matchIndex;
            assert(matchIndex >= dictLimit);   /* ensures this is true if dictMode != ZSTD_extDict */
-            if (match[ml] == ip[ml])   /* potentially better */
+            /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+            if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3))   /* potentially better */
                currentMl = ZSTD_count(ip, match, iLimit);
        } else {
            const BYTE* const match = dictBase + matchIndex;
@ -1244,7 +1245,8 @@ size_t ZSTD_RowFindBestMatch(
            if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
                const BYTE* const match = base + matchIndex;
                assert(matchIndex >= dictLimit);   /* ensures this is true if dictMode != ZSTD_extDict */
-                if (match[ml] == ip[ml])   /* potentially better */
+                /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+                if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3))   /* potentially better */
                    currentMl = ZSTD_count(ip, match, iLimit);
            } else {
                const BYTE* const match = dictBase + matchIndex;
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@ -2059,7 +2059,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                    if (ZSTD_isError(decompressedSize)) return decompressedSize;
                    DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
                    ip = istart + cSize;
-                    op += decompressedSize;
+                    op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
                    zds->expected = 0;
                    zds->streamStage = zdss_init;
                    someMoreWork = 0;
@ -2177,14 +2177,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                break;
            }
        case zdss_flush:
-            {   size_t const toFlushSize = zds->outEnd - zds->outStart;
+            {
+                size_t const toFlushSize = zds->outEnd - zds->outStart;
                size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
-                op += flushedSize;
+
+                op = op ? op + flushedSize : op;
+
                zds->outStart += flushedSize;
                if (flushedSize == toFlushSize) {  /* flush completed */
                    zds->streamStage = zdss_read;
                    if ( (zds->outBuffSize < zds->fParams.frameContentSize)
-                      && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+                        && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
                        DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
                                (int)(zds->outBuffSize - zds->outStart),
                                (U32)zds->fParams.blockSizeMax);
--- a/lib/libzstd.mk
+++ b/lib/libzstd.mk
@ -120,11 +120,10 @@ endif

 HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 GREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
+ifeq ($(HAVE_COLORNEVER), 1)
  GREP_OPTIONS += --color=never
 endif
 GREP = grep $(GREP_OPTIONS)
-SED_ERE_OPT ?= -E

 ZSTD_COMMON_FILES := $(sort $(wildcard $(LIBZSTD)/common/*.c))
 ZSTD_COMPRESS_FILES := $(sort $(wildcard $(LIBZSTD)/compress/*.c))
--- a/lib/zstd.h
+++ b/lib/zstd.h
@ -187,8 +187,8 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
 *  "empty", "unknown" and "error" results to the same return value (0),
 *  while ZSTD_getFrameContentSize() gives them separate return values.
 * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
-ZSTDLIB_API
 ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);

 /*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
@ -823,13 +823,31 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer

 /*===== Streaming decompression functions =====*/

-/* This function is redundant with the advanced API and equivalent to:
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
 *
+ * Note : This function is redundant with the advanced API and equivalent to:
 *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
 *     ZSTD_DCtx_refDDict(zds, NULL);
 */
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);

+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ *   on the next call.
+ * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+ *   call ZSTD_decompressStream() again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ *           or an error code, which can be tested using ZSTD_isError(),
+ *           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ */
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);

 ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
@ -1378,6 +1396,15 @@ typedef enum {
  ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
 } ZSTD_sequenceFormat_e;

+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ *  @return : upper-bound for the number of sequences that can be generated
+ *            from a buffer of srcSize bytes
+ *
+ *  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
 /*! ZSTD_generateSequences() :
 * Generate sequences using ZSTD_compress2(), given a source buffer.
 *
@ -1683,8 +1710,8 @@ ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressio
 *  Note : this function is now DEPRECATED.
 *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
 *  This prototype will generate compilation warnings. */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
                                          void* dst, size_t dstCapacity,
                                    const void* src, size_t srcSize,
@ -1695,8 +1722,8 @@ size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
 *  Note : this function is now DEPRECATED.
 *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
 *  This prototype will generate compilation warnings. */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                              void* dst, size_t dstCapacity,
                                        const void* src, size_t srcSize,
@ -2199,8 +2226,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
 *  This instruction is mandatory to decode data without a fully-formed header,
 *  such ZSTD_f_zstd1_magicless for example.
 * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);

 /*! ZSTD_decompressStream_simpleArgs() :
@ -2236,8 +2263,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
 * "0" also disables frame content size field. It may be enabled in the future.
 * This prototype will generate compilation warnings.
 */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
                         int compressionLevel,
                         unsigned long long pledgedSrcSize);
@ -2254,8 +2281,8 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
 * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
 * This prototype will generate compilation warnings.
 */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
                     const void* dict, size_t dictSize,
                           int compressionLevel);
@ -2275,8 +2302,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
 * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
 * This prototype will generate compilation warnings.
 */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                    const void* dict, size_t dictSize,
                          ZSTD_parameters params,
@ -2290,8 +2317,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
 * note : cdict will just be referenced, and must outlive compression session
 * This prototype will generate compilation warnings.
 */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);

 /*! ZSTD_initCStream_usingCDict_advanced() :
@ -2309,8 +2336,8 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
 * value ZSTD_CONTENTSIZE_UNKNOWN.
 * This prototype will generate compilation warnings.
 */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                               const ZSTD_CDict* cdict,
                                     ZSTD_frameParameters fParams,
@ -2334,8 +2361,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
 * @return : 0, or an error code (which can be tested using ZSTD_isError())
 *  This prototype will generate compilation warnings.
 */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);


@ -2448,18 +2475,20 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
-ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTDLIB_STATIC_API
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */

 ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);

 /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
-ZSTDLIB_STATIC_API
 ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 /**
  Buffer-less streaming decompression (synchronous mode)
--- a/programs/Makefile
+++ b/programs/Makefile
@ -129,10 +129,10 @@ SET_CACHE_DIRECTORY = \


 .PHONY: all
-all: zstd
+all: zstd zstd-compress zstd-decompress zstd-small

 .PHONY: allVariants
-allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-frugal zstd-nolegacy zstd-dictBuilder
+allVariants: all zstd-frugal zstd-nolegacy zstd-dictBuilder

 .PHONY: zstd  # must always be run
 zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
@ -180,11 +180,13 @@ zstd : $(BUILD_DIR)/zstd
 endif  # BUILD_DIR


+CLEAN += zstd
 .PHONY: zstd-release
 zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
 zstd-release: DEBUGFLAGS_LD :=
 zstd-release: zstd

+CLEAN += zstd32
 zstd32 : CPPFLAGS += $(THREAD_CPP)
 zstd32 : LDFLAGS  += $(THREAD_LD)
 zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
@ -194,7 +196,8 @@ endif
 zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
 	$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)

-## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed
+## zstd-nolegacy: same scope as zstd, with removed support of legacy formats
+CLEAN += zstd-nolegacy
 zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
 zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
 zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
@ -229,33 +232,42 @@ zstd-dll : zstd
 ## zstd-pgo: zstd executable optimized with PGO.
 .PHONY: zstd-pgo
 zstd-pgo :
-	$(MAKE) clean
-	$(MAKE) zstd MOREFLAGS=-fprofile-generate
+	$(MAKE) clean HASH_DIR=$(HASH_DIR)
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-generate
 	./zstd -b19i1 $(PROFILE_WITH)
 	./zstd -b16i1 $(PROFILE_WITH)
 	./zstd -b9i2 $(PROFILE_WITH)
 	./zstd -b $(PROFILE_WITH)
 	./zstd -b7i2 $(PROFILE_WITH)
 	./zstd -b5 $(PROFILE_WITH)
-	$(RM) zstd *.o
+ifndef BUILD_DIR
+	$(RM) zstd obj/$(HASH_DIR)/zstd obj/$(HASH_DIR)/*.o
+else
+	$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
+endif
 	case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac
-	$(MAKE) zstd MOREFLAGS=-fprofile-use
+	$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS=-fprofile-use

 ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
+CLEAN += zstd-small zstd-frugal
 zstd-small: CFLAGS = -Os -Wl,-s
 zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)

+CLEAN += zstd-decompress
 zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)

+CLEAN += zstd-compress
 zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)

 ## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
+CLEAN += zstd-dictBuilder
 zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)

+CLEAN += zstdmt
 zstdmt: zstd
 	ln -sf zstd zstdmt

@ -273,12 +285,8 @@ endif

 .PHONY: clean
 clean:
-	$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
-        zstd$(EXT) zstd32$(EXT) zstd-dll$(EXT) \
-        zstd-compress$(EXT) zstd-decompress$(EXT) \
-        zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
-        zstd-dictBuilder$(EXT) \
-        *.gcda default*.profraw default.profdata have_zlib$(EXT)
+	$(RM) $(CLEAN) core *.o tmp* result* dictionary *.zst \
+        *.gcda default*.profraw default.profdata have_zlib
 	$(RM) -r obj/*
 	@echo Cleaning completed

@ -339,7 +347,7 @@ ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD D

 HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 EGREP_OPTIONS ?=
-ifeq ($HAVE_COLORNEVER, 1)
+ifeq ($(HAVE_COLORNEVER), 1)
  EGREP_OPTIONS += --color=never
 endif
 EGREP = egrep $(EGREP_OPTIONS)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@ -290,6 +290,7 @@ FIO_prefs_t* FIO_createPreferences(void)
    ret->excludeCompressedFiles = 0;
    ret->allowBlockDevices = 0;
    ret->asyncIO = AIO_supported();
+    ret->passThrough = -1;
    return ret;
 }

@ -338,7 +339,7 @@ void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t comp

 void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }

-void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
+void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; }

 void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }

@ -371,7 +372,7 @@ void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
    prefs->overlapLog = overlapLog;
 }

-void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
+void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) {
    if ((adapt>0) && (prefs->nbWorkers==0))
        EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
    prefs->adaptiveMode = adapt;
@ -453,7 +454,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
    prefs->contentSize = value != 0;
 }

-void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) {
+void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
 #ifdef ZSTD_MULTITHREAD
    prefs->asyncIO = value;
 #else
@ -463,6 +464,10 @@ void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value) {
 #endif
 }

+void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
+    prefs->passThrough = (value != 0);
+}
+
 /* FIO_ctx_t functions */

 void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@ -1306,7 +1311,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
                windowLog = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
            } else {
                const ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, fileSize, 0);
-                windowLog = cParams.windowLog;
+                windowLog = (int)cParams.windowLog;
            }
        }
        windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
@ -1726,14 +1731,15 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
    return result;
 }

-static const char* checked_index(const char* options[], size_t length, size_t index) {
+static const char*
+checked_index(const char* options[], size_t length, size_t index) {
    assert(index < length);
    /* Necessary to avoid warnings since -O3 will omit the above `assert` */
    (void) length;
    return options[index];
 }

-#define INDEX(options, index) checked_index((options), sizeof(options)  / sizeof(char*), (index))
+#define INDEX(options, index) checked_index((options), sizeof(options)  / sizeof(char*), (size_t)(index))

 void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) {
    static const char* formatOptions[5] = {ZSTD_EXTENSION, GZ_EXTENSION, XZ_EXTENSION,
@ -2335,6 +2341,16 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
 {
    unsigned readSomething = 0;
    unsigned long long filesize = 0;
+    int passThrough = prefs->passThrough;
+
+    if (passThrough == -1) {
+        /* If pass-through mode is not explicitly enabled or disabled,
+         * default to the legacy behavior of enabling it if we are writing
+         * to stdout with the overwrite flag enabled.
+         */
+        passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
+    }
+    assert(passThrough == 0 || passThrough == 1);

    /* for each frame */
    for ( ; ; ) {
@ -2352,7 +2368,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
        }
        readSomething = 1;   /* there is at least 1 byte in srcFile */
        if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
-            if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
+            if (passThrough) {
                return FIO_passThrough(&ress);
            }
            DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
@ -2390,7 +2406,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
            DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
            return 1;
 #endif
-        } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) {  /* pass-through mode */
+        } else if (passThrough) {
            return FIO_passThrough(&ress);
        } else {
            DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
--- a/programs/fileio.h
+++ b/programs/fileio.h
@ -12,8 +12,9 @@
 #ifndef FILEIO_H_23981798732
 #define FILEIO_H_23981798732

-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
 #include "fileio_types.h"
+#include "util.h"                  /* FileNamesTable */
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
 #include "../lib/zstd.h"           /* ZSTD_* */

 #if defined (__cplusplus)
@ -70,7 +71,7 @@ void FIO_freeContext(FIO_ctx_t* const fCtx);
 /* FIO_prefs_t functions */
 void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType);
 void FIO_overwriteMode(FIO_prefs_t* const prefs);
-void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
+void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
 void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
 void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
 void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
@ -86,7 +87,7 @@ void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
 void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers);
 void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog);
 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
-void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
+void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
 void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
@ -103,7 +104,8 @@ void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices);
 void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
 void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
 void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
-void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, unsigned value);
+void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
+void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);

 /* FIO_ctx_t functions */
 void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
--- a/programs/fileio_types.h
+++ b/programs/fileio_types.h
@ -34,13 +34,13 @@ typedef struct FIO_prefs_s {

    /* Algorithm preferences */
    FIO_compressionType_t compressionType;
-    U32 sparseFileSupport;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+    int sparseFileSupport;   /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
    int dictIDFlag;
    int checksumFlag;
    int blockSize;
    int overlapLog;
-    U32 adaptiveMode;
-    U32 useRowMatchFinder;
+    int adaptiveMode;
+    int useRowMatchFinder;
    int rsyncable;
    int minAdaptLevel;
    int maxAdaptLevel;
@ -56,9 +56,9 @@ typedef struct FIO_prefs_s {
    ZSTD_paramSwitch_e literalCompressionMode;

    /* IO preferences */
-    U32 removeSrcFile;
-    U32 overwrite;
-    U32 asyncIO;
+    int removeSrcFile;
+    int overwrite;
+    int asyncIO;

    /* Computation resources preferences */
    unsigned memLimit;
@ -68,6 +68,7 @@ typedef struct FIO_prefs_s {
    int patchFromMode;
    int contentSize;
    int allowBlockDevices;
+    int passThrough;
 } FIO_prefs_t;

 #endif /* FILEIO_TYPES_HEADER */
--- a/programs/zstd.1
+++ b/programs/zstd.1
@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "June 2022" "zstd 1.5.3" "User Commands"
+.TH "ZSTD" "1" "August 2022" "zstd 1.5.3" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@ -162,7 +162,7 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
 \fB\-o FILE\fR: save result into \fBFILE\fR
 .
 .IP "\(bu" 4
-\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\.
+\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
 .
 .IP "\(bu" 4
 \fB\-c\fR, \fB\-\-stdout\fR: write to standard output (even if it is the console); keep original files unchanged\.
@ -171,6 +171,9 @@ Additionally, this can be used to limit memory for dictionary training\. This pa
 \fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
 .
 .IP "\(bu" 4
+\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\-f) option is set\.
+.
+.IP "\(bu" 4
 \fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. If used in combination with \-o, will trigger a confirmation prompt (which can be silenced with \-f), as this is a destructive operation\.
 .
 .IP "\(bu" 4
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@ -211,6 +211,8 @@ the last one takes effect.
 * `-f`, `--force`:
    disable input and output checks. Allows overwriting existing files, input
    from console, output to stdout, operating on links, block devices, etc.
+    During decompression and when the output destination is stdout, pass-through
+    unrecognized formats as-is.
 * `-c`, `--stdout`:
    write to standard output (even if it is the console); keep original files unchanged.
 * `--[no-]sparse`:
@ -221,6 +223,12 @@ the last one takes effect.
    default: enabled when output is into a file,
    and disabled when output is stdout.
    This setting overrides default and can force sparse mode over stdout.
+* `--[no-]pass-through`
+    enable / disable passing through uncompressed files as-is. During
+    decompression when pass-through is enabled, unrecognized formats will be
+    copied as-is from the input to the output. By default, pass-through will
+    occur when the output destination is stdout and the force (-f) option is
+    set.
 * `--rm`:
    remove source file(s) after successful compression or decompression. If used in combination with
    -o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@ -154,7 +154,8 @@ static void usage(FILE* f, const char* programName)
 #endif
    DISPLAY_F(f, "  -f, --force          disable input and output checks. Allows overwriting existing files,\n");
    DISPLAY_F(f, "                       input from console, output to stdout, operating on links,\n");
-    DISPLAY_F(f, "                       block devices, etc.\n");
+    DISPLAY_F(f, "                       block devices, etc. During decompression and when the output\n");
+    DISPLAY_F(f, "                       destination is stdout, pass-through unrecognized formats as-is.\n");
    DISPLAY_F(f, "      --rm             remove source file(s) after successful de/compression\n");
    DISPLAY_F(f, "  -k, --keep           preserve source file(s) (default) \n");
 #ifdef ZSTD_GZCOMPRESS
@ -264,6 +265,15 @@ static void usage_advanced(const char* programName)
 # else
    DISPLAYOUT("     --[no-]sparse     sparse mode (default: disabled)\n");
 # endif
+    {
+        char const* passThroughDefault = "disabled";
+        if (exeNameMatch(programName, ZSTD_CAT) ||
+            exeNameMatch(programName, ZSTD_ZCAT) ||
+            exeNameMatch(programName, ZSTD_GZCAT)) {
+            passThroughDefault = "enabled";
+        }
+        DISPLAYOUT("     --[no-]pass-through : passes through uncompressed files as-is (default: %s)\n", passThroughDefault);
+    }
 #endif  /* ZSTD_NODECOMPRESS */

 #ifndef ZSTD_NODICT
@ -776,6 +786,18 @@ static unsigned init_nbThreads(void) {
    const char* __nb;             \
    NEXT_FIELD(__nb);             \
    val32 = readU32FromChar(&__nb); \
+    if(*__nb != 0) {         \
+        errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
+    }                             \
+}
+
+#define NEXT_TSIZE(valTsize) {      \
+    const char* __nb;             \
+    NEXT_FIELD(__nb);             \
+    valTsize = readSizeTFromChar(&__nb); \
+    if(*__nb != 0) {         \
+        errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
+    }                             \
 }

 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
@ -801,7 +823,6 @@ int main(int argCount, const char* argv[])
        hasStdout = 0,
        ldmFlag = 0,
        main_pause = 0,
-        nbWorkers = 0,
        adapt = 0,
        useRowMatchFinder = 0,
        adaptMin = MINCLEVEL,
@ -816,6 +837,7 @@ int main(int argCount, const char* argv[])
        showDefaultCParams = 0,
        ultra=0,
        contentSize=1;
+    unsigned nbWorkers = 0;
    double compressibility = 0.5;
    unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
    size_t blockSize = 0;
@ -870,14 +892,14 @@ int main(int argCount, const char* argv[])
    /* preset behaviors */
    if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
    if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
-    if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }     /* supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }    /* behave like zcat, also supports multiple formats */
    if (exeNameMatch(programName, ZSTD_GZ)) {   /* behave like gzip */
        suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1);
        dictCLevel = cLevel = 6;  /* gzip default is -6 */
    }
    if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); }                                                     /* behave like gunzip, also supports multiple formats */
-    if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
+    if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; }   /* behave like gzcat, also supports multiple formats */
    if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); }    /* behave like lzma */
    if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
    if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); }          /* behave like xz */
@ -926,6 +948,8 @@ int main(int argCount, const char* argv[])
                if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
                if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
                if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
+                if (!strcmp(argument, "--pass-through")) { FIO_setPassThroughFlag(prefs, 1); continue; }
+                if (!strcmp(argument, "--no-pass-through")) { FIO_setPassThroughFlag(prefs, 0); continue; }
                if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
                if (!strcmp(argument, "--asyncio")) { FIO_setAsyncIOFlag(prefs, 1); continue;}
                if (!strcmp(argument, "--no-asyncio")) { FIO_setAsyncIOFlag(prefs, 0); continue;}
@ -1004,13 +1028,13 @@ int main(int argCount, const char* argv[])
                if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
                if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
                if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
-                if (longCommandWArg(&argument, "--block-size=")) { blockSize = readSizeTFromChar(&argument); continue; }
+                if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
                if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
                if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
                if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badusage(programName); CLEAN_RETURN(1); } continue; }
-                if (longCommandWArg(&argument, "--stream-size=")) { streamSrcSize = readSizeTFromChar(&argument); continue; }
-                if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
-                if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
+                if (longCommandWArg(&argument, "--stream-size")) { NEXT_TSIZE(streamSrcSize); continue; }
+                if (longCommandWArg(&argument, "--target-compressed-block-size")) { NEXT_TSIZE(targetCBlockSize); continue; }
+                if (longCommandWArg(&argument, "--size-hint")) { NEXT_TSIZE(srcSizeHint); continue; }
                if (longCommandWArg(&argument, "--output-dir-flat")) {
                    NEXT_FIELD(outDirName);
                    if (strlen(outDirName) == 0) {
@ -1051,6 +1075,8 @@ int main(int argCount, const char* argv[])
                        /* Invalid character following --long */
                        badusage(programName);
                        CLEAN_RETURN(1);
+                    } else {
+                        ldmWindowLog = g_defaultMaxWindowLog;
                    }
                    /* Only set windowLog if not already set by --zstd */
                    if (compressionParams.windowLog == 0)
@ -1200,7 +1226,7 @@ int main(int argCount, const char* argv[])
                    /* nb of threads (hidden option) */
                case 'T':
                    argument++;
-                    nbWorkers = (int)readU32FromChar(&argument);
+                    nbWorkers = readU32FromChar(&argument);
                    break;

                    /* Dictionary Selection level */
@ -1246,10 +1272,10 @@ int main(int argCount, const char* argv[])
    if ((nbWorkers==0) && (!singleThread)) {
        /* automatically set # workers based on # of reported cpus */
        if (defaultLogicalCores) {
-            nbWorkers = UTIL_countLogicalCores();
+            nbWorkers = (unsigned)UTIL_countLogicalCores();
            DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
        } else {
-            nbWorkers = UTIL_countPhysicalCores();
+            nbWorkers = (unsigned)UTIL_countPhysicalCores();
            DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
        }
    }
@ -1313,7 +1339,7 @@ int main(int argCount, const char* argv[])
    if (operation==zom_bench) {
 #ifndef ZSTD_NOBENCH
        benchParams.blockSize = blockSize;
-        benchParams.nbWorkers = nbWorkers;
+        benchParams.nbWorkers = (int)nbWorkers;
        benchParams.realTime = (unsigned)setRealTimePrio;
        benchParams.nbSeconds = bench_nbSeconds;
        benchParams.ldmFlag = ldmFlag;
@ -1474,7 +1500,7 @@ int main(int argCount, const char* argv[])
    if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
        FIO_setContentSize(prefs, contentSize);
-        FIO_setNbWorkers(prefs, nbWorkers);
+        FIO_setNbWorkers(prefs, (int)nbWorkers);
        FIO_setBlockSize(prefs, (int)blockSize);
        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
        FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
@ -1482,7 +1508,7 @@ int main(int argCount, const char* argv[])
        FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
-        FIO_setAdaptiveMode(prefs, (unsigned)adapt);
+        FIO_setAdaptiveMode(prefs, adapt);
        FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
        FIO_setAdaptMin(prefs, adaptMin);
        FIO_setAdaptMax(prefs, adaptMax);
--- a/programs/zstdgrep.1
+++ b/programs/zstdgrep.1
@ -1,17 +1,26 @@
-.TH "ZSTDGREP" "1" "April 2022" "zstd 1.5.2" "User Commands"
+.
+.TH "ZSTDGREP" "1" "August 2022" "zstd 1.5.3" "User Commands"
+.
 .SH "NAME"
 \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
+.
 .SH "SYNOPSIS"
-\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \|\.\|\.\|\.]
+\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.]
+.
 .SH "DESCRIPTION"
 \fBzstdgrep\fR runs \fBgrep (1)\fR on files, or \fBstdin\fR if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
+.
 .P
 The grep\-flags and pattern arguments are passed on to \fBgrep (1)\fR\. If an \fB\-e\fR flag is found in the \fBgrep\-flags\fR, \fBzstdgrep\fR will not look for a pattern argument\.
+.
 .P
 Note that modern \fBgrep\fR alternatives such as \fBripgrep\fR (\fBrg\fR) support \fBzstd\fR\-compressed files out of the box, and can prove better alternatives than \fBzstdgrep\fR notably for unsupported complex pattern searches\. Note though that such alternatives may also feature some minor command line differences\.
+.
 .SH "EXIT STATUS"
 In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\.
+.
 .SH "SEE ALSO"
 \fBzstd (1)\fR
+.
 .SH "AUTHORS"
 Thomas Klausner \fIwiz@NetBSD\.org\fR
--- a/programs/zstdless.1
+++ b/programs/zstdless.1
@ -1,9 +1,14 @@
-.TH "ZSTDLESS" "1" "April 2022" "zstd 1.5.2" "User Commands"
+.
+.TH "ZSTDLESS" "1" "August 2022" "zstd 1.5.3" "User Commands"
+.
 .SH "NAME"
 \fBzstdless\fR \- view zstandard\-compressed files
+.
 .SH "SYNOPSIS"
-\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \|\.\|\.\|\.]
+\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.]
+.
 .SH "DESCRIPTION"
 \fBzstdless\fR runs \fBless (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
+.
 .SH "SEE ALSO"
 \fBzstd (1)\fR
--- a/tests/.gitignore
+++ b/tests/.gitignore
@ -12,6 +12,7 @@ zstreamtest
 zstreamtest32
 zstreamtest_asan
 zstreamtest_tsan
+zstreamtest_ubsan
 zstreamtest-dll
 datagen
 paramgrill
--- a/tests/Makefile
+++ b/tests/Makefile
@ -1,5 +1,5 @@

- # ################################################################
+# ################################################################
 # Copyright (c) Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
@ -37,7 +37,7 @@ TESTARTEFACT := versionsTest
 DEBUGFLAGS += -g -Wno-c++-compat
 CPPFLAGS   += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
              -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-			  -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
+              -DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1

 ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
 ZSTDCOMP_FILES   := $(sort $(ZSTD_COMPRESS_FILES))
@ -134,14 +134,17 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
 zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S
 	$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@

+FULLBENCHS := fullbench fullbench32
+CLEAN += $(FULLBENCHS)
 fullbench32: CPPFLAGS += -m32
-fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
-fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
-fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
-fullbench fullbench32 : $(ZSTD_FILES)
-fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
+$(FULLBENCHS) : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
+$(FULLBENCHS) : LDFLAGS += $(MULTITHREAD_LD)
+$(FULLBENCHS) : DEBUGFLAGS = -DNDEBUG  # turn off assert() for speed measurements
+$(FULLBENCHS) : $(ZSTD_FILES)
+$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
 	$(LINK.c) $^ -o $@$(EXT)

+CLEAN += fullbench-lib
 fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
 fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c
 	$(LINK.c) $^ -o $@$(EXT)
@ -151,6 +154,7 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR
 #	$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
 	$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)

+CLEAN += fuzzer fuzzer32
 fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
 fuzzer : LDFLAGS += $(MULTITHREAD_LD)
 fuzzer : $(ZSTDMT_OBJECTS)
@ -164,6 +168,7 @@ fuzzer32 : $(ZSTD_FILES)
 fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)

+CLEAN += zstreamtest zstreamtest32
 ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
 ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
 ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
@ -175,42 +180,58 @@ zstreamtest32 : $(ZSTREAMFILES)
 zstreamtest zstreamtest32 :
 	$(LINK.c) $^ -o $@$(EXT)

+CLEAN += zstreamtest_asan
 zstreamtest_asan : CFLAGS += -fsanitize=address
 zstreamtest_asan : $(ZSTREAMFILES)
 	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)

+CLEAN += zstreamtest_tsan
 zstreamtest_tsan : CFLAGS += -fsanitize=thread
 zstreamtest_tsan : $(ZSTREAMFILES)
 	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)

+CLEAN += zstreamtest_ubsan
+zstreamtest_ubsan : CFLAGS += -fsanitize=undefined
+zstreamtest_ubsan : $(ZSTREAMFILES)
+	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
 # note : broken : requires symbols unavailable from dynamic library
 zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c  # xxh symbols not exposed from dll
 zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)

+CLEAN += paramgrill
 paramgrill : DEBUGFLAGS =   # turn off debug for speed measurements
 paramgrill : LDLIBS += -lm
 paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c

+CLEAN += datagen
 datagen : $(PRGDIR)/datagen.c datagencli.c
 	$(LINK.c) $^ -o $@$(EXT)

+CLEAN += roundTripCrash
 roundTripCrash: CFLAGS += $(MULTITHREAD)
 roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c

+CLEAN += longmatch
 longmatch : $(ZSTD_OBJECTS) longmatch.c

+CLEAN += bigdict
 bigdict: CFLAGS += $(MULTITHREAD)
 bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c

+CLEAN += invalidDictionaries
 invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c

+CLEAN += legacy
 legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
 legacy : $(ZSTD_FILES) $(sort $(wildcard $(ZSTDDIR)/legacy/*.c)) legacy.c

+CLEAN += decodecorpus
 decodecorpus : LDLIBS += -lm
 decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c

+CLEAN += poolTests
 poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
 	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)

@ -222,7 +243,8 @@ versionsTest: clean
 automated_benchmarking: clean
 	$(PYTHON) automated_benchmarking.py

-# make checkTag
+# make checkTag : check that release tag corresponds to release version
+CLEAN += checkTag
 checkTag.o : $(ZSTDDIR)/zstd.h

 .PHONY: clean
@ -231,28 +253,22 @@ clean:
 	$(MAKE) -C $(PRGDIR) clean
 	$(RM) -fR $(TESTARTEFACT)
 	$(RM) -rf tmp*  # some test directories are named tmp*
-	$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
+	$(RM) $(CLEAN) core *.o *.tmp result* *.gcda dictionary *.zst \
        $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
-        fullbench$(EXT) fullbench32$(EXT) \
-        fullbench-lib$(EXT) fullbench-dll$(EXT) \
-        fuzzer$(EXT) fuzzer32$(EXT) \
-        fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \
-        zstreamtest$(EXT) zstreamtest32$(EXT) \
-        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
-        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
-        decodecorpus$(EXT) checkTag$(EXT) bigdict$(EXT)
+        fullbench-dll$(EXT) fuzzer-dll$(EXT) zstreamtest-dll$(EXT)
 	@echo Cleaning completed


 #----------------------------------------------------------------------------------
-# valgrind tests are validated only for some posix platforms
+# valgrind tests validated only for some posix platforms
 #----------------------------------------------------------------------------------
 UNAME := $(shell uname)
 ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX))
 HOST_OS = POSIX

-valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
-valgrindTest: zstd datagen fuzzer fullbench
+.PHONY: test-valgrind
+test-valgrind: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
+test-valgrind: zstd datagen fuzzer fullbench
 	@echo "\n ---- valgrind tests : memory analyzer ----"
 	$(VALGRIND) ./datagen -g50M > $(VOID)
 	$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
@ -261,7 +277,7 @@ valgrindTest: zstd datagen fuzzer fullbench
 	./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
 	$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
 	./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
-	@rm tmp
+	$(RM) tmp
 	$(VALGRIND) ./fuzzer -T1mn -t1
 	$(VALGRIND) ./fullbench -i1

@ -306,9 +322,9 @@ endif
 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32

 .PHONY: test-all
-test-all: test test32 valgrindTest test-decodecorpus-cli
+test-all: test test32 test-decodecorpus-cli

-.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep
+.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
 test-zstd: ZSTD = $(PRGDIR)/zstd
 test-zstd: zstd

@ -322,29 +338,36 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen
 	file $(ZSTD)
 	EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)

+.PHONY: test-cli-tests
 test-cli-tests: ZSTD = $(PRGDIR)/zstd
 test-cli-tests: zstd datagen
 	file $(ZSTD)
 	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen

-
+.PHONY: test-fullbench
 test-fullbench: fullbench datagen
 	$(QEMU_SYS) ./fullbench -i1
 	$(QEMU_SYS) ./fullbench -i1 -P0

+.PHONY: test-fullbench32
 test-fullbench32: fullbench32 datagen
 	$(QEMU_SYS) ./fullbench32 -i1
 	$(QEMU_SYS) ./fullbench32 -i1 -P0

+.PHONY: test-fuzzer
 test-fuzzer: fuzzer
 	$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)

+# Note : this test presumes `fuzzer` will be built
+.PHONY: test-fuzzer-stackmode
 test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0
 test-fuzzer-stackmode: test-fuzzer

+.PHONY: test-fuzzer32
 test-fuzzer32: fuzzer32
 	$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)

+.PHONY: test-zstream
 test-zstream: zstreamtest
 	$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
 	$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
--- a/tests/cli-tests/basic/help.sh.stdout.glob
+++ b/tests/cli-tests/basic/help.sh.stdout.glob
@ -7,7 +7,8 @@ Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standa
  -d, --decompress     decompression
  -f, --force          disable input and output checks. Allows overwriting existing files,
                       input from console, output to stdout, operating on links,
-                       block devices, etc.
+                       block devices, etc. During decompression and when the output
+                       destination is stdout, pass-through unrecognized formats as-is.
      --rm             remove source file(s) after successful de/compression
  -k, --keep           preserve source file(s) (default)
  -D DICT              use DICT as Dictionary for compression or decompression
--- a/tests/cli-tests/basic/memlimit.sh
+++ b/tests/cli-tests/basic/memlimit.sh
@ -0,0 +1,40 @@
+#!/bin/sh
+
+echo "some data" > file
+
+println "+ zstd --memory=32LB file"
+zstd --memory=32LB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32LiB file"
+zstd --memory=32LiB file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32A file"
+zstd --memory=32A file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32r82347dn83 file"
+zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
+println "+ zstd --memory=32asbdf file"
+zstd --memory=32asbdf file && die "Should not allow bogus suffix"
+println "+ zstd --memory=hello file"
+zstd --memory=hello file && die "Should not allow non-numeric parameter"
+println "+ zstd --memory=1 file"
+zstd --memory=1 file && die "Should allow numeric parameter without suffix"
+rm file.zst
+println "+ zstd --memory=1K file"
+zstd --memory=1K file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KB file"
+zstd --memory=1KB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1KiB file"
+zstd --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1M file"
+zstd --memory=1M file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MB file"
+zstd --memory=1MB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1MiB file"
+zstd --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+
+rm file
+exit 0
--- a/tests/cli-tests/basic/memlimit.sh.stderr.exact
+++ b/tests/cli-tests/basic/memlimit.sh.stderr.exact
@ -0,0 +1,13 @@
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+Should allow numeric parameter without suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
--- a/tests/cli-tests/basic/memlimit.sh.stdout.exact
+++ b/tests/cli-tests/basic/memlimit.sh.stdout.exact
@ -0,0 +1,13 @@
+ zstd --memory=32LB file
+ zstd --memory=32LiB file
+ zstd --memory=32A file
+ zstd --memory=32r82347dn83 file
+ zstd --memory=32asbdf file
+ zstd --memory=hello file
+ zstd --memory=1 file
+ zstd --memory=1K file
+ zstd --memory=1KB file
+ zstd --memory=1KiB file
+ zstd --memory=1M file
+ zstd --memory=1MB file
+ zstd --memory=1MiB file
--- a/tests/cli-tests/compression/verbose-wlog.sh
+++ b/tests/cli-tests/compression/verbose-wlog.sh
@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+zstd < file -vv -19 -o file.19.zst
+zstd -vv -l file.19.zst
+
+zstd < file -vv -19 --long -o file.19.long.zst
+zstd -vv -l file.19.long.zst
--- a/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob
+++ b/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob
@ -0,0 +1,5 @@
+...
+*wlog=23*
+...
+*wlog=27*
+...
--- a/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob
+++ b/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob
@ -0,0 +1,5 @@
+...
+*Window Size: 8388608 B*
+...
+*Window Size: 134217728 B*
+...
--- a/tests/cli-tests/decompression/pass-through.sh
+++ b/tests/cli-tests/decompression/pass-through.sh
@ -0,0 +1,57 @@
+#!/bin/sh
+
+set -e
+
+. "$COMMON/platform.sh"
+
+echo "" > 1
+echo "2" > 2
+echo "23" > 3
+echo "234" > 4
+echo "some data" > file
+
+println "+ passthrough enabled"
+
+zstd file
+
+# Test short files
+zstd -dc --pass-through 1 2 3 4
+
+# Test *cat symlinks
+zstdcat file
+"$ZSTD_SYMLINK_DIR/zcat" file
+"$ZSTD_SYMLINK_DIR/gzcat" file
+
+# Test multiple files with mix of compressed & not
+zstdcat file file.zst
+zstdcat file.zst file
+
+# Test --pass-through
+zstd -dc --pass-through file
+zstd -d --pass-through file -o pass-through-file
+
+# Test legacy implicit passthrough with -fc
+zstd -dcf file
+zstd -dcf file file.zst
+zstd -df < file
+zstd -dcf < file file.zst -
+zstd -dcf < file.zst file -
+
+$DIFF file pass-through-file
+
+println "+ passthrough disabled"
+
+# Test *cat
+zstdcat --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
+"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
+# Test zstd without implicit passthrough
+zstd -d file -o no-pass-through-file && die "should fail"
+zstd -d < file && die "should fail"
+
+# Test legacy implicit passthrough with -fc
+zstd --no-pass-through -dcf file && die "should fail"
+zstd --no-pass-through -dcf file file.zst && die "should fail"
+zstd --no-pass-through -df < file && die "should fail"
+zstd --no-pass-through -dcf < file file.zst - && die "should fail"
+zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:
--- a/tests/cli-tests/decompression/pass-through.sh.stderr.exact
+++ b/tests/cli-tests/decompression/pass-through.sh.stderr.exact
@ -0,0 +1,10 @@
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: file: unsupported format 
+zstd: file: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: /*stdin*\: unsupported format 
+zstd: file: unsupported format 
--- a/tests/cli-tests/decompression/pass-through.sh.stdout.exact
+++ b/tests/cli-tests/decompression/pass-through.sh.stdout.exact
@ -0,0 +1,25 @@
+ passthrough enabled
+
+2
+23
+234
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+some data
+ passthrough disabled
+some data
+some data
+some data
--- a/tests/fuzz/huf_round_trip.c
+++ b/tests/fuzz/huf_round_trip.c
@ -83,7 +83,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
    HUF_DTable* dt = (HUF_DTable*)FUZZ_malloc(HUF_DTABLE_SIZE(tableLog) * sizeof(HUF_DTable));
    dt[0] = tableLog * 0x01000001;

-    tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol);
+    HUF_depth_mode depthMode = rand() & 1 ? HUF_depth_fast : HUF_depth_optimal;
+
+    tableLog = HUF_optimalTableLog(tableLog, size, maxSymbol, wksp, wkspSize, ct, count, depthMode);
    FUZZ_ASSERT(tableLog <= 12);
    tableLog = HUF_buildCTable_wksp(ct, count, maxSymbol, tableLog, wksp, wkspSize);
    FUZZ_ZASSERT(tableLog);
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@ -522,7 +522,7 @@ static int basicUnitTests(U32 seed, double compressibility)
    }
    DISPLAYLEVEL(3, "OK \n");

-    DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
    inBuff.src = NULL;
    inBuff.size = 0;
    inBuff.pos = 0;
@ -548,6 +548,36 @@ static int basicUnitTests(U32 seed, double compressibility)
    {   size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
        if (ret != 0) goto _output_error;
    }
+    DISPLAYLEVEL(3, "OK\n");
+
+    DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
+    {
+        const char* test = "aa";
+        inBuff.src = test;
+        inBuff.size = 2;
+        inBuff.pos = 0;
+        outBuff.dst = NULL;
+        outBuff.size = 0;
+        outBuff.pos = 0;
+        CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
+        CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
+        CHECK_Z( ZSTD_endStream(zc, &outBuff) );
+        outBuff.dst = (char*)(compressedBuffer);
+        outBuff.size = compressedBufferSize;
+        outBuff.pos = 0;
+        {   size_t const r = ZSTD_endStream(zc, &outBuff);
+            CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
+        }
+        inBuff.src = outBuff.dst;
+        inBuff.size = outBuff.pos;
+        inBuff.pos = 0;
+        outBuff.dst = NULL;
+        outBuff.size = 0;
+        outBuff.pos = 0;
+        CHECK_Z( ZSTD_initDStream(zd) );
+        CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
+    }
+
    DISPLAYLEVEL(3, "OK\n");
    /* _srcSize compression test */
    DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
@ -1944,8 +1974,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
        /* multi-segments compression test */
        XXH64_reset(&xxhState, 0);
        {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
-            U32 n;
-            for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
+            cSize=0;
+            totalTestSize=0;
+            while(totalTestSize < maxTestSize) {
                /* compress random chunks into randomly sized dst buffers */
                {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
                    size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@ -66,9 +66,10 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd
 	./zwrapbench -qi1b3B1K $(TEST_FILE)
 	./zwrapbench -rqi1b1e3 ../lib

-#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
-valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
-valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
+.PHONY: test-valgrind
+#test-valgrind: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
+test-valgrind: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
+test-valgrind: clean example fitblk example_zstd fitblk_zstd zwrapbench
 	@echo "\n ---- valgrind tests ----"
 	$(VALGRIND) ./example
 	$(VALGRIND) ./example_zstd