sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-09 00:11:14 -05:00
Code Issues Projects Releases Wiki Activity

Merge branch 'facebook:dev' into optimal-huff-depth-speed

Browse Source
This commit is contained in:
daniellerozenblit 2022-12-14 10:24:07 -05:00 committed by GitHub
commit 326e442dd9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 87 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"

4
.github/workflows/dev-long-tests.yml vendored
View File

@ -71,7 +71,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: gcc-8 + ASan + UBSan + Test Zstd - name: gcc-8 + ASan + UBSan + Test Zstd
# See https://askubuntu.com/a/1428822
run: | run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update sudo apt-get -qqq update
make gcc8install make gcc8install
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1 CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
@ -102,7 +104,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: gcc-8 + ASan + UBSan + Fuzz Test - name: gcc-8 + ASan + UBSan + Fuzz Test
# See https://askubuntu.com/a/1428822
run: | run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update sudo apt-get -qqq update
make gcc8install make gcc8install
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest

14
.github/workflows/dev-short-tests.yml vendored
View File

@ -52,7 +52,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: gcc-7 + libzstdmt compilation - name: gcc-7 + libzstdmt compilation
# See https://askubuntu.com/a/1428822
run: | run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update sudo apt-get -qqq update
make gcc7install make gcc7install
CC=gcc-7 CFLAGS=-Werror make -j all CC=gcc-7 CFLAGS=-Werror make -j all
@ -158,7 +160,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: gcc-8 build - name: gcc-8 build
# See https://askubuntu.com/a/1428822
run: | run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update sudo apt-get -qqq update
make gcc8install make gcc8install
CC=gcc-8 CFLAGS="-Werror" make -j all CC=gcc-8 CFLAGS="-Werror" make -j all
@ -187,7 +191,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Add MSBuild to PATH - name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2 uses: microsoft/setup-msbuild@v1.1.3
- name: Build - name: Build
working-directory: ${{env.GITHUB_WORKSPACE}} working-directory: ${{env.GITHUB_WORKSPACE}}
run: | run: |
@ -206,7 +210,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Add MSBuild to PATH - name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2 uses: microsoft/setup-msbuild@v1.1.3
- name: Build - name: Build
working-directory: ${{env.GITHUB_WORKSPACE}} working-directory: ${{env.GITHUB_WORKSPACE}}
# See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@ -225,7 +229,7 @@ jobs:
# steps: # steps:
# - uses: actions/checkout@v3 # - uses: actions/checkout@v3
# - name: Add MSBuild to PATH # - name: Add MSBuild to PATH
# uses: microsoft/setup-msbuild@v1.0.2 # uses: microsoft/setup-msbuild@v1.1.3
# - name: Build # - name: Build
# working-directory: ${{env.GITHUB_WORKSPACE}} # working-directory: ${{env.GITHUB_WORKSPACE}}
# run: > # run: >
@ -271,7 +275,7 @@ jobs:
qemu-consistency: qemu-consistency:
name: QEMU ${{ matrix.name }} name: QEMU ${{ matrix.name }}
runs-on: ubuntu-latest runs-on: ubuntu-20.04
strategy: strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed. fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix: matrix:
@ -376,7 +380,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Add MSBuild to PATH - name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2 uses: microsoft/setup-msbuild@v1.1.3
- name: Build and run tests - name: Build and run tests
working-directory: ${{env.GITHUB_WORKSPACE}} working-directory: ${{env.GITHUB_WORKSPACE}}
env: env:

2
.github/workflows/publish-release-artifacts.yml vendored
View File

@ -68,7 +68,7 @@ jobs:
fi fi
- name: Publish - name: Publish
uses: skx/github-action-publish-binaries@release-1.3 uses: skx/github-action-publish-binaries@release-2.0
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with: with:

2
.github/workflows/scorecards.yml vendored
View File

@ -32,7 +32,7 @@ jobs:
persist-credentials: false persist-credentials: false
- name: "Run analysis" - name: "Run analysis"
uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3 uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6
with: with:
results_file: results.sarif results_file: results.sarif
results_format: sarif results_format: sarif

14
build/meson/meson.build
View File

@ -23,7 +23,9 @@ project('zstd',
# so this isn't safe # so this isn't safe
#'werror=true' #'werror=true'
], ],
version: 'DUMMY', version: run_command(
find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h',
check: true).stdout().strip(),
meson_version: '>=0.48.0') meson_version: '>=0.48.0')
cc = meson.get_compiler('c') cc = meson.get_compiler('c')
@ -45,16 +47,6 @@ compiler_msvc = 'msvc'
zstd_version = meson.project_version() zstd_version = meson.project_version()
zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true)
r = run_command(GetZstdLibraryVersion_py, zstd_h_file)
if r.returncode() == 0
zstd_version = r.stdout().strip()
message('Project version is now: @0@'.format(zstd_version))
else
error('Cannot find project version in @0@'.format(zstd_h_file))
endif
zstd_libversion = zstd_version zstd_libversion = zstd_version
# ============================================================================= # =============================================================================

6
contrib/linux-kernel/test/test.c
View File

@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) {
asm volatile("" : "+r"(x)); asm volatile("" : "+r"(x));
} }
static void __attribute__((noinline)) fill_stack(void) {
memset(g_stack, 0x33, 8192);
}
static void __attribute__((noinline)) set_stack(void) { static void __attribute__((noinline)) set_stack(void) {
char stack[8192]; char stack[8192];
g_stack = stack; g_stack = stack;
memset(g_stack, 0x33, 8192);
use(g_stack); use(g_stack);
} }
@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) {
static void test_stack_usage(test_data_t const *data) { static void test_stack_usage(test_data_t const *data) {
set_stack(); set_stack();
fill_stack();
test_f2fs(); test_f2fs();
test_btrfs(data); test_btrfs(data);
test_decompress_unzstd(data); test_decompress_unzstd(data);

15
lib/compress/zstd_compress.c
View File

@ -265,9 +265,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
} }
/* Returns 1 if compression parameters are such that we should /* Returns ZSTD_ps_enable if compression parameters are such that we should
* enable long distance matching (wlog >= 27, strategy >= btopt). * enable long distance matching (wlog >= 27, strategy >= btopt).
* Returns 0 otherwise. * Returns ZSTD_ps_disable otherwise.
*/ */
static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
const ZSTD_compressionParameters* const cParams) { const ZSTD_compressionParameters* const cParams) {
@ -482,8 +482,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
return bounds; return bounds;
case ZSTD_c_enableLongDistanceMatching: case ZSTD_c_enableLongDistanceMatching:
bounds.lowerBound = 0; bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = 1; bounds.upperBound = (int)ZSTD_ps_disable;
return bounds; return bounds;
case ZSTD_c_ldmHashLog: case ZSTD_c_ldmHashLog:
@ -854,6 +854,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
return (size_t)CCtxParams->enableDedicatedDictSearch; return (size_t)CCtxParams->enableDedicatedDictSearch;
case ZSTD_c_enableLongDistanceMatching : case ZSTD_c_enableLongDistanceMatching :
BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
return CCtxParams->ldmParams.enableLdm; return CCtxParams->ldmParams.enableLdm;
@ -1096,7 +1097,7 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't set pledgedSrcSize when not in init stage."); "Can't set pledgedSrcSize when not in init stage.");
cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
@ -1369,8 +1370,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
} }
/* resize windowLog if input is small enough, to use less memory */ /* resize windowLog if input is small enough, to use less memory */
if ( (srcSize < maxWindowResize) if ( (srcSize <= maxWindowResize)
&& (dictSize < maxWindowResize) ) { && (dictSize <= maxWindowResize) ) {
U32 const tSize = (U32)(srcSize + dictSize); U32 const tSize = (U32)(srcSize + dictSize);
static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :

3
lib/compress/zstd_lazy.c
View File

@ -1078,7 +1078,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
} }
# endif /* ZSTD_ARCH_ARM_NEON */ # endif /* ZSTD_ARCH_ARM_NEON */
/* SWAR */ /* SWAR */
{ const size_t chunkSize = sizeof(size_t); { const int chunkSize = sizeof(size_t);
const size_t shiftAmount = ((chunkSize * 8) - chunkSize); const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
const size_t xFF = ~((size_t)0); const size_t xFF = ~((size_t)0);
const size_t x01 = xFF / 0xFF; const size_t x01 = xFF / 0xFF;
@ -2123,7 +2123,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
size_t ZSTD_compressBlock_lazy2_extDict_row( size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize) void const* src, size_t srcSize)
{ {
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
} }

14
programs/fileio.c
View File

@ -2749,6 +2749,7 @@ typedef struct {
int numSkippableFrames; int numSkippableFrames;
int decompUnavailable; int decompUnavailable;
int usesCheck; int usesCheck;
BYTE checksum[4];
U32 nbFiles; U32 nbFiles;
unsigned dictID; unsigned dictID;
} fileInfo_t; } fileInfo_t;
@ -2843,8 +2844,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) { if (contentChecksumFlag) {
info->usesCheck = 1; info->usesCheck = 1;
ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
info_frame_error, "Error: could not skip past checksum"); info_frame_error, "Error: could not read checksum");
} } } }
info->numActualFrames++; info->numActualFrames++;
} }
@ -2936,7 +2937,16 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
(unsigned long long)info->decompressedSize); (unsigned long long)info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio); DISPLAYOUT("Ratio: %.4f\n", ratio);
} }
if (info->usesCheck && info->numActualFrames == 1) {
DISPLAYOUT("Check: %s %02x%02x%02x%02x\n", checkString,
info->checksum[3], info->checksum[2],
info->checksum[1], info->checksum[0]
);
} else {
DISPLAYOUT("Check: %s\n", checkString); DISPLAYOUT("Check: %s\n", checkString);
}
DISPLAYOUT("\n"); DISPLAYOUT("\n");
} }
} }

7
programs/util.c
View File

@ -569,7 +569,7 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
size_t const curLen = strlen(table2->fileNames[idx2]); size_t const curLen = strlen(table2->fileNames[idx2]);
memcpy(buf+pos, table2->fileNames[idx2], curLen); memcpy(buf+pos, table2->fileNames[idx2], curLen);
assert(newTableIdx <= newTable->tableSize); assert(newTableIdx < newTable->tableSize);
newTable->fileNames[newTableIdx] = buf+pos; newTable->fileNames[newTableIdx] = buf+pos;
pos += curLen+1; pos += curLen+1;
} } } }
@ -693,8 +693,11 @@ static int UTIL_prepareFileList(const char *dirName,
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
assert(newListSize >= 0); assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
if (*bufStart != NULL) {
*bufEnd = *bufStart + newListSize; *bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; } } else {
free(path); closedir(dir); return 0;
}
} }
if (*bufStart + *pos + pathLength < *bufEnd) { if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */

6
programs/zstdcli.c
View File

@ -216,7 +216,6 @@ static void usage_advanced(const char* programName)
DISPLAYOUT("\n"); DISPLAYOUT("\n");
DISPLAYOUT("Advanced compression options :\n"); DISPLAYOUT("Advanced compression options :\n");
DISPLAYOUT(" --ultra enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel()); DISPLAYOUT(" --ultra enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --fast[=#] switch to very fast compression levels (default: %u)\n", 1); DISPLAYOUT(" --fast[=#] switch to very fast compression levels (default: %u)\n", 1);
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */ if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
@ -224,9 +223,9 @@ static void usage_advanced(const char* programName)
DISPLAYOUT(" --no-name do not store original filename when compressing\n"); DISPLAYOUT(" --no-name do not store original filename when compressing\n");
} }
#endif #endif
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n"); DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n"); DISPLAYOUT(" --patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n");
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n");
# ifdef ZSTD_MULTITHREAD # ifdef ZSTD_MULTITHREAD
DISPLAYOUT(" -T# spawn # compression threads (default: 1, 0==# cores) \n"); DISPLAYOUT(" -T# spawn # compression threads (default: 1, 0==# cores) \n");
DISPLAYOUT(" -B# select size of each job (default: 0==automatic) \n"); DISPLAYOUT(" -B# select size of each job (default: 0==automatic) \n");
@ -240,6 +239,7 @@ static void usage_advanced(const char* programName)
DISPLAYOUT(" --target-compressed-block-size=# : generate compressed block of approximately targeted size \n"); DISPLAYOUT(" --target-compressed-block-size=# : generate compressed block of approximately targeted size \n");
DISPLAYOUT(" --no-dictID don't write dictID into header (dictionary compression only)\n"); DISPLAYOUT(" --no-dictID don't write dictID into header (dictionary compression only)\n");
DISPLAYOUT(" --[no-]compress-literals : force (un)compressed literals\n"); DISPLAYOUT(" --[no-]compress-literals : force (un)compressed literals\n");
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --format=zstd compress files to the .zst format (default)\n"); DISPLAYOUT(" --format=zstd compress files to the .zst format (default)\n");
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS

9
tests/cli-tests/compression/window-resize.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
datagen -g1G > file
zstd --long=31 -1 --single-thread --no-content-size -f file
zstd -l -v file.zst
# We want to ignore stderr (its outputting "*** zstd command line interface
# 64-bits v1.5.3, by Yann Collet ***")
rm file file.zst

0
tests/cli-tests/compression/window-resize.sh.stderr.ignore Normal file
View File

3
tests/cli-tests/compression/window-resize.sh.stdout.glob Normal file
View File

@ -0,0 +1,3 @@
...
Window Size: 1.000 GiB (1073741824 B)
...

2
tests/fuzz/zstd_helpers.c
View File

@ -80,7 +80,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer);
setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer);
/* Select long distance matching parameters */ /* Select long distance matching parameters */
setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer); setRand(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_auto, ZSTD_ps_disable, producer);
setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer);
setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
ZSTD_LDM_MINMATCH_MAX, producer); ZSTD_LDM_MINMATCH_MAX, producer);

22
tests/fuzzer.c
View File

@ -851,7 +851,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(dict, size, 0.5, 0.5, seed); RDG_genBuffer(dict, size, 0.5, 0.5, seed);
RDG_genBuffer(src, size, 0.5, 0.5, seed); RDG_genBuffer(src, size, 0.5, 0.5, seed);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
@ -875,7 +875,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize); CHECK_Z(cSize);
@ -900,7 +900,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
memcpy(dict + testSize, CNBuffer, testSize); memcpy(dict + testSize, CNBuffer, testSize);
for (level = 1; level <= 5; ++level) { for (level = 1; level <= 5; ++level) {
for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) { for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
size_t cSize0; size_t cSize0;
XXH64_hash_t compressedChecksum0; XXH64_hash_t compressedChecksum0;
@ -956,7 +956,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Enable MT, LDM, and opt parser */ /* Enable MT, LDM, and opt parser */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
@ -995,7 +995,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Disable content size to skip single-pass decompression. */ /* Disable content size to skip single-pass decompression. */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
@ -1092,7 +1092,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* compress on level 1 using refPrefix and ldm */ /* compress on level 1 using refPrefix and ldm */
ZSTD_CCtx_refPrefix(cctx, dict, size);; ZSTD_CCtx_refPrefix(cctx, dict, size);;
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)) CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
assert(!ZSTD_isError(refPrefixLdmCompressedSize)); assert(!ZSTD_isError(refPrefixLdmCompressedSize));
@ -2820,7 +2820,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
assert(cctx != NULL); assert(cctx != NULL);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
{ size_t const compressedSize = ZSTD_compress2(cctx, { size_t const compressedSize = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize), compressedBuffer, ZSTD_compressBound(inputSize),
@ -2836,7 +2836,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx(); { ZSTD_CCtx* cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
{ size_t const result = ZSTD_compress2(cctx, { size_t const result = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize), compressedBuffer, ZSTD_compressBound(inputSize),
@ -3492,7 +3492,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Enable MT, LDM, and use refPrefix() for a small dict */ /* Enable MT, LDM, and use refPrefix() for a small dict */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
@ -3686,7 +3686,7 @@ static int longUnitTests(U32 const seed, double compressibility)
/* Enable checksum to verify round trip. */ /* Enable checksum to verify round trip. */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
/* Round trip once with ldm. */ /* Round trip once with ldm. */
@ -3696,7 +3696,7 @@ static int longUnitTests(U32 const seed, double compressibility)
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
/* Round trip once without ldm. */ /* Round trip once without ldm. */

4
tests/regression/config.c
View File

@ -183,7 +183,7 @@ static config_t no_pledged_src_size_with_dict = {
}; };
static param_value_t const ldm_param_values[] = { static param_value_t const ldm_param_values[] = {
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
}; };
static config_t ldm = { static config_t ldm = {
@ -204,7 +204,7 @@ static config_t mt = {
static param_value_t const mt_ldm_param_values[] = { static param_value_t const mt_ldm_param_values[] = {
{.param = ZSTD_c_nbWorkers, .value = 2}, {.param = ZSTD_c_nbWorkers, .value = 2},
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
}; };
static config_t mt_ldm = { static config_t mt_ldm = {

2
tests/zstreamtest.c
View File

@ -2281,7 +2281,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
/* mess with long distance matching parameters */ /* mess with long distance matching parameters */
if (bigTests) { if (bigTests) {
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );