sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-04 00:02:33 -04:00
Code Issues Projects Releases Wiki Activity

huf log speed optimization: unidirectional scan of logs + break when regressing

Browse Source
This commit is contained in:
Danielle Rozenblit 2022-12-20 12:27:38 -08:00
parent c2638212af
commit 482689b995
21 changed files with 96 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"

4
.github/workflows/dev-long-tests.yml vendored
View File

@ -71,7 +71,9 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: gcc-8 + ASan + UBSan + Test Zstd
# See https://askubuntu.com/a/1428822
run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update
make gcc8install
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
@ -102,7 +104,9 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: gcc-8 + ASan + UBSan + Fuzz Test
# See https://askubuntu.com/a/1428822
run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update
make gcc8install
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest

14
.github/workflows/dev-short-tests.yml vendored
View File

@ -52,7 +52,9 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: gcc-7 + libzstdmt compilation
# See https://askubuntu.com/a/1428822
run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update
make gcc7install
CC=gcc-7 CFLAGS=-Werror make -j all
@ -158,7 +160,9 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: gcc-8 build
# See https://askubuntu.com/a/1428822
run: |
echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
sudo apt-get -qqq update
make gcc8install
CC=gcc-8 CFLAGS="-Werror" make -j all
@ -187,7 +191,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2
uses: microsoft/setup-msbuild@v1.1.3
- name: Build
working-directory: ${{env.GITHUB_WORKSPACE}}
run: |
@ -206,7 +210,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2
uses: microsoft/setup-msbuild@v1.1.3
- name: Build
working-directory: ${{env.GITHUB_WORKSPACE}}
# See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@ -225,7 +229,7 @@ jobs:
# steps:
# - uses: actions/checkout@v3
# - name: Add MSBuild to PATH
# uses: microsoft/setup-msbuild@v1.0.2
# uses: microsoft/setup-msbuild@v1.1.3
# - name: Build
# working-directory: ${{env.GITHUB_WORKSPACE}}
# run: >
@ -271,7 +275,7 @@ jobs:
qemu-consistency:
name: QEMU ${{ matrix.name }}
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
@ -376,7 +380,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.0.2
uses: microsoft/setup-msbuild@v1.1.3
- name: Build and run tests
working-directory: ${{env.GITHUB_WORKSPACE}}
env:

2
.github/workflows/publish-release-artifacts.yml vendored
View File

@ -68,7 +68,7 @@ jobs:
fi
- name: Publish
uses: skx/github-action-publish-binaries@release-1.3
uses: skx/github-action-publish-binaries@release-2.0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:

2
.github/workflows/scorecards.yml vendored
View File

@ -32,7 +32,7 @@ jobs:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@865b4092859256271290c77adbd10a43f4779972 # tag=v2.0.3
uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # tag=v2.0.6
with:
results_file: results.sarif
results_format: sarif

14
build/meson/meson.build
View File

@ -23,7 +23,9 @@ project('zstd',
# so this isn't safe
#'werror=true'
],
version: 'DUMMY',
version: run_command(
find_program('GetZstdLibraryVersion.py'), '../../lib/zstd.h',
check: true).stdout().strip(),
meson_version: '>=0.48.0')
cc = meson.get_compiler('c')
@ -45,16 +47,6 @@ compiler_msvc = 'msvc'
zstd_version = meson.project_version()
zstd_h_file = join_paths(meson.current_source_dir(), '../../lib/zstd.h')
GetZstdLibraryVersion_py = find_program('GetZstdLibraryVersion.py', native : true)
r = run_command(GetZstdLibraryVersion_py, zstd_h_file)
if r.returncode() == 0
zstd_version = r.stdout().strip()
message('Project version is now: @0@'.format(zstd_version))
else
error('Cannot find project version in @0@'.format(zstd_h_file))
endif
zstd_libversion = zstd_version
# =============================================================================

6
contrib/linux-kernel/test/test.c
View File

@ -186,11 +186,14 @@ static void __attribute__((noinline)) use(void *x) {
asm volatile("" : "+r"(x));
}
static void __attribute__((noinline)) fill_stack(void) {
memset(g_stack, 0x33, 8192);
}
static void __attribute__((noinline)) set_stack(void) {
char stack[8192];
g_stack = stack;
memset(g_stack, 0x33, 8192);
use(g_stack);
}
@ -208,6 +211,7 @@ static void __attribute__((noinline)) check_stack(void) {
static void test_stack_usage(test_data_t const *data) {
set_stack();
fill_stack();
test_f2fs();
test_btrfs(data);
test_decompress_unzstd(data);

2
lib/common/huf.h
View File

@ -173,7 +173,7 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
/* ****************************************
* HUF detailed API
* ****************************************/
#define HUF_OPTIMAL_DEPTH_THRESHOLD 3
#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
typedef enum {
HUF_depth_fast, /** Use heuristic to find the table depth**/
HUF_depth_optimal /** Test possible table depths to find the one that produces the smallest header + encoded size**/

32
lib/compress/huf_compress.c
View File

@ -1253,14 +1253,6 @@ unsigned HUF_minTableLog(unsigned symbolCardinality)
return minBitsSymbols;
}
#define ESTIMATE_TOTAL_SIZE(huffLog) {\
maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, huffLog, workSpace, wkspSize);\
if (ERR_isError(maxBits)) continue;\
hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);\
if (ERR_isError(hSize)) continue;\
newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;\
}\
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, size_t wkspSize, HUF_CElt* table, const unsigned* count, HUF_depth_mode depthMode)
{
unsigned optLog = FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
@ -1273,28 +1265,20 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
size_t optSize = ((size_t) ~0);
unsigned optLogGuess = optLog;
if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) return optLog; /** Assert workspace is large enough **/
/* Search below estimate log until size increases */
for (; optLogGuess >= minTableLog; optLogGuess--) {
ESTIMATE_TOTAL_SIZE(optLogGuess);
/* Search until size increases */
for (unsigned optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
if (newSize > optSize) {
break;
}
optSize = newSize;
optLog = optLogGuess;
}
if (ERR_isError(maxBits)) continue;
if (optSize < ((size_t) ~0)) {
return optLog;
}
hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
/* Search above estimate log until size increases */
for (; optLogGuess <= maxTableLog; optLogGuess++) {
ESTIMATE_TOTAL_SIZE(optLogGuess);
if (ERR_isError(hSize)) continue;
newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
if (newSize > optSize) {
break;

15
lib/compress/zstd_compress.c
View File

@ -265,9 +265,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
}
/* Returns 1 if compression parameters are such that we should
/* Returns ZSTD_ps_enable if compression parameters are such that we should
* enable long distance matching (wlog >= 27, strategy >= btopt).
* Returns 0 otherwise.
* Returns ZSTD_ps_disable otherwise.
*/
static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
@ -482,8 +482,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
return bounds;
case ZSTD_c_enableLongDistanceMatching:
bounds.lowerBound = 0;
bounds.upperBound = 1;
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_ldmHashLog:
@ -854,6 +854,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
return (size_t)CCtxParams->enableDedicatedDictSearch;
case ZSTD_c_enableLongDistanceMatching :
BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
return CCtxParams->ldmParams.enableLdm;
@ -1096,7 +1097,7 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
{
DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't set pledgedSrcSize when not in init stage.");
cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
@ -1369,8 +1370,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
}
/* resize windowLog if input is small enough, to use less memory */
if ( (srcSize < maxWindowResize)
&& (dictSize < maxWindowResize) ) {
if ( (srcSize <= maxWindowResize)
&& (dictSize <= maxWindowResize) ) {
U32 const tSize = (U32)(srcSize + dictSize);
static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :

3
lib/compress/zstd_lazy.c
View File

@ -1078,7 +1078,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
}
# endif /* ZSTD_ARCH_ARM_NEON */
/* SWAR */
{ const size_t chunkSize = sizeof(size_t);
{ const int chunkSize = sizeof(size_t);
const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
const size_t xFF = ~((size_t)0);
const size_t x01 = xFF / 0xFF;
@ -2123,7 +2123,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
}

16
programs/fileio.c
View File

@ -2749,6 +2749,7 @@ typedef struct {
int numSkippableFrames;
int decompUnavailable;
int usesCheck;
BYTE checksum[4];
U32 nbFiles;
unsigned dictID;
} fileInfo_t;
@ -2843,8 +2844,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
info->usesCheck = 1;
ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
info_frame_error, "Error: could not skip past checksum");
ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
info_frame_error, "Error: could not read checksum");
} }
info->numActualFrames++;
}
@ -2936,7 +2937,16 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
(unsigned long long)info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
if (info->usesCheck && info->numActualFrames == 1) {
DISPLAYOUT("Check: %s %02x%02x%02x%02x\n", checkString,
info->checksum[3], info->checksum[2],
info->checksum[1], info->checksum[0]
);
} else {
DISPLAYOUT("Check: %s\n", checkString);
}
DISPLAYOUT("\n");
}
}

9
programs/util.c
View File

@ -569,7 +569,7 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
size_t const curLen = strlen(table2->fileNames[idx2]);
memcpy(buf+pos, table2->fileNames[idx2], curLen);
assert(newTableIdx <= newTable->tableSize);
assert(newTableIdx < newTable->tableSize);
newTable->fileNames[newTableIdx] = buf+pos;
pos += curLen+1;
} }
@ -693,8 +693,11 @@ static int UTIL_prepareFileList(const char *dirName,
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
if (*bufStart != NULL) {
*bufEnd = *bufStart + newListSize;
} else {
free(path); closedir(dir); return 0;
}
}
if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */

6
programs/zstdcli.c
View File

@ -216,7 +216,6 @@ static void usage_advanced(const char* programName)
DISPLAYOUT("\n");
DISPLAYOUT("Advanced compression options :\n");
DISPLAYOUT(" --ultra enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --fast[=#] switch to very fast compression levels (default: %u)\n", 1);
#ifdef ZSTD_GZCOMPRESS
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
@ -224,9 +223,9 @@ static void usage_advanced(const char* programName)
DISPLAYOUT(" --no-name do not store original filename when compressing\n");
}
#endif
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n");
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n");
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n");
# ifdef ZSTD_MULTITHREAD
DISPLAYOUT(" -T# spawn # compression threads (default: 1, 0==# cores) \n");
DISPLAYOUT(" -B# select size of each job (default: 0==automatic) \n");
@ -240,6 +239,7 @@ static void usage_advanced(const char* programName)
DISPLAYOUT(" --target-compressed-block-size=# : generate compressed block of approximately targeted size \n");
DISPLAYOUT(" --no-dictID don't write dictID into header (dictionary compression only)\n");
DISPLAYOUT(" --[no-]compress-literals : force (un)compressed literals\n");
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --format=zstd compress files to the .zst format (default)\n");
#ifdef ZSTD_GZCOMPRESS

9
tests/cli-tests/compression/window-resize.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
datagen -g1G > file
zstd --long=31 -1 --single-thread --no-content-size -f file
zstd -l -v file.zst
# We want to ignore stderr (its outputting "*** zstd command line interface
# 64-bits v1.5.3, by Yann Collet ***")
rm file file.zst

0
tests/cli-tests/compression/window-resize.sh.stderr.ignore Normal file
View File

3
tests/cli-tests/compression/window-resize.sh.stdout.glob Normal file
View File

@ -0,0 +1,3 @@
...
Window Size: 1.000 GiB (1073741824 B)
...

2
tests/fuzz/zstd_helpers.c
View File

@ -80,7 +80,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer);
setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer);
/* Select long distance matching parameters */
setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer);
setRand(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_auto, ZSTD_ps_disable, producer);
setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer);
setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
ZSTD_LDM_MINMATCH_MAX, producer);

22
tests/fuzzer.c
View File

@ -851,7 +851,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(dict, size, 0.5, 0.5, seed);
RDG_genBuffer(src, size, 0.5, 0.5, seed);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
ZSTD_freeCCtx(cctx);
@ -875,7 +875,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
@ -900,7 +900,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
memcpy(dict + testSize, CNBuffer, testSize);
for (level = 1; level <= 5; ++level) {
for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) {
for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
size_t cSize0;
XXH64_hash_t compressedChecksum0;
@ -956,7 +956,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Enable MT, LDM, and opt parser */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
@ -995,7 +995,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Disable content size to skip single-pass decompression. */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
@ -1092,7 +1092,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* compress on level 1 using refPrefix and ldm */
ZSTD_CCtx_refPrefix(cctx, dict, size);;
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1))
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
assert(!ZSTD_isError(refPrefixLdmCompressedSize));
@ -2820,7 +2820,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
assert(cctx != NULL);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
{ size_t const compressedSize = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize),
@ -2836,7 +2836,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
{ size_t const result = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize),
@ -3492,7 +3492,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Enable MT, LDM, and use refPrefix() for a small dict */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
@ -3686,7 +3686,7 @@ static int longUnitTests(U32 const seed, double compressibility)
/* Enable checksum to verify round trip. */
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
/* Round trip once with ldm. */
@ -3696,7 +3696,7 @@ static int longUnitTests(U32 const seed, double compressibility)
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
/* Round trip once without ldm. */

4
tests/regression/config.c
View File

@ -183,7 +183,7 @@ static config_t no_pledged_src_size_with_dict = {
};
static param_value_t const ldm_param_values[] = {
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
{.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
};
static config_t ldm = {
@ -204,7 +204,7 @@ static config_t mt = {
static param_value_t const mt_ldm_param_values[] = {
{.param = ZSTD_c_nbWorkers, .value = 2},
{.param = ZSTD_c_enableLongDistanceMatching, .value = 1},
{.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
};
static config_t mt_ldm = {

2
tests/zstreamtest.c
View File

@ -2281,7 +2281,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
/* mess with long distance matching parameters */
if (bigTests) {
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );