sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-16 00:03:55 -05:00
Code Issues Projects Releases Wiki Activity

Merge pull request #936 from facebook/qemu32panic

Browse Source 
Reduce memory stress for qemu tests
This commit is contained in:
Yann Collet 2017-12-04 14:16:55 -08:00 committed by GitHub
commit 896980ec15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 30 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/compress/zstd_compress.c
View File

@ -241,6 +241,7 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams(
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value)
{ {
DEBUGLOG(4, "ZSTD_CCtx_setParameter");
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
switch(param) switch(param)
@ -269,13 +270,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content * even when referencing into Dictionary content.
* default : 0 when using a CDict, 1 when using a Prefix */ * default : 0 when using a CDict, 1 when using a Prefix */
cctx->loadedDictEnd = 0;
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_nbThreads: case ZSTD_p_nbThreads:
if (value > 1 && cctx->staticSize) { if ((value > 1) && cctx->staticSize) {
return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ return ERROR(parameter_unsupported); /* MT not compatible with static alloc */
} }
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
@ -320,6 +320,7 @@ size_t ZSTD_CCtxParam_setParameter(
return CCtxParams->compressionLevel; return CCtxParams->compressionLevel;
case ZSTD_p_windowLog : case ZSTD_p_windowLog :
DEBUGLOG(4, "setting windowLog=%u", value);
if (value) { /* 0 : does not change current windowLog */ if (value) { /* 0 : does not change current windowLog */
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
ZSTD_cLevelToCCtxParams(CCtxParams); ZSTD_cLevelToCCtxParams(CCtxParams);
@ -459,13 +460,9 @@ size_t ZSTD_CCtxParam_setParameter(
} }
} }
/** /** ZSTD_CCtx_setParametersUsingCCtxParams() :
* This function should be updated whenever ZSTD_CCtx_params is updated. * just applies `params` into `cctx`
* Parameters are copied manually before the dictionary is loaded. * no action is performed, parameters are merely stored.
* The multithreading parameters jobSize and overlapSizeLog are set only if
* nbThreads > 1.
*
* Pledged srcSize is treated as unknown.
*/ */
size_t ZSTD_CCtx_setParametersUsingCCtxParams( size_t ZSTD_CCtx_setParametersUsingCCtxParams(
ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params)
@ -473,27 +470,8 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
/* Assume the compression and frame parameters are validated */ cctx->requestedParams = *params;
cctx->requestedParams.cParams = params->cParams;
cctx->requestedParams.fParams = params->fParams;
cctx->requestedParams.compressionLevel = params->compressionLevel;
/* Set force window explicitly since it sets cctx->loadedDictEnd */
CHECK_F( ZSTD_CCtx_setParameter(
cctx, ZSTD_p_forceMaxWindow, params->forceWindow) );
/* Set multithreading parameters explicitly */
CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) );
if (params->nbThreads > 1) {
CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) );
CHECK_F( ZSTD_CCtx_setParameter(
cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) );
}
/* Copy long distance matching parameters */
cctx->requestedParams.ldmParams = params->ldmParams;
/* customMem is used only for create/free params and can be ignored */
return 0; return 0;
} }
@ -2091,7 +2069,6 @@ size_t ZSTD_compressBegin_advanced_internal(
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal"); DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal");
DEBUGLOG(4, "contentSizeFlag : %u", params.fParams.contentSizeFlag);
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,

4
lib/compress/zstdmt_compress.c
View File

@ -140,7 +140,7 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
return poolSize + totalBufferSize; return poolSize + totalBufferSize;
} }
static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize) static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize)
{ {
ZSTD_pthread_mutex_lock(&bufPool->poolMutex); ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize);
@ -359,7 +359,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
assert(job->firstChunk); /* should only happen for first segment */ assert(job->firstChunk); /* should only happen for first segment */
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */ } else { /* srcStart points at reloaded section */
ZSTD_CCtx_params jobParams = job->params; ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */
size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN; U64 const pledgedSrcSize = job->firstChunk ? job->fullFrameSize : ZSTD_CONTENTSIZE_UNKNOWN;
/* load dictionary in "content-only" mode (no header analysis) */ /* load dictionary in "content-only" mode (no header analysis) */

2
programs/fileio.c
View File

@ -639,7 +639,6 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
memset(&prefs, 0, sizeof(prefs)); memset(&prefs, 0, sizeof(prefs));
assert(blockSize <= ress->srcBufferSize); assert(blockSize <= ress->srcBufferSize);
assert(LZ4F_compressBound(blockSize) <= ress->dstBufferSize);
prefs.autoFlush = 1; prefs.autoFlush = 1;
prefs.compressionLevel = compressionLevel; prefs.compressionLevel = compressionLevel;
@ -649,6 +648,7 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
#if LZ4_VERSION_NUMBER >= 10600 #if LZ4_VERSION_NUMBER >= 10600
prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
#endif #endif
assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize);
{ {
size_t readSize; size_t readSize;

33
tests/zstreamtest.c
View File

@ -1482,43 +1482,47 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
cParams = ZSTD_adjustCParams(cParams, 0, 0); cParams = ZSTD_adjustCParams(cParams, 0, 0);
if (FUZ_rand(&lseed) & 1) { if (FUZ_rand(&lseed) & 1) {
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) ); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_windowLog, cParams.windowLog, useOpaqueAPI) );
assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
} }
if (FUZ_rand(&lseed) & 1) { if (FUZ_rand(&lseed) & 1) {
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog); DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_hashLog, cParams.hashLog, useOpaqueAPI) );
} }
if (FUZ_rand(&lseed) & 1) { if (FUZ_rand(&lseed) & 1) {
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog); DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_chainLog, cParams.chainLog, useOpaqueAPI) );
} }
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_searchLog, cParams.searchLog, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_minMatch, cParams.searchLength, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_targetLength, cParams.targetLength, useOpaqueAPI) );
/* mess with long distance matching parameters */ /* mess with long distance matching parameters */
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) ); if (bigTests) {
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), useOpaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), useOpaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) ); if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_LDM_BUCKETSIZELOG_MAX), useOpaqueAPI) );
if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_ldmHashEveryLog, FUZ_randomClampedLength(&lseed, 0, ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN), useOpaqueAPI) );
}
/* mess with frame parameters */ /* mess with frame parameters */
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); if (FUZ_rand(&lseed) & 1) {
DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize); DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (U32)pledgedSrcSize);
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
}
/* multi-threading parameters */ /* multi-threading parameters */
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); U32 const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads); DISPLAYLEVEL(5, "t%u: nbThreads : %u \n", testNb, nbThreads);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_nbThreads, nbThreads, useOpaqueAPI) );
if (nbThreads > 1) { if (nbThreads > 1) {
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) ); CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10, useOpaqueAPI) );
@ -1526,10 +1530,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
} }
} }
if (FUZ_rand(&lseed) & 1) CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1, useOpaqueAPI) );
/* Apply parameters */ /* Apply parameters */
if (useOpaqueAPI) { if (useOpaqueAPI) {
DISPLAYLEVEL(6," t%u: applying CCtxParams \n", testNb);
CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) ); CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
} }
@ -1569,8 +1574,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
outBuff.size = outBuff.pos + dstBuffSize; outBuff.size = outBuff.pos + dstBuffSize;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
DISPLAYLEVEL(6, "compress consumed %u bytes (total : %u) \n", DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) \n",
(U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos)); testNb, (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);