sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-09 00:03:18 -05:00
Code Issues Projects Releases Wiki Activity

ZSTD_compressBlock() limits block size depending on windowLog parameter

Browse Source
This commit is contained in:
Yann Collet 2016-07-15 11:56:53 +02:00
parent 227cc39e15
commit 961b6a0e34
3 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/common/zstd.h
View File

@ -441,7 +441,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
User will have to take in charge required information to regenerate data, such as compressed and content sizes. User will have to take in charge required information to regenerate data, such as compressed and content sizes.
A few rules to respect : A few rules to respect :
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) - Uncompressed block size must be <= MIN (128 KB, 1 << windowLog)
+ If you need to compress more, cut data into multiple blocks + If you need to compress more, cut data into multiple blocks
+ Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
- Compressing and decompressing require a context structure - Compressing and decompressing require a context structure

11
lib/compress/zstd_compress.c
View File

@ -274,9 +274,9 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
note : 'params' is expected to be validated */ note : 'params' is expected to be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params, U64 frameContentSize, ZSTD_parameters params, U64 frameContentSize,
U32 reset, U32 fullBlockSize) U32 reset)
{ /* note : params considered validated here */ { /* note : params considered validated here */
const size_t blockSize = fullBlockSize ? ZSTD_BLOCKSIZE_MAX : MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
const size_t maxNbSeq = blockSize / divider; const size_t maxNbSeq = blockSize / divider;
const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t tokenSpace = blockSize + 11*maxNbSeq;
@ -358,7 +358,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
if (srcCCtx->stage!=1) return ERROR(stage_wrong); if (srcCCtx->stage!=1) return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0, 1); ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0);
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
/* copy tables */ /* copy tables */
@ -2560,7 +2560,8 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc,
size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << zc->params.cParams.windowLog);
if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.cParams.searchLength); ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.cParams.searchLength);
return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0);
} }
@ -2695,7 +2696,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize) ZSTD_parameters params, U64 pledgedSrcSize)
{ {
size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1, (pledgedSrcSize==0) ); size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1);
if (ZSTD_isError(resetError)) return resetError; if (ZSTD_isError(resetError)) return resetError;
return ZSTD_compress_insertDictionary(zc, dict, dictSize); return ZSTD_compress_insertDictionary(zc, dict, dictSize);

7
lib/dictBuilder/zdict.c
View File

@ -581,13 +581,14 @@ typedef struct
#define MAXREPOFFSET 1024 #define MAXREPOFFSET 1024
static void ZDICT_countEStats(EStats_ress_t esr, static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog);
size_t cSize; size_t cSize;
if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
} }
@ -721,7 +722,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
/* collect stats on all files */ /* collect stats on all files */
for (u=0; u<nbFiles; u++) { for (u=0; u<nbFiles; u++) {
ZDICT_countEStats(esr, ZDICT_countEStats(esr, params,
countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset, countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
(const char*)srcBuffer + pos, fileSizes[u]); (const char*)srcBuffer + pos, fileSizes[u]);
pos += fileSizes[u]; pos += fileSizes[u];