sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-10 00:03:36 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #739 from facebook/refPrefix

Browse Source 
ZSTD_refPrefix
This commit is contained in:
Yann Collet 2017-06-29 04:36:03 -07:00 committed by GitHub
commit 590937df20
8 changed files with 221 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
doc/zstd_manual.html
View File

@ -412,7 +412,7 @@ size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDCtxSize(void); size_t ZSTD_estimateDCtxSize(void);
</b><p> These functions make it possible to estimate memory usage </b><p> These functions make it possible to estimate memory usage
of a future {D,C}Ctx, before its creation. of a future {D,C}Ctx, before its creation.
ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one. ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case. It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
@ -423,7 +423,7 @@ size_t ZSTD_estimateDCtxSize(void);
size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams); size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDStreamSize(size_t windowSize); size_t ZSTD_estimateDStreamSize(size_t windowSize);
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
</b><p> ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one. </b><p> ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case. It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
@ -483,7 +483,10 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict
</p></pre><BR> </p></pre><BR>
<pre><b>typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e; <pre><b>typedef enum { ZSTD_dm_auto=0, </b>/* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, rawContent otherwize */<b>
ZSTD_dm_rawContent, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b>
ZSTD_dm_fullDict </b>/* refuses to load a dictionary if it does not respect Zstandard's specification */<b>
} ZSTD_dictMode_e;
</b></pre><BR> </b></pre><BR>
<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, <pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_dictMode_e dictMode, unsigned byReference, ZSTD_dictMode_e dictMode,
@ -804,26 +807,17 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
* The higher the value of selected strategy, the more complex it is, * The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression. * resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */ * Special: value 0 means "do not change strategy". */
#if 0
ZSTD_p_windowSize, </b>/* Maximum allowed back-reference distance.<b>
* Can be set to a more precise value than windowLog.
* Will be transparently reduced to closest possible inferior value
* (see Zstandard compression format) */
</b>/* Not ready yet ! */<b>
#endif
</b>/* frame parameters */<b> </b>/* frame parameters */<b>
ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1) */<b> ZSTD_p_contentSizeFlag=200, </b>/* Content size is written into frame header _whenever known_ (default:1) */<b>
ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b> ZSTD_p_checksumFlag, </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
ZSTD_p_dictIDFlag, </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b> ZSTD_p_dictIDFlag, </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
</b>/* dictionary parameters */<b> </b>/* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */<b>
ZSTD_p_refDictContent=300, </b>/* Content of dictionary content will be referenced, instead of copied (default:0).<b> ZSTD_p_dictMode=300, </b>/* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.<b>
* This avoids duplicating dictionary content.
* But it also requires that dictionary buffer outlives its users */
</b>/* Not ready yet ! <=================================== */<b>
ZSTD_p_dictMode, </b>/* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.<b>
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */ * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
ZSTD_p_refDictContent, </b>/* Dictionary content will be referenced, instead of copied (default:0==byCopy).<b>
* It requires that dictionary buffer outlives its users */
</b>/* multi-threading parameters */<b> </b>/* multi-threading parameters */<b>
ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b> ZSTD_p_nbThreads=400, </b>/* Select how many threads a compression job can spawn (default:1)<b>
@ -838,9 +832,9 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */ * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
</b>/* advanced parameters - may not remain available after API update */<b> </b>/* advanced parameters - may not remain available after API update */<b>
ZSTD_p_forceMaxWindow=1100, </b>/* Force back-references to remain < windowSize,<b> ZSTD_p_forceMaxWindow=1100, </b>/* Force back-reference distances to remain < windowSize,<b>
* even when referencing into Dictionary content. * even when referencing into Dictionary content (default:0) */
* default : 0 when using a CDict, 1 when using a Prefix */
} ZSTD_cParameter; } ZSTD_cParameter;
</b></pre><BR> </b></pre><BR>
<pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); <pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
@ -866,8 +860,8 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
@result : 0, or an error code (which can be tested with ZSTD_isError()). @result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
meaning "return to no-dictionary mode". meaning "return to no-dictionary mode".
Note 1 : Dictionary content will be copied internally, Note 1 : `dict` content will be copied internally,
except if ZSTD_p_refDictContent is set. except if ZSTD_p_refDictContent is set before loading.
Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters. Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
For this reason, compression parameters cannot be changed anymore after loading a dictionary. For this reason, compression parameters cannot be changed anymore after loading a dictionary.
It's also a CPU-heavy operation, with non-negligible impact on latency. It's also a CPU-heavy operation, with non-negligible impact on latency.
@ -876,7 +870,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); <pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
</b><p> Ref a prepared dictionary, to be used for all next compression jobs. </b><p> Reference a prepared dictionary, to be used for all next compression jobs.
Note that compression parameters are enforced from within CDict, Note that compression parameters are enforced from within CDict,
and supercede any compression parameter previously set within CCtx. and supercede any compression parameter previously set within CCtx.
The dictionary will remain valid for future compression jobs using same CCtx. The dictionary will remain valid for future compression jobs using same CCtx.
@ -888,16 +882,18 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); </b>/* Not ready yet ! <===================================== */<b> <pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
</b><p> Reference a prefix (content-only dictionary) to bootstrap next compression job. </b><p> Reference a prefix (single-usage dictionary) for next compression job.
Decompression will have to use same prefix. Decompression need same prefix to properly regenerate data.
Prefix is only used once. Tables are discarded at end of compression job. Prefix is **only used once**. Tables are discarded at end of compression job.
If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict. Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
@result : 0, or an error code (which can be tested with ZSTD_isError()). @result : 0, or an error code (which can be tested with ZSTD_isError()).
Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode". Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
Note 1 : Prefix buffer is referenced. It must outlive compression job. Note 1 : Prefix buffer is referenced. It must outlive compression job.
Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters. Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
It's a CPU-heavy operation, with non-negligible impact on latency. It's a CPU-heavy operation, with non-negligible impact on latency.
Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter()
</p></pre><BR> </p></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {

10
lib/common/zstd_internal.h
View File

@ -71,9 +71,14 @@
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
# include <stdio.h> # include <stdio.h>
static unsigned g_debugLevel = ZSTD_DEBUG; /* recommended values for ZSTD_DEBUG display levels :
* 2 : reserved for currently active debugging path
* 3 : events once per object lifetime (CCtx, CDict)
* 4 : events once per frame
* 5 : events once per block
* 6 : events once per sequence (*very* verbose) */
# define DEBUGLOG(l, ...) { \ # define DEBUGLOG(l, ...) { \
if (l<=g_debugLevel) { \ if (l<=ZSTD_DEBUG) { \
fprintf(stderr, __FILE__ ": "); \ fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \ fprintf(stderr, " \n"); \
@ -98,7 +103,6 @@
* Common constants * Common constants
***************************************/ ***************************************/
#define ZSTD_OPT_NUM (1<<12) #define ZSTD_OPT_NUM (1<<12)
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
#define ZSTD_REP_NUM 3 /* number of repcodes */ #define ZSTD_REP_NUM 3 /* number of repcodes */
#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ #define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */

121
lib/compress/zstd_compress.c
View File

@ -88,7 +88,6 @@ struct ZSTD_CCtx_s {
U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */ U32 loadedDictEnd; /* index of end of dictionary */
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
ZSTD_compressionStage_e stage; ZSTD_compressionStage_e stage;
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
U32 repToConfirm[ZSTD_REP_NUM]; U32 repToConfirm[ZSTD_REP_NUM];
@ -118,8 +117,6 @@ struct ZSTD_CCtx_s {
unsigned* entropyScratchSpace; unsigned* entropyScratchSpace;
/* streaming */ /* streaming */
ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inToCompress; size_t inToCompress;
@ -132,6 +129,14 @@ struct ZSTD_CCtx_s {
ZSTD_cStreamStage streamStage; ZSTD_cStreamStage streamStage;
U32 frameEnded; U32 frameEnded;
/* Dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
U32 dictContentByRef;
ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
const void* prefix;
size_t prefixSize;
/* Multi-threading */ /* Multi-threading */
U32 nbThreads; U32 nbThreads;
ZSTDMT_CCtx* mtctx; ZSTDMT_CCtx* mtctx;
@ -322,11 +327,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value; cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
return 0; return 0;
#if 0
case ZSTD_p_windowSize : /* to be done later */
return ERROR(compressionParameter_unsupported);
#endif
case ZSTD_p_contentSizeFlag : case ZSTD_p_contentSizeFlag :
DEBUGLOG(5, "set content size flag = %u", (value>0)); DEBUGLOG(5, "set content size flag = %u", (value>0));
/* Content size written in frame header _when known_ (default:1) */ /* Content size written in frame header _when known_ (default:1) */
@ -343,10 +343,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->requestedParams.fParams.noDictIDFlag = (value==0); cctx->requestedParams.fParams.noDictIDFlag = (value==0);
return 0; return 0;
case ZSTD_p_refDictContent : /* to be done later */ /* Dictionary parameters */
return ERROR(compressionParameter_unsupported);
case ZSTD_p_dictMode : case ZSTD_p_dictMode :
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
/* restrict dictionary mode, to "rawContent" or "fullDict" only */ /* restrict dictionary mode, to "rawContent" or "fullDict" only */
ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent); ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
if (value > (unsigned)ZSTD_dm_fullDict) if (value > (unsigned)ZSTD_dm_fullDict)
@ -354,6 +353,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->dictMode = (ZSTD_dictMode_e)value; cctx->dictMode = (ZSTD_dictMode_e)value;
return 0; return 0;
case ZSTD_p_refDictContent :
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
/* dictionary content will be referenced, instead of copied */
cctx->dictContentByRef = value>0;
return 0;
case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content * even when referencing into Dictionary content
* default : 0 when using a CDict, 1 when using a Prefix */ * default : 0 when using a CDict, 1 when using a Prefix */
@ -417,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
ZSTD_getCParams(cctx->compressionLevel, 0, dictSize); ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced( cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize, dict, dictSize,
0 /* byReference */, cctx->dictMode, cctx->dictContentByRef, cctx->dictMode,
cParams, cctx->customMem); cParams, cctx->customMem);
cctx->cdict = cctx->cdictLocal; cctx->cdict = cctx->cdictLocal;
if (cctx->cdictLocal == NULL) if (cctx->cdictLocal == NULL)
@ -426,19 +431,22 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
return 0; return 0;
} }
/* Not ready yet ! */
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
{
(void)cctx; (void)prefix; (void)prefixSize; /* to be done later */
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
return ERROR(compressionParameter_unsupported);
}
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{ {
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->cdict = cdict; cctx->cdict = cdict;
return ERROR(compressionParameter_unsupported); cctx->prefix = NULL; /* exclusive */
cctx->prefixSize = 0;
return 0;
}
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->cdict = NULL; /* prefix discards any prior cdict */
cctx->prefix = prefix;
cctx->prefixSize = prefixSize;
return 0;
} }
static void ZSTD_startNewCompression(ZSTD_CCtx* cctx) static void ZSTD_startNewCompression(ZSTD_CCtx* cctx)
@ -3180,7 +3188,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
if (dictMode==ZSTD_dm_rawContent) if (dictMode==ZSTD_dm_rawContent)
return ZSTD_loadDictionaryContent(cctx, dict, dictSize); return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) { if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
if (dictMode == ZSTD_dm_auto) { if (dictMode == ZSTD_dm_auto) {
DEBUGLOG(5, "raw content dictionary detected"); DEBUGLOG(5, "raw content dictionary detected");
return ZSTD_loadDictionaryContent(cctx, dict, dictSize); return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
@ -3621,14 +3629,18 @@ size_t ZSTD_CStreamOutSize(void)
} }
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
ZSTD_parameters params, const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
unsigned long long pledgedSrcSize) const ZSTD_CDict* cdict,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(5, "ZSTD_resetCStream_internal"); DEBUGLOG(5, "ZSTD_resetCStream_internal");
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
CHECK_F( ZSTD_compressBegin_internal(zcs, CHECK_F( ZSTD_compressBegin_internal(zcs,
NULL, 0, ZSTD_dm_auto, dict, dictSize, dictMode,
zcs->cdict, cdict,
params, pledgedSrcSize, params, pledgedSrcSize,
ZSTDb_buffered) ); ZSTDb_buffered) );
@ -3649,10 +3661,11 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) { if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */); params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
} }
return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
} }
/*! ZSTD_initCStream_internal() : /*! ZSTD_initCStream_internal() :
* Note : not static, but hidden (not exposed). Used by zstdmt_compress.c
* Assumption 1 : params are valid * Assumption 1 : params are valid
* Assumption 2 : either dict, or cdict, is defined, not both */ * Assumption 2 : either dict, or cdict, is defined, not both */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
@ -3671,7 +3684,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
} }
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byReference */, ZSTD_dm_auto, zcs->dictContentByRef, zcs->dictMode,
params.cParams, zcs->customMem); params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@ -3687,7 +3700,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
zcs->requestedParams = params; zcs->requestedParams = params;
zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM; zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
} }
/* ZSTD_initCStream_usingCDict_advanced() : /* ZSTD_initCStream_usingCDict_advanced() :
@ -3911,26 +3924,30 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (cctx->streamStage == zcss_init) { if (cctx->streamStage == zcss_init) {
/* transparent reset */ /* transparent reset */
const void* const prefix = cctx->prefix;
size_t const prefixSize = cctx->prefixSize;
ZSTD_parameters params = cctx->requestedParams; ZSTD_parameters params = cctx->requestedParams;
if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM) if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
params.cParams = ZSTD_getCParams(cctx->compressionLevel, params.cParams = ZSTD_getCParams(cctx->compressionLevel,
cctx->pledgedSrcSizePlusOne-1, 0 /* dictSize */); cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
cctx->prefix = NULL; cctx->prefixSize = 0; /* single usage */
assert(prefix==NULL || cctx->cdict==NULL); /* only one can be set */
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
if (cctx->nbThreads > 1) { if (cctx->nbThreads > 1) {
DEBUGLOG(4, "call ZSTDMT_initCStream_internal"); DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", cctx->nbThreads);
CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
cctx->streamStage = zcss_load; cctx->streamStage = zcss_load;
} else } else
#endif #endif
{ {
CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne-1) ); CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
} } } }
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
if (cctx->nbThreads > 1) { if (cctx->nbThreads > 1) {
size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
DEBUGLOG(4, "ZSTDMT_compressStream_generic : %u", (U32)flushMin); DEBUGLOG(5, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
if ( ZSTD_isError(flushMin) if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
ZSTD_startNewCompression(cctx); ZSTD_startNewCompression(cctx);
@ -4098,42 +4115,24 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
}, },
}; };
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
/* This function just controls /* This function just controls
* the monotonic memory budget increase of ZSTD_defaultCParameters[0]. * the monotonic memory budget increase of ZSTD_defaultCParameters[0].
* Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled * Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
*/ */
MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void) MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
{ {
int level;
for (level=1; level<ZSTD_maxCLevel(); level++) {
ZSTD_compressionParameters const c1 = ZSTD_defaultCParameters[0][level];
ZSTD_compressionParameters const c2 = ZSTD_defaultCParameters[0][level+1];
DEBUGLOG(3, "controlling compression params level %i", level);
assert(c1.windowLog <= c2.windowLog);
# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c))) # define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field) assert(ZSTD_TABLECOST(c1.hashLog, c1.chainLog) <= ZSTD_TABLECOST(c2.hashLog, c2.chainLog));
# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \
assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
} }
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
assert(ZSTD_maxCLevel()==22);
} }
#endif
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. * @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.

2
lib/compress/zstdmt_compress.c
View File

@ -586,7 +586,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
if (dict) { if (dict) {
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byRef */, ZSTD_dm_auto, 0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */
params.cParams, zcs->cMem); params.cParams, zcs->cMem);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);

44
lib/decompress/zstd_decompress.c
View File

@ -750,6 +750,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const BYTE* const istart = (const BYTE* const)src; const BYTE* const istart = (const BYTE* const)src;
const BYTE* const iend = istart + srcSize; const BYTE* const iend = istart + srcSize;
const BYTE* ip = istart; const BYTE* ip = istart;
DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
/* check */ /* check */
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
@ -930,12 +931,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
seq.offset = offset; seq.offset = offset;
} }
seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ seq.matchLength = ML_base[mlCode]
+ ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ seq.litLength = LL_base[llCode]
if (MEM_32bits() || + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); if ( MEM_32bits()
|| (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) )
BIT_reloadDStream(&seqState->DStream);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
/* ANS state update */ /* ANS state update */
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
@ -975,7 +982,8 @@ size_t ZSTD_execSequence(BYTE* op,
/* copy Match */ /* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) { if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix -> go into extDict */ /* offset beyond prefix -> go into extDict */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
match = dictEnd + (match - base); match = dictEnd + (match - base);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength); memmove(oLitEnd, match, sequence.matchLength);
@ -1043,9 +1051,12 @@ static size_t ZSTD_decompressSequences(
const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq; int nbSeq;
DEBUGLOG(5, "ZSTD_decompressSequences");
/* Build Decoding Tables */ /* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i",
(U32)seqHSize, nbSeq);
if (ZSTD_isError(seqHSize)) return seqHSize; if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize; ip += seqHSize;
} }
@ -1064,11 +1075,13 @@ static size_t ZSTD_decompressSequences(
nbSeq--; nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState); { seq_t const sequence = ZSTD_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} } } }
/* check if reached exact end */ /* check if reached exact end */
DEBUGLOG(5, "after decode loop, remaining nbSeq : %i", nbSeq);
if (nbSeq) return ERROR(corruption_detected); if (nbSeq) return ERROR(corruption_detected);
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
@ -1355,11 +1368,13 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ /* blockType == blockCompressed */ { /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
DEBUGLOG(5, "ZSTD_decompressBlock_internal");
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
/* Decode literals section */ /* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
if (ZSTD_isError(litCSize)) return litCSize; if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize; ip += litCSize;
srcSize -= litCSize; srcSize -= litCSize;
@ -1697,6 +1712,7 @@ static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skip
* or an error code, which can be tested using ZSTD_isError() */ * or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
DEBUGLOG(5, "ZSTD_decompressContinue");
/* Sanity check */ /* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* unauthorized */ if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* unauthorized */
if (dstCapacity) ZSTD_checkContinuity(dctx, dst); if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
@ -1756,10 +1772,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
} }
case ZSTDds_decompressLastBlock: case ZSTDds_decompressLastBlock:
case ZSTDds_decompressBlock: case ZSTDds_decompressBlock:
DEBUGLOG(5, "case ZSTDds_decompressBlock");
{ size_t rSize; { size_t rSize;
switch(dctx->bType) switch(dctx->bType)
{ {
case bt_compressed: case bt_compressed:
DEBUGLOG(5, "case bt_compressed");
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
break; break;
case bt_raw : case bt_raw :
@ -1884,7 +1902,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
{ {
if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
{ U32 const magic = MEM_readLE32(dict); { U32 const magic = MEM_readLE32(dict);
if (magic != ZSTD_DICT_MAGIC) { if (magic != ZSTD_MAGIC_DICTIONARY) {
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
} } } }
dctx->dictID = MEM_readLE32((const char*)dict + 4); dctx->dictID = MEM_readLE32((const char*)dict + 4);
@ -1964,7 +1982,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
ddict->entropyPresent = 0; ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0; if (ddict->dictSize < 8) return 0;
{ U32 const magic = MEM_readLE32(ddict->dictContent); { U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_DICT_MAGIC) return 0; /* pure content mode */ if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */
} }
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4); ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
@ -2083,7 +2101,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
{ {
if (dictSize < 8) return 0; if (dictSize < 8) return 0;
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0; if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
return MEM_readLE32((const char*)dict + 4); return MEM_readLE32((const char*)dict + 4);
} }
@ -2323,7 +2341,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} } } }
/* Consume header (see ZSTDds_decodeFrameHeader) */ /* Consume header (see ZSTDds_decodeFrameHeader) */
DEBUGLOG(5, "Consume header"); DEBUGLOG(4, "Consume header");
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
@ -2336,7 +2354,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
/* control buffer memory usage */ /* control buffer memory usage */
DEBUGLOG(5, "Control max buffer memory usage"); DEBUGLOG(4, "Control max buffer memory usage");
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
@ -2346,12 +2364,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->blockSize = blockSize; zds->blockSize = blockSize;
if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) { if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
size_t const bufferSize = blockSize + neededOutSize; size_t const bufferSize = blockSize + neededOutSize;
DEBUGLOG(5, "inBuff : from %u to %u", DEBUGLOG(4, "inBuff : from %u to %u",
(U32)zds->inBuffSize, (U32)blockSize); (U32)zds->inBuffSize, (U32)blockSize);
DEBUGLOG(5, "outBuff : from %u to %u", DEBUGLOG(4, "outBuff : from %u to %u",
(U32)zds->outBuffSize, (U32)neededOutSize); (U32)zds->outBuffSize, (U32)neededOutSize);
if (zds->staticSize) { /* static DCtx */ if (zds->staticSize) { /* static DCtx */
DEBUGLOG(5, "staticSize : %u", (U32)zds->staticSize); DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx)) if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
return ERROR(memory_allocation); return ERROR(memory_allocation);

6
lib/dictBuilder/zdict.c
View File

@ -94,7 +94,7 @@ const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(error
unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
{ {
if (dictSize < 8) return 0; if (dictSize < 8) return 0;
if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0; if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0;
return MEM_readLE32((const char*)dictBuffer + 4); return MEM_readLE32((const char*)dictBuffer + 4);
} }
@ -854,7 +854,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall); if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
/* dictionary header */ /* dictionary header */
MEM_writeLE32(header, ZSTD_DICT_MAGIC); MEM_writeLE32(header, ZSTD_MAGIC_DICTIONARY);
{ U64 const randomID = XXH64(customDictContent, dictContentSize, 0); { U64 const randomID = XXH64(customDictContent, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID; U32 const dictID = params.dictID ? params.dictID : compliantID;
@ -906,7 +906,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
} }
/* add dictionary header (after entropy tables) */ /* add dictionary header (after entropy tables) */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); MEM_writeLE32(dictBuffer, ZSTD_MAGIC_DICTIONARY);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID; U32 const dictID = params.dictID ? params.dictID : compliantID;

53
lib/zstd.h
View File

@ -360,6 +360,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
/* --- Constants ---*/ /* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
#define ZSTD_WINDOWLOG_MAX_32 27 #define ZSTD_WINDOWLOG_MAX_32 27
#define ZSTD_WINDOWLOG_MAX_64 27 #define ZSTD_WINDOWLOG_MAX_64 27
@ -379,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
#define ZSTD_FRAMEHEADERSIZE_MIN 6 #define ZSTD_FRAMEHEADERSIZE_MIN 6
static const size_t ZSTD_frameHeaderSize_prefix = 5; static const size_t ZSTD_frameHeaderSize_prefix = 5; /* minimum input size to know frame header size */
static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
@ -574,7 +575,10 @@ ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter par
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e; typedef enum { ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, rawContent otherwize */
ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
} ZSTD_dictMode_e;
/*! ZSTD_createCDict_advanced() : /*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
@ -944,26 +948,17 @@ typedef enum {
* The higher the value of selected strategy, the more complex it is, * The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression. * resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */ * Special: value 0 means "do not change strategy". */
#if 0
ZSTD_p_windowSize, /* Maximum allowed back-reference distance.
* Can be set to a more precise value than windowLog.
* Will be transparently reduced to closest possible inferior value
* (see Zstandard compression format) */
/* Not ready yet ! */
#endif
/* frame parameters */ /* frame parameters */
ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */ ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */ ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
/* dictionary parameters */ /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0). ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
* This avoids duplicating dictionary content.
* But it also requires that dictionary buffer outlives its users */
/* Not ready yet ! <=================================== */
ZSTD_p_dictMode, /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */ * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
* It requires that dictionary buffer outlives its users */
/* multi-threading parameters */ /* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1) ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
@ -978,9 +973,9 @@ typedef enum {
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */ * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
/* advanced parameters - may not remain available after API update */ /* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize, ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content. * even when referencing into Dictionary content (default:0) */
* default : 0 when using a CDict, 1 when using a Prefix */
} ZSTD_cParameter; } ZSTD_cParameter;
@ -1007,8 +1002,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
* meaning "return to no-dictionary mode". * meaning "return to no-dictionary mode".
* Note 1 : Dictionary content will be copied internally, * Note 1 : `dict` content will be copied internally,
* except if ZSTD_p_refDictContent is set. * except if ZSTD_p_refDictContent is set before loading.
* Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters. * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
* For this reason, compression parameters cannot be changed anymore after loading a dictionary. * For this reason, compression parameters cannot be changed anymore after loading a dictionary.
* It's also a CPU-heavy operation, with non-negligible impact on latency. * It's also a CPU-heavy operation, with non-negligible impact on latency.
@ -1017,7 +1012,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() : /*! ZSTD_CCtx_refCDict() :
* Ref a prepared dictionary, to be used for all next compression jobs. * Reference a prepared dictionary, to be used for all next compression jobs.
* Note that compression parameters are enforced from within CDict, * Note that compression parameters are enforced from within CDict,
* and supercede any compression parameter previously set within CCtx. * and supercede any compression parameter previously set within CCtx.
* The dictionary will remain valid for future compression jobs using same CCtx. * The dictionary will remain valid for future compression jobs using same CCtx.
@ -1030,16 +1025,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
/*! ZSTD_CCtx_refPrefix() : /*! ZSTD_CCtx_refPrefix() :
* Reference a prefix (content-only dictionary) to bootstrap next compression job. * Reference a prefix (single-usage dictionary) for next compression job.
* Decompression will have to use same prefix. * Decompression need same prefix to properly regenerate data.
* Prefix is only used once. Tables are discarded at end of compression job. * Prefix is **only used once**. Tables are discarded at end of compression job.
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict. * Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
* @result : 0, or an error code (which can be tested with ZSTD_isError()). * @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode". * Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
* Note 1 : Prefix buffer is referenced. It must outlive compression job. * Note 1 : Prefix buffer is referenced. It must outlive compression job.
* Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters. * Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
* It's a CPU-heavy operation, with non-negligible impact on latency. */ * It's a CPU-heavy operation, with non-negligible impact on latency.
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); /* Not ready yet ! <===================================== */ * Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);

66
tests/zstreamtest.c
View File

@ -415,7 +415,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* CDict scenario */ /* CDict scenario */
DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++); DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1); { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict); size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
if (ZSTD_isError(initError)) goto _output_error; if (ZSTD_isError(initError)) goto _output_error;
cSize = 0; cSize = 0;
@ -522,6 +522,55 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
} }
DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
{ size_t const refErr = ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled);
if (ZSTD_isError(refErr)) goto _output_error; }
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
DISPLAYLEVEL(3, "test%3i : decompress with dictionary : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(zd,
decodedBuffer, CNBufferSize,
compressedBuffer, cSize,
dictionary.start, dictionary.filled);
if (ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
DISPLAYLEVEL(3, "OK \n");
}
DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
}
DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compress_generic : ", testNb++);
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
inBuff.src = CNBuffer;
inBuff.size = CNBufferSize;
inBuff.pos = 0;
{ size_t const r = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
if (ZSTD_isError(r)) goto _output_error; }
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
if (ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
DISPLAYLEVEL(3, "OK \n");
}
/* Empty srcSize */ /* Empty srcSize */
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
@ -1266,11 +1315,11 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
} }
/* compression init */ /* compression init */
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) { && oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );
{ int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) ); CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) );
} }
@ -1294,7 +1343,6 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
/* mess with compression parameters */ /* mess with compression parameters */
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* always cancel previous dict, to make user it's possible to pass compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1; cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
@ -1311,12 +1359,15 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) ); if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
/* unconditionally set, to be sync with decoder */ /* unconditionally set, to be sync with decoder */
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
if (dict && dictSize) { if (dict && dictSize) {
/* test that compression parameters are correctly rejected after setting a dictionary */ /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ; size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ;
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed");
} } else {
CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
} }
/* mess with frame parameters */ /* mess with frame parameters */
@ -1380,8 +1431,10 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
/* multi - fragments decompression test */ /* multi - fragments decompression test */
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict);
CHECK_Z( ZSTD_resetDStream(zd) ); CHECK_Z( ZSTD_resetDStream(zd) );
} else { } else {
DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize);
CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
} }
{ size_t decompressionResult = 1; { size_t decompressionResult = 1;
@ -1393,7 +1446,8 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize; inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize; outBuff.size = inBuff.pos + dstBuffSize;
DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes \n", (U32)readCSrcSize); DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
(U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize);
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize); DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);