sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-08 00:04:02 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1062 from facebook/advancedDecompress

Browse Source 
Advanced decompression API
This commit is contained in:
Yann Collet 2018-03-21 14:52:13 -07:00 committed by GitHub
commit 1be8660eca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 292 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
doc/zstd_manual.html
View File

@ -345,10 +345,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} ZSTD_parameters; } ZSTD_parameters;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {
ZSTD_dm_auto=0, </b>/* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */<b> ZSTD_dct_auto=0, </b>/* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */<b>
ZSTD_dm_rawContent, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b> ZSTD_dct_rawContent, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b>
ZSTD_dm_fullDict </b>/* refuses to load a dictionary if it does not respect Zstandard's specification */<b> ZSTD_dct_fullDict </b>/* refuses to load a dictionary if it does not respect Zstandard's specification */<b>
} ZSTD_dictMode_e; } ZSTD_dictContentType_e;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {
ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b> ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b>
@ -890,7 +890,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); <pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
</b><p> Create an internal CDict from `dict` buffer. </b><p> Create an internal CDict from `dict` buffer.
Decompression will have to use same dictionary. Decompression will have to use same dictionary.
@result : 0, or an error code (which can be tested with ZSTD_isError()). @result : 0, or an error code (which can be tested with ZSTD_isError()).
@ -921,7 +921,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); <pre><b>size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
</b><p> Reference a prefix (single-usage dictionary) for next compression job. </b><p> Reference a prefix (single-usage dictionary) for next compression job.
Decompression need same prefix to properly regenerate data. Decompression need same prefix to properly regenerate data.
Prefix is **only used once**. Tables are discarded at end of compression job. Prefix is **only used once**. Tables are discarded at end of compression job.
@ -1048,9 +1048,9 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR> </p></pre><BR>
<h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR> <h3>Advanced parameters for decompression API</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b> <pre><b>size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); </b>/* not implemented */<b> size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); </b>/* not implemented */<b> size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
</b><p> Create an internal DDict from dict buffer, </b><p> Create an internal DDict from dict buffer,
to be used to decompress next frames. to be used to decompress next frames.
@result : 0, or an error code (which can be tested with ZSTD_isError()). @result : 0, or an error code (which can be tested with ZSTD_isError()).
@ -1079,7 +1079,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); </b>/* not implemented */<b> <pre><b>size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); </b>/* not implemented */<b>
size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); </b>/* not implemented */<b> size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); </b>/* not implemented */<b>
</b><p> Reference a prefix (single-usage dictionary) for next compression job. </b><p> Reference a prefix (single-usage dictionary) for next compression job.
Prefix is **only used once**. It must be explicitly referenced before each frame. Prefix is **only used once**. It must be explicitly referenced before each frame.
If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead. If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_DDict instead.

72
lib/compress/zstd_compress.c
View File

@ -505,7 +505,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
size_t ZSTD_CCtx_loadDictionary_advanced( size_t ZSTD_CCtx_loadDictionary_advanced(
ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode) ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
{ {
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */
@ -519,7 +519,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize); ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced( cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize, dict, dictSize,
dictLoadMethod, dictMode, dictLoadMethod, dictContentType,
cParams, cctx->customMem); cParams, cctx->customMem);
cctx->cdict = cctx->cdictLocal; cctx->cdict = cctx->cdictLocal;
if (cctx->cdictLocal == NULL) if (cctx->cdictLocal == NULL)
@ -532,13 +532,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(
ZSTD_CCtx* cctx, const void* dict, size_t dictSize) ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
{ {
return ZSTD_CCtx_loadDictionary_advanced( return ZSTD_CCtx_loadDictionary_advanced(
cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto); cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
} }
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
{ {
return ZSTD_CCtx_loadDictionary_advanced( return ZSTD_CCtx_loadDictionary_advanced(
cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto); cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
} }
@ -552,17 +552,17 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
{ {
return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dm_rawContent); return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent);
} }
size_t ZSTD_CCtx_refPrefix_advanced( size_t ZSTD_CCtx_refPrefix_advanced(
ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode) ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
{ {
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->cdict = NULL; /* prefix discards any prior cdict */ cctx->cdict = NULL; /* prefix discards any prior cdict */
cctx->prefixDict.dict = prefix; cctx->prefixDict.dict = prefix;
cctx->prefixDict.dictSize = prefixSize; cctx->prefixDict.dictSize = prefixSize;
cctx->prefixDict.dictMode = dictMode; cctx->prefixDict.dictContentType = dictContentType;
return 0; return 0;
} }
@ -2346,7 +2346,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matc
static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms,
ZSTD_CCtx_params const* params, ZSTD_CCtx_params const* params,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
void* workspace) void* workspace)
{ {
DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
@ -2355,15 +2355,15 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZS
ZSTD_reset_compressedBlockState(bs); ZSTD_reset_compressedBlockState(bs);
/* dict restricted modes */ /* dict restricted modes */
if (dictMode==ZSTD_dm_rawContent) if (dictContentType == ZSTD_dct_rawContent)
return ZSTD_loadDictionaryContent(ms, params, dict, dictSize); return ZSTD_loadDictionaryContent(ms, params, dict, dictSize);
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
if (dictMode == ZSTD_dm_auto) { if (dictContentType == ZSTD_dct_auto) {
DEBUGLOG(4, "raw content dictionary detected"); DEBUGLOG(4, "raw content dictionary detected");
return ZSTD_loadDictionaryContent(ms, params, dict, dictSize); return ZSTD_loadDictionaryContent(ms, params, dict, dictSize);
} }
if (dictMode == ZSTD_dm_fullDict) if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_wrong); return ERROR(dictionary_wrong);
assert(0); /* impossible */ assert(0); /* impossible */
} }
@ -2376,7 +2376,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZS
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, U64 pledgedSrcSize, ZSTD_CCtx_params params, U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff) ZSTD_buffered_policy_e zbuff)
@ -2397,7 +2397,7 @@ size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
{ {
size_t const dictID = ZSTD_compress_insertDictionary( size_t const dictID = ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState, cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&params, dict, dictSize, dictMode, cctx->entropyWorkspace); &params, dict, dictSize, dictContentType, cctx->entropyWorkspace);
if (ZSTD_isError(dictID)) return dictID; if (ZSTD_isError(dictID)) return dictID;
assert(dictID <= (size_t)(U32)-1); assert(dictID <= (size_t)(U32)-1);
cctx->dictID = (U32)dictID; cctx->dictID = (U32)dictID;
@ -2407,7 +2407,7 @@ size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
@ -2416,7 +2416,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
return ZSTD_compressBegin_internal(cctx, return ZSTD_compressBegin_internal(cctx,
dict, dictSize, dictMode, dict, dictSize, dictContentType,
cdict, cdict,
params, pledgedSrcSize, params, pledgedSrcSize,
ZSTDb_not_buffered); ZSTDb_not_buffered);
@ -2431,7 +2431,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
ZSTD_CCtx_params const cctxParams = ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
return ZSTD_compressBegin_advanced_internal(cctx, return ZSTD_compressBegin_advanced_internal(cctx,
dict, dictSize, ZSTD_dm_auto, dict, dictSize, ZSTD_dct_auto,
NULL /*cdict*/, NULL /*cdict*/,
cctxParams, pledgedSrcSize); cctxParams, pledgedSrcSize);
} }
@ -2442,7 +2442,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
ZSTD_CCtx_params const cctxParams = ZSTD_CCtx_params const cctxParams =
ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (U32)dictSize); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (U32)dictSize);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, NULL,
cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
} }
@ -2553,7 +2553,7 @@ size_t ZSTD_compress_advanced_internal(
{ {
DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)",
(U32)srcSize); (U32)srcSize);
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, NULL,
params, srcSize, ZSTDb_not_buffered) ); params, srcSize, ZSTDb_not_buffered) );
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
@ -2616,10 +2616,10 @@ static size_t ZSTD_initCDict_internal(
ZSTD_CDict* cdict, ZSTD_CDict* cdict,
const void* dictBuffer, size_t dictSize, const void* dictBuffer, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
DEBUGLOG(3, "ZSTD_initCDict_internal, mode %u", (U32)dictMode); DEBUGLOG(3, "ZSTD_initCDict_internal, dictContentType %u", (U32)dictContentType);
assert(!ZSTD_checkCParams(cParams)); assert(!ZSTD_checkCParams(cParams));
cdict->cParams = cParams; cdict->cParams = cParams;
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
@ -2654,7 +2654,7 @@ static size_t ZSTD_initCDict_internal(
{ size_t const dictID = ZSTD_compress_insertDictionary( { size_t const dictID = ZSTD_compress_insertDictionary(
&cdict->cBlockState, &cdict->matchState, &params, &cdict->cBlockState, &cdict->matchState, &params,
cdict->dictContent, cdict->dictContentSize, cdict->dictContent, cdict->dictContentSize,
dictMode, cdict->workspace); dictContentType, cdict->workspace);
if (ZSTD_isError(dictID)) return dictID; if (ZSTD_isError(dictID)) return dictID;
assert(dictID <= (size_t)(U32)-1); assert(dictID <= (size_t)(U32)-1);
cdict->dictID = (U32)dictID; cdict->dictID = (U32)dictID;
@ -2666,10 +2666,10 @@ static size_t ZSTD_initCDict_internal(
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem) ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{ {
DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (U32)dictContentType);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@ -2686,7 +2686,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
cdict->workspaceSize = workspaceSize; cdict->workspaceSize = workspaceSize;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict, if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dictBuffer, dictSize, dictBuffer, dictSize,
dictLoadMethod, dictMode, dictLoadMethod, dictContentType,
cParams) )) { cParams) )) {
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
return NULL; return NULL;
@ -2700,7 +2700,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
{ {
ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
return ZSTD_createCDict_advanced(dict, dictSize, return ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, ZSTD_dm_auto, ZSTD_dlm_byCopy, ZSTD_dct_auto,
cParams, ZSTD_defaultCMem); cParams, ZSTD_defaultCMem);
} }
@ -2708,7 +2708,7 @@ ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int
{ {
ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
return ZSTD_createCDict_advanced(dict, dictSize, return ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byRef, ZSTD_dm_auto, ZSTD_dlm_byRef, ZSTD_dct_auto,
cParams, ZSTD_defaultCMem); cParams, ZSTD_defaultCMem);
} }
@ -2740,7 +2740,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0);
@ -2765,7 +2765,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
if (ZSTD_isError( ZSTD_initCDict_internal(cdict, if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize, dict, dictSize,
ZSTD_dlm_byRef, dictMode, ZSTD_dlm_byRef, dictContentType,
cParams) )) cParams) ))
return NULL; return NULL;
@ -2799,7 +2799,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
} }
params.fParams = fParams; params.fParams = fParams;
return ZSTD_compressBegin_internal(cctx, return ZSTD_compressBegin_internal(cctx,
NULL, 0, ZSTD_dm_auto, NULL, 0, ZSTD_dct_auto,
cdict, cdict,
params, pledgedSrcSize, params, pledgedSrcSize,
ZSTDb_not_buffered); ZSTDb_not_buffered);
@ -2878,7 +2878,7 @@ size_t ZSTD_CStreamOutSize(void)
} }
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
const void* const dict, size_t const dictSize, ZSTD_dictMode_e const dictMode, const void* const dict, size_t const dictSize, ZSTD_dictContentType_e const dictContentType,
const ZSTD_CDict* const cdict, const ZSTD_CDict* const cdict,
ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize) ZSTD_CCtx_params const params, unsigned long long const pledgedSrcSize)
{ {
@ -2889,7 +2889,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx,
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
CHECK_F( ZSTD_compressBegin_internal(cctx, CHECK_F( ZSTD_compressBegin_internal(cctx,
dict, dictSize, dictMode, dict, dictSize, dictContentType,
cdict, cdict,
params, pledgedSrcSize, params, pledgedSrcSize,
ZSTDb_buffered) ); ZSTDb_buffered) );
@ -2913,7 +2913,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
params.cParams = ZSTD_getCParamsFromCCtxParams(&params, pledgedSrcSize, 0); params.cParams = ZSTD_getCParamsFromCCtxParams(&params, pledgedSrcSize, 0);
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
} }
/*! ZSTD_initCStream_internal() : /*! ZSTD_initCStream_internal() :
@ -2936,7 +2936,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
} }
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, ZSTD_dm_auto, ZSTD_dlm_byCopy, ZSTD_dct_auto,
params.cParams, zcs->customMem); params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@ -2949,7 +2949,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
zcs->cdict = cdict; zcs->cdict = cdict;
} }
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize);
} }
/* ZSTD_initCStream_usingCDict_advanced() : /* ZSTD_initCStream_usingCDict_advanced() :
@ -3227,14 +3227,14 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
CHECK_F( ZSTDMT_initCStream_internal( CHECK_F( ZSTDMT_initCStream_internal(
cctx->mtctx, cctx->mtctx,
prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent, prefixDict.dict, prefixDict.dictSize, ZSTD_dct_rawContent,
cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
cctx->streamStage = zcss_load; cctx->streamStage = zcss_load;
cctx->appliedParams.nbWorkers = params.nbWorkers; cctx->appliedParams.nbWorkers = params.nbWorkers;
} else } else
#endif #endif
{ CHECK_F( ZSTD_resetCStream_internal(cctx, { CHECK_F( ZSTD_resetCStream_internal(cctx,
prefixDict.dict, prefixDict.dictSize, prefixDict.dictMode, prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
cctx->cdict, cctx->cdict,
params, cctx->pledgedSrcSizePlusOne-1) ); params, cctx->pledgedSrcSizePlusOne-1) );
assert(cctx->streamStage == zcss_load); assert(cctx->streamStage == zcss_load);

4
lib/compress/zstd_compress_internal.h
View File

@ -49,7 +49,7 @@ typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
typedef struct ZSTD_prefixDict_s { typedef struct ZSTD_prefixDict_s {
const void* dict; const void* dict;
size_t dictSize; size_t dictSize;
ZSTD_dictMode_e dictMode; ZSTD_dictContentType_e dictContentType;
} ZSTD_prefixDict; } ZSTD_prefixDict;
typedef struct { typedef struct {
@ -671,7 +671,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
* Private use only. To be called from zstdmt_compress.c. */ * Private use only. To be called from zstdmt_compress.c. */
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize); unsigned long long pledgedSrcSize);

16
lib/compress/zstdmt_compress.c
View File

@ -625,7 +625,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
/* init */ /* init */
if (job->cdict) { if (job->cdict) {
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dm_auto, job->cdict, jobParams, job->fullFrameSize); size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, job->cdict, jobParams, job->fullFrameSize);
assert(job->firstJob); /* only allowed for first job */ assert(job->firstJob); /* only allowed for first job */
if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
} else { /* srcStart points at reloaded section */ } else { /* srcStart points at reloaded section */
@ -636,7 +636,7 @@ void ZSTDMT_compressionJob(void* jobDescription)
goto _endJob; goto _endJob;
} } } }
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
job->prefix.start, job->prefix.size, ZSTD_dm_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
NULL, /*cdict*/ NULL, /*cdict*/
jobParams, pledgedSrcSize); jobParams, pledgedSrcSize);
if (ZSTD_isError(initError)) { if (ZSTD_isError(initError)) {
@ -1217,7 +1217,7 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
size_t ZSTDMT_initCStream_internal( size_t ZSTDMT_initCStream_internal(
ZSTDMT_CCtx* mtctx, ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict, ZSTD_CCtx_params params, const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
{ {
@ -1257,7 +1257,7 @@ size_t ZSTDMT_initCStream_internal(
if (dict) { if (dict) {
ZSTD_freeCDict(mtctx->cdictLocal); ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */ ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */
params.cParams, mtctx->cMem); params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal; mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
@ -1326,7 +1326,7 @@ size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
DEBUGLOG(4, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize); DEBUGLOG(4, "ZSTDMT_initCStream_advanced (pledgedSrcSize=%u)", (U32)pledgedSrcSize);
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dm_auto, NULL, return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dct_auto, NULL,
cctxParams, pledgedSrcSize); cctxParams, pledgedSrcSize);
} }
@ -1339,7 +1339,7 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */ if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */
cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict); cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict);
cctxParams.fParams = fParams; cctxParams.fParams = fParams;
return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dm_auto, cdict, return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dct_auto, cdict,
cctxParams, pledgedSrcSize); cctxParams, pledgedSrcSize);
} }
@ -1351,7 +1351,7 @@ size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize) size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize)
{ {
if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; if (!pledgedSrcSize) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dm_auto, 0, mtctx->params, return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, 0, mtctx->params,
pledgedSrcSize); pledgedSrcSize);
} }
@ -1361,7 +1361,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) {
DEBUGLOG(4, "ZSTDMT_initCStream (cLevel=%i)", compressionLevel); DEBUGLOG(4, "ZSTDMT_initCStream (cLevel=%i)", compressionLevel);
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN); return ZSTDMT_initCStream_internal(mtctx, NULL, 0, ZSTD_dct_auto, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN);
} }

2
lib/compress/zstdmt_compress.h
View File

@ -144,7 +144,7 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
* must receive dict, or cdict, or none, but not both. * must receive dict, or cdict, or none, but not both.
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);

174
lib/decompress/zstd_decompress.c
View File

@ -14,8 +14,9 @@
*****************************************************************/ *****************************************************************/
/*! /*!
* HEAPMODE : * HEAPMODE :
* Select how default decompression function ZSTD_decompress() will allocate memory, * Select how default decompression function ZSTD_decompress() allocates its context,
* in memory stack (0), or in memory heap (1, requires malloc()) * on stack (0), or into heap (1, default; requires malloc()).
* Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
*/ */
#ifndef ZSTD_HEAPMODE #ifndef ZSTD_HEAPMODE
# define ZSTD_HEAPMODE 1 # define ZSTD_HEAPMODE 1
@ -31,8 +32,9 @@
/*! /*!
* MAXWINDOWSIZE_DEFAULT : * MAXWINDOWSIZE_DEFAULT :
* maximum window size accepted by DStream, by default. * maximum window size accepted by DStream __by default__.
* Frames requiring more memory will be rejected. * Frames requiring more memory will be rejected.
* It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
*/ */
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1) # define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
@ -2335,13 +2337,23 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddi
return 0; return 0;
} }
static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict) static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e dictContentType)
{ {
ddict->dictID = 0; ddict->dictID = 0;
ddict->entropyPresent = 0; ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0; if (dictContentType == ZSTD_dct_rawContent) return 0;
if (ddict->dictSize < 8) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
{ U32 const magic = MEM_readLE32(ddict->dictContent); { U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */ if (magic != ZSTD_MAGIC_DICTIONARY) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
} }
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize); ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
@ -2352,7 +2364,10 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
} }
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{ {
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL; ddict->dictBuffer = NULL;
@ -2368,12 +2383,15 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */ /* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_inDDict(ddict) ); CHECK_F( ZSTD_loadEntropy_inDDict(ddict, dictContentType) );
return 0; return 0;
} }
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_customMem customMem) ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
@ -2381,7 +2399,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_di
if (!ddict) return NULL; if (!ddict) return NULL;
ddict->cMem = customMem; ddict->cMem = customMem;
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod) )) { if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod, dictContentType) )) {
ZSTD_freeDDict(ddict); ZSTD_freeDDict(ddict);
return NULL; return NULL;
} }
@ -2397,7 +2415,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_di
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{ {
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, allocator); return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
} }
/*! ZSTD_createDDict_byReference() : /*! ZSTD_createDDict_byReference() :
@ -2407,14 +2425,15 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{ {
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, allocator); return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
} }
const ZSTD_DDict* ZSTD_initStaticDDict( const ZSTD_DDict* ZSTD_initStaticDDict(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod) ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{ {
size_t const neededSpace = size_t const neededSpace =
sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
@ -2427,7 +2446,7 @@ const ZSTD_DDict* ZSTD_initStaticDDict(
memcpy(ddict+1, dict, dictSize); /* local copy */ memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1; dict = ddict+1;
} }
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef) )) if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef, dictContentType) ))
return NULL; return NULL;
return ddict; return ddict;
} }
@ -2546,19 +2565,49 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
{
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
ZSTD_freeDDict(dctx->ddictLocal);
if (dict && dictSize >= 8) {
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
if (dctx->ddictLocal == NULL) return ERROR(memory_allocation);
} else {
dctx->ddictLocal = NULL;
}
dctx->ddict = dctx->ddictLocal;
return 0;
}
size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
}
size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
}
size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
{
return ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType);
}
size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
{
return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);
}
/* ZSTD_initDStream_usingDict() :
* return : expected size, aka ZSTD_frameHeaderSize_prefix.
* this function cannot fail */
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
{ {
DEBUGLOG(4, "ZSTD_initDStream_usingDict"); DEBUGLOG(4, "ZSTD_initDStream_usingDict");
zds->streamStage = zdss_loadHeader; zds->streamStage = zdss_init;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) );
ZSTD_freeDDict(zds->ddictLocal);
if (dict && dictSize >= 8) {
zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
} else zds->ddictLocal = NULL;
zds->ddict = zds->ddictLocal;
zds->legacyVersion = 0;
zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix; return ZSTD_frameHeaderSize_prefix;
} }
@ -2569,38 +2618,46 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0); return ZSTD_initDStream_usingDict(zds, NULL, 0);
} }
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
dctx->ddict = ddict;
return 0;
}
/* ZSTD_initDStream_usingDDict() : /* ZSTD_initDStream_usingDDict() :
* ddict will just be referenced, and must outlive decompression session * ddict will just be referenced, and must outlive decompression session
* this function cannot fail */ * this function cannot fail */
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
{ {
size_t const initResult = ZSTD_initDStream(zds); size_t const initResult = ZSTD_initDStream(dctx);
zds->ddict = ddict; dctx->ddict = ddict;
return initResult; return initResult;
} }
size_t ZSTD_resetDStream(ZSTD_DStream* zds) /* ZSTD_resetDStream() :
* return : expected size, aka ZSTD_frameHeaderSize_prefix.
* this function cannot fail */
size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
{ {
DEBUGLOG(4, "ZSTD_resetDStream"); DEBUGLOG(4, "ZSTD_resetDStream");
zds->streamStage = zdss_loadHeader; dctx->streamStage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0;
zds->legacyVersion = 0; dctx->legacyVersion = 0;
zds->hostageByte = 0; dctx->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix; return ZSTD_frameHeaderSize_prefix;
} }
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_setDStreamParameter(ZSTD_DStream* dctx,
ZSTD_DStreamParameter_e paramType, unsigned paramValue) ZSTD_DStreamParameter_e paramType, unsigned paramValue)
{ {
ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init); if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
if ((unsigned)zds->streamStage > (unsigned)zdss_loadHeader)
return ERROR(stage_wrong);
switch(paramType) switch(paramType)
{ {
default : return ERROR(parameter_unsupported); default : return ERROR(parameter_unsupported);
case DStream_p_maxWindowSize : case DStream_p_maxWindowSize :
DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10); DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10);
zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); dctx->maxWindowSize = paramValue ? paramValue : (U32)(-1);
break; break;
} }
return 0; return 0;
@ -2608,9 +2665,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
{ {
ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init); if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader)
return ERROR(stage_wrong);
dctx->maxWindowSize = maxWindowSize; dctx->maxWindowSize = maxWindowSize;
return 0; return 0;
} }
@ -2618,17 +2673,15 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
{ {
DEBUGLOG(4, "ZSTD_DCtx_setFormat : %u", (unsigned)format); DEBUGLOG(4, "ZSTD_DCtx_setFormat : %u", (unsigned)format);
ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init); if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader)
return ERROR(stage_wrong);
dctx->format = format; dctx->format = format;
return 0; return 0;
} }
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
{ {
return ZSTD_sizeof_DCtx(zds); return ZSTD_sizeof_DCtx(dctx);
} }
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
@ -2695,23 +2748,25 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
if (zds->legacyVersion) {
/* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation);
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
}
#endif
while (someMoreWork) { while (someMoreWork) {
switch(zds->streamStage) switch(zds->streamStage)
{ {
case zdss_init : case zdss_init :
DEBUGLOG(5, "stage zdss_init => transparent reset ");
ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */
/* fall-through */ /* fall-through */
case zdss_loadHeader : case zdss_loadHeader :
DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
if (zds->legacyVersion) {
/* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation);
{ size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
if (hint==0) zds->streamStage = zdss_init;
return hint;
} }
#endif
{ size_t const hSize = ZSTD_getFrameHeader_internal(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); { size_t const hSize = ZSTD_getFrameHeader_internal(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
DEBUGLOG(5, "header size : %u", (U32)hSize); DEBUGLOG(5, "header size : %u", (U32)hSize);
if (ZSTD_isError(hSize)) { if (ZSTD_isError(hSize)) {
@ -2720,14 +2775,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (legacyVersion) { if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL; const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
/* legacy support is incompatible with static dctx */ /* legacy support is incompatible with static dctx */
if (zds->staticSize) return ERROR(memory_allocation); if (zds->staticSize) return ERROR(memory_allocation);
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext,
zds->previousLegacyVersion, legacyVersion, zds->previousLegacyVersion, legacyVersion,
dict, dictSize)); dict, dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
} if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
return hint;
} }
#endif #endif
return hSize; /* error */ return hSize; /* error */
} }
@ -2837,6 +2895,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (ip==iend) { someMoreWork = 0; break; } /* no more input */ if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->streamStage = zdss_load; zds->streamStage = zdss_load;
/* fall-through */ /* fall-through */
case zdss_load: case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
size_t const toLoad = neededInSize - zds->inPos; size_t const toLoad = neededInSize - zds->inPos;
@ -2863,6 +2922,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} } } }
zds->streamStage = zdss_flush; zds->streamStage = zdss_flush;
/* fall-through */ /* fall-through */
case zdss_flush: case zdss_flush:
{ size_t const toFlushSize = zds->outEnd - zds->outStart; { size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
@ -2909,8 +2969,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
return 1; return 1;
} /* nextSrcSizeHint==0 */ } /* nextSrcSizeHint==0 */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ assert(zds->inPos <= nextSrcSizeHint);
nextSrcSizeHint -= zds->inPos; /* already loaded*/ nextSrcSizeHint -= zds->inPos; /* part already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;
} }
} }

2
lib/legacy/zstd_legacy.h
View File

@ -246,6 +246,7 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion,
const void* dict, size_t dictSize) const void* dict, size_t dictSize)
{ {
DEBUGLOG(5, "ZSTD_initLegacyStream for v0.%u", newVersion);
if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion);
switch(newVersion) switch(newVersion)
{ {
@ -304,6 +305,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
ZSTD_outBuffer* output, ZSTD_inBuffer* input) ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{ {
DEBUGLOG(5, "ZSTD_decompressLegacyStream for v0.%u", version);
switch(version) switch(version)
{ {
default : default :

158
lib/legacy/zstd_v04.c
View File

@ -16,37 +16,6 @@
/* ****************************************************************** /* ******************************************************************
mem.h mem.h
low-level memory access routines
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */ ****************************************************************** */
#ifndef MEM_H_MODULE #ifndef MEM_H_MODULE
#define MEM_H_MODULE #define MEM_H_MODULE
@ -103,6 +72,44 @@ extern "C" {
#endif #endif
/*-*************************************
* Debug
***************************************/
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
# include <assert.h>
#else
# ifndef assert
# define assert(condition) ((void)0)
# endif
#endif
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
# include <stdio.h>
extern int g_debuglog_enable;
/* recommended values for ZSTD_DEBUG display levels :
* 1 : no display, enables assert() only
* 2 : reserved for currently active debug path
* 3 : events once per object lifetime (CCtx, CDict, etc.)
* 4 : events once per frame
* 5 : events once per block
* 6 : events once per sequence (*very* verbose) */
# define RAWLOG(l, ...) { \
if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) { \
fprintf(stderr, __VA_ARGS__); \
} }
# define DEBUGLOG(l, ...) { \
if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) { \
fprintf(stderr, __FILE__ ": " __VA_ARGS__); \
fprintf(stderr, " \n"); \
} }
#else
# define RAWLOG(l, ...) {} /* disabled */
# define DEBUGLOG(l, ...) {} /* disabled */
#endif
/**************************************************************** /****************************************************************
* Memory I/O * Memory I/O
*****************************************************************/ *****************************************************************/
@ -255,34 +262,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
/* /*
zstd - standard compression library zstd - standard compression library
Header File for static linking only Header File for static linking only
Copyright (C) 2014-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/ */
#ifndef ZSTD_STATIC_H #ifndef ZSTD_STATIC_H
#define ZSTD_STATIC_H #define ZSTD_STATIC_H
@ -392,34 +371,6 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstS
/* /*
zstd_internal - common functions to include zstd_internal - common functions to include
Header File for include Header File for include
Copyright (C) 2014-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/ */
#ifndef ZSTD_CCOMMON_H_MODULE #ifndef ZSTD_CCOMMON_H_MODULE
#define ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE
@ -507,36 +458,6 @@ static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
/* ****************************************************************** /* ******************************************************************
FSE : Finite State Entropy coder FSE : Finite State Entropy coder
header file header file
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */ ****************************************************************** */
#ifndef FSE_H #ifndef FSE_H
#define FSE_H #define FSE_H
@ -3528,12 +3449,14 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
char* const oend = ostart + *maxDstSizePtr; char* const oend = ostart + *maxDstSizePtr;
U32 notDone = 1; U32 notDone = 1;
DEBUGLOG(5, "ZBUFF_decompressContinue");
while (notDone) while (notDone)
{ {
switch(zbc->stage) switch(zbc->stage)
{ {
case ZBUFFds_init : case ZBUFFds_init :
DEBUGLOG(5, "ZBUFF_decompressContinue: stage==ZBUFFds_init => ERROR(init_missing)");
return ERROR(init_missing); return ERROR(init_missing);
case ZBUFFds_readHeader : case ZBUFFds_readHeader :
@ -3741,6 +3664,7 @@ size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* src, s
size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr) size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
{ {
DEBUGLOG(5, "ZBUFFv04_decompressContinue");
return ZBUFF_decompressContinue(dctx, dst, maxDstSizePtr, src, srcSizePtr); return ZBUFF_decompressContinue(dctx, dst, maxDstSizePtr, src, srcSizePtr);
} }

32
lib/zstd.h
View File

@ -433,10 +433,10 @@ typedef struct {
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
typedef enum { typedef enum {
ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ ZSTD_dct_auto=0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ ZSTD_dct_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */ ZSTD_dct_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
} ZSTD_dictMode_e; } ZSTD_dictContentType_e;
typedef enum { typedef enum {
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
@ -574,13 +574,14 @@ ZSTDLIB_API const ZSTD_CDict* ZSTD_initStaticCDict(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams); ZSTD_compressionParameters cParams);
ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict(
void* workspace, size_t workspaceSize, void* workspace, size_t workspaceSize,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod); ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType);
/*! Custom memory allocation : /*! Custom memory allocation :
* These prototypes make it possible to pass your own allocation/free functions. * These prototypes make it possible to pass your own allocation/free functions.
@ -599,12 +600,13 @@ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictMode_e dictMode, ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams, ZSTD_compressionParameters cParams,
ZSTD_customMem customMem); ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_customMem customMem); ZSTD_customMem customMem);
@ -1096,7 +1098,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
* to precisely select how dictionary content must be interpreted. */ * to precisely select how dictionary content must be interpreted. */
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_CCtx_refCDict() : /*! ZSTD_CCtx_refCDict() :
@ -1125,7 +1127,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */ * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_CCtx_reset() : /*! ZSTD_CCtx_reset() :
* Return a CCtx to clean state. * Return a CCtx to clean state.
@ -1266,9 +1268,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
* how dictionary content will be interpreted and loaded. * how dictionary content will be interpreted and loaded.
*/ */
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_DCtx_refDDict() : /*! ZSTD_DCtx_refDDict() :
@ -1280,7 +1282,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void
* Special : adding a NULL DDict means "return to no-dictionary mode". * Special : adding a NULL DDict means "return to no-dictionary mode".
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
*/ */
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
/*! ZSTD_DCtx_refPrefix() : /*! ZSTD_DCtx_refPrefix() :
@ -1294,8 +1296,8 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
*/ */
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); /* not implemented */ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
/*! ZSTD_DCtx_setMaxWindowSize() : /*! ZSTD_DCtx_setMaxWindowSize() :

3
programs/platform.h
View File

@ -106,8 +106,7 @@ extern "C" {
# include <io.h> /* _isatty */ # include <io.h> /* _isatty */
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <stdio.h> /* FILE */ # include <stdio.h> /* FILE */
static __inline int IS_CONSOLE(FILE* stdStream) static __inline int IS_CONSOLE(FILE* stdStream) {
{
DWORD dummy; DWORD dummy;
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
} }

4
tests/Makefile
View File

@ -100,11 +100,11 @@ zstd-nolegacy:
gzstd: gzstd:
$(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" $(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)"
.PHONY: .PHONY: zstd-dll
zstd-dll : zstd-dll :
$(MAKE) -C $(ZSTDDIR) libzstd $(MAKE) -C $(ZSTDDIR) libzstd
.PHONY: .PHONY: zstd-staticLib
zstd-staticLib : zstd-staticLib :
$(MAKE) -C $(ZSTDDIR) libzstd.a $(MAKE) -C $(ZSTDDIR) libzstd.a

18
tests/fuzzer.c
View File

@ -649,7 +649,7 @@ static int basicUnitTests(U32 seed, double compressibility)
{ size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
void* ddictBuffer = malloc(ddictBufferSize); void* ddictBuffer = malloc(ddictBufferSize);
if (ddictBuffer == NULL) goto _output_error; if (ddictBuffer == NULL) goto _output_error;
{ const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy); { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
if (r != CNBuffSize - dictSize) goto _output_error; if (r != CNBuffSize - dictSize) goto _output_error;
} }
@ -753,7 +753,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
ZSTD_dlm_byRef, ZSTD_dm_auto, ZSTD_dlm_byRef, ZSTD_dct_auto,
cParams, ZSTD_defaultCMem); cParams, ZSTD_defaultCMem);
DISPLAYLEVEL(3, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict)); DISPLAYLEVEL(3, "(size : %u) : ", (U32)ZSTD_sizeof_CDict(cdict));
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
@ -788,7 +788,7 @@ static int basicUnitTests(U32 seed, double compressibility)
{ const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
cdictBuffer, cdictSize, cdictBuffer, cdictSize,
dictBuffer, dictSize, dictBuffer, dictSize,
ZSTD_dlm_byCopy, ZSTD_dm_auto, ZSTD_dlm_byCopy, ZSTD_dct_auto,
cParams); cParams);
if (cdict == NULL) { if (cdict == NULL) {
DISPLAY("ZSTD_initStaticCDict failed "); DISPLAY("ZSTD_initStaticCDict failed ");
@ -808,7 +808,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
{ ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize, cdict, fParams); CNBuffer, CNBuffSize, cdict, fParams);
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
@ -859,7 +859,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++); DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict==NULL) goto _output_error; if (cdict==NULL) goto _output_error;
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
} }
@ -867,7 +867,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++); DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
if (cdict!=NULL) goto _output_error; if (cdict!=NULL) goto _output_error;
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
} }
@ -878,7 +878,7 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t ret; size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
ret = ZSTD_CCtx_loadDictionary_advanced( ret = ZSTD_CCtx_loadDictionary_advanced(
cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dm_auto); cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto);
if (!ZSTD_isError(ret)) goto _output_error; if (!ZSTD_isError(ret)) goto _output_error;
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@ -888,7 +888,7 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t ret; size_t ret;
MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
ret = ZSTD_CCtx_loadDictionary_advanced( ret = ZSTD_CCtx_loadDictionary_advanced(
cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dm_rawContent); cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
if (ZSTD_isError(ret)) goto _output_error; if (ZSTD_isError(ret)) goto _output_error;
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@ -924,7 +924,7 @@ static int basicUnitTests(U32 seed, double compressibility)
BYTE data[1024]; BYTE data[1024];
ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
ZSTD_dlm_byRef, ZSTD_dm_auto, ZSTD_dlm_byRef, ZSTD_dct_auto,
cParams, ZSTD_defaultCMem); cParams, ZSTD_defaultCMem);
memset(data, 'x', sizeof(data)); memset(data, 'x', sizeof(data));
cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,

25
tests/legacy.c
View File

@ -94,21 +94,19 @@ int testStreamingAPI(void)
if (needsInit) { if (needsInit) {
size_t const ret = ZSTD_initDStream(stream); size_t const ret = ZSTD_initDStream(stream);
if (ZSTD_isError(ret)) { if (ZSTD_isError(ret)) {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); DISPLAY("ERROR: ZSTD_initDStream: %s\n", ZSTD_getErrorName(ret));
return 1; return 1;
} } }
}
{ { size_t const ret = ZSTD_decompressStream(stream, &output, &input);
size_t const ret = ZSTD_decompressStream(stream, &output, &input);
if (ZSTD_isError(ret)) { if (ZSTD_isError(ret)) {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); DISPLAY("ERROR: ZSTD_decompressStream: %s\n", ZSTD_getErrorName(ret));
return 1; return 1;
} }
if (ret == 0) { if (ret == 0) {
needsInit = 1; needsInit = 1;
} } }
}
if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n"); DISPLAY("ERROR: Wrong decoded output produced\n");
@ -128,15 +126,12 @@ int testStreamingAPI(void)
int main(void) int main(void)
{ {
int ret; { int const ret = testSimpleAPI();
if (ret) return ret; }
ret = testSimpleAPI(); { int const ret = testStreamingAPI();
if (ret) return ret; if (ret) return ret; }
ret = testStreamingAPI();
if (ret) return ret;
DISPLAY("OK\n"); DISPLAY("OK\n");
return 0; return 0;
} }

47
tests/zstreamtest.c
View File

@ -239,7 +239,7 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Create dictionary */ /* Create dictionary */
DISPLAYLEVEL(3, "creating dictionary for unit tests \n"); DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 2, 8 KB, 40 KB); dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
if (!dictionary.start) { if (!dictionary.start) {
DISPLAY("Error creating dictionary, aborting \n"); DISPLAY("Error creating dictionary, aborting \n");
goto _output_error; goto _output_error;
@ -576,17 +576,20 @@ static int basicUnitTests(U32 seed, double compressibility)
{ size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
ZSTD_DCtx_reset(zd); /* leave zd in good shape for next tests */
DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++); DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
{ ZSTD_DCtx* const dctx = ZSTD_createDCtx(); { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); int const maxLevel = 16; /* first level with zstd_opt */
int const maxLevel = ZSTD_maxCLevel();
int level; int level;
assert(maxLevel < ZSTD_maxCLevel());
CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
for (level = 1; level <= maxLevel; ++level) { for (level = 1; level <= maxLevel; ++level) {
ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level); ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
int const maxSize = MIN(1 MB, CNBufferSize); size_t const maxSize = MIN(1 MB, CNBufferSize);
int size; size_t size;
for (size = 512; size <= maxSize; size <<= 1) { for (size = 512; size <= maxSize; size <<= 1) {
U64 const crcOrig = XXH64(CNBuffer, size, 0);
ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_CCtx* const cctx = ZSTD_createCCtx();
outBuff.dst = compressedBuffer; outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize; outBuff.size = compressedBufferSize;
@ -597,12 +600,18 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end)); CHECK_Z(ZSTD_compress_generic(cctx, &outBuff, &inBuff, ZSTD_e_end));
if (inBuff.pos != inBuff.size) goto _output_error; if (inBuff.pos != inBuff.size) goto _output_error;
CHECK_Z(ZSTD_decompress_usingDDict(dctx, decodedBuffer, size, outBuff.dst, outBuff.pos, ddict)); { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
CHECK_Z( ZSTD_decompress_generic(dctx, &decOut, &decIn) );
if (decIn.pos != decIn.size) goto _output_error;
if (decOut.pos != size) goto _output_error;
{ U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
if (crcDec != crcOrig) goto _output_error;
} }
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
} }
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
} }
ZSTD_freeDDict(ddict);
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
} }
DISPLAYLEVEL(3, "OK\n"); DISPLAYLEVEL(3, "OK\n");
@ -610,7 +619,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++); DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */}; ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_auto, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize); size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
if (ZSTD_isError(initError)) goto _output_error; if (ZSTD_isError(initError)) goto _output_error;
cSize = 0; cSize = 0;
@ -654,14 +663,18 @@ static int basicUnitTests(U32 seed, double compressibility)
cSize = outBuff.pos; cSize = outBuff.pos;
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100); DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
DISPLAYLEVEL(3, "test%3i : decompress with dictionary : ", testNb++); DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
{ size_t const r = ZSTD_decompress_usingDict(zd, CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
decodedBuffer, CNBufferSize, outBuff.dst = decodedBuffer;
compressedBuffer, cSize, outBuff.size = CNBufferSize;
dictionary.start, dictionary.filled); outBuff.pos = 0;
if (ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ inBuff.src = compressedBuffer;
inBuff.size = cSize;
inBuff.pos = 0;
CHECK_Z( ZSTD_decompress_generic(zd, &outBuff, &inBuff) );
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
}
DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++); DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
@ -776,7 +789,7 @@ static int basicUnitTests(U32 seed, double compressibility)
inBuff.pos = 0; inBuff.pos = 0;
} }
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */ { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionnally lies on estimatedSrcSize, to push cdict into targeting a small window size */
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog); DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) ); CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) ); CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end) );
@ -819,7 +832,7 @@ static int basicUnitTests(U32 seed, double compressibility)
XXH64_reset(&xxh, 0); XXH64_reset(&xxh, 0);
cParams.windowLog = kMaxWindowLog; cParams.windowLog = kMaxWindowLog;
cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dm_fullDict, cParams, ZSTD_defaultCMem); cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
if (!cdict || !ddict) goto _output_error; if (!cdict || !ddict) goto _output_error;

2
zlibWrapper/examples/zwrapbench.c
View File

@ -233,7 +233,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
if (compressor == BMK_ZSTD) { if (compressor == BMK_ZSTD) {
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
ZSTD_customMem const cmem = { NULL, NULL, NULL }; ZSTD_customMem const cmem = { NULL, NULL, NULL };
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem); ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_auto, zparams.cParams, cmem);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
do { do {