Create a ZSTD compression context using external alloc and free functions
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); -Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. - Set streaming to 1 if the CCtx will be used for streaming (CStream). - Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict. - Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size +
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); +Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); @@ -519,6 +516,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; vAdvanced streaming functions
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +/*! ZSTD_estimateCStreamSize() : + * Provides amount of memory needed to allocate ZSTD_CStream with a set of compression parameters. + * Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict. + * Use ZSTD_estimateCDictSize() to estimate its size, and add for total CStream size */ +size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams); size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< same as ZSTD_sizeof_CCtx */ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c5687b21c..37590fca7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -251,7 +251,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u } -size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming) +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.searchLength==3) ? 3 : 4; @@ -272,12 +272,7 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned stream size_t const optSpace = ((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optBudget : 0; size_t const neededSpace = entropySpace + tableSpace + tokenSpace + optSpace; - size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; - size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingBudget = inBuffSize + outBuffSize; - size_t const streamingSize = streaming ? streamingBudget : 0; - - return sizeof(ZSTD_CCtx) + neededSpace + streamingSize; + return sizeof(ZSTD_CCtx) + neededSpace; } @@ -2989,7 +2984,7 @@ struct ZSTD_CDict_s { size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize) { cParams = ZSTD_adjustCParams(cParams, 0, dictSize); - return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0); + return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams); } size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) @@ -3158,6 +3153,17 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) return ZSTD_freeCCtx(zcs); /* same object */ } +size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams) +{ + size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); + size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; + size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; + size_t const streamingSize = inBuffSize + outBuffSize; + + return sizeof(ZSTD_CCtx) + CCtxSize + streamingSize; +} + /*====== Initialization ======*/ diff --git a/lib/zstd.h b/lib/zstd.h index c3a8a52a3..86f1c4405 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -463,11 +463,8 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); /*! ZSTD_estimateCCtxSize() : - * Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. - * Set streaming to 1 if the CCtx will be used for streaming (CStream). - * Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict. - * Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */ -ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); + * Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); /*! ZSTD_sizeofCCtx() : * amount of used memory is variable, depending primarily on compression level */ @@ -609,6 +606,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /*===== Advanced Streaming compression functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +/*! ZSTD_estimateCStreamSize() : + * Provides amount of memory needed to allocate ZSTD_CStream with a set of compression parameters. + * Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict. + * Use ZSTD_estimateCDictSize() to estimate its size, and add for total CStream size */ +ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< same as ZSTD_sizeof_CCtx */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 3cc916e01..1913b54d0 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -388,8 +388,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, 0/*streaming*/); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, 0); + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);