sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

separated ZSTD_estimateCStreamSize() from ZSTD_estimateCCtxSize()

Browse Source 
for clarity
This commit is contained in:
Yann Collet 2017-05-08 18:24:16 -07:00
parent 51652522a2
commit fa8dadb294
4 changed files with 32 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
doc/zstd_manual.html
View File

@ -1,10 +1,10 @@
<html> <html>
<head> <head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>zstd 1.2.0 Manual</title> <title>zstd 1.3.0 Manual</title>
</head> </head>
<body> <body>
<h1>zstd 1.2.0 Manual</h1> <h1>zstd 1.3.0 Manual</h1>
<hr> <hr>
<a name="Contents"></a><h2>Contents</h2> <a name="Contents"></a><h2>Contents</h2>
<ol> <ol>
@ -376,11 +376,8 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
</b><p> Create a ZSTD compression context using external alloc and free functions </b><p> Create a ZSTD compression context using external alloc and free functions
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming); <pre><b>size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
</b><p> Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. </b><p> Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters.
Set streaming to 1 if the CCtx will be used for streaming (CStream).
Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); <pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@ -519,6 +516,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre> <a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre>
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
</b>/*! ZSTD_estimateCStreamSize() :<b>
* Provides amount of memory needed to allocate ZSTD_CStream with a set of compression parameters.
* Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
* Use ZSTD_estimateCDictSize() to estimate its size, and add for total CStream size */
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); </b>/**< same as ZSTD_sizeof_CCtx */<b> size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); </b>/**< same as ZSTD_sizeof_CCtx */<b>
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b> size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */<b> size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */<b>

22
lib/compress/zstd_compress.c
View File

@ -251,7 +251,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
} }
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming) size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
{ {
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
U32 const divider = (cParams.searchLength==3) ? 3 : 4; U32 const divider = (cParams.searchLength==3) ? 3 : 4;
@ -272,12 +272,7 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned stream
size_t const optSpace = ((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optBudget : 0; size_t const optSpace = ((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optBudget : 0;
size_t const neededSpace = entropySpace + tableSpace + tokenSpace + optSpace; size_t const neededSpace = entropySpace + tableSpace + tokenSpace + optSpace;
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; return sizeof(ZSTD_CCtx) + neededSpace;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
size_t const streamingBudget = inBuffSize + outBuffSize;
size_t const streamingSize = streaming ? streamingBudget : 0;
return sizeof(ZSTD_CCtx) + neededSpace + streamingSize;
} }
@ -2989,7 +2984,7 @@ struct ZSTD_CDict_s {
size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize) size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize)
{ {
cParams = ZSTD_adjustCParams(cParams, 0, dictSize); cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams, 0); return sizeof(ZSTD_CDict) + dictSize + ZSTD_estimateCCtxSize(cParams);
} }
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
@ -3158,6 +3153,17 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
return ZSTD_freeCCtx(zcs); /* same object */ return ZSTD_freeCCtx(zcs); /* same object */
} }
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams)
{
size_t const CCtxSize = ZSTD_estimateCCtxSize(cParams);
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
size_t const streamingSize = inBuffSize + outBuffSize;
return sizeof(ZSTD_CCtx) + CCtxSize + streamingSize;
}
/*====== Initialization ======*/ /*====== Initialization ======*/

12
lib/zstd.h
View File

@ -463,11 +463,8 @@ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_estimateCCtxSize() : /*! ZSTD_estimateCCtxSize() :
* Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. * Provides amount of memory needed to allocate ZSTD_CCtx with a set of compression parameters. */
* Set streaming to 1 if the CCtx will be used for streaming (CStream). ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
* Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
* Use ZSTD_estimateCDictSize() and add this value to estimate total CCtx size */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams, unsigned streaming);
/*! ZSTD_sizeofCCtx() : /*! ZSTD_sizeofCCtx() :
* amount of used memory is variable, depending primarily on compression level */ * amount of used memory is variable, depending primarily on compression level */
@ -609,6 +606,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/ /*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
/*! ZSTD_estimateCStreamSize() :
* Provides amount of memory needed to allocate ZSTD_CStream with a set of compression parameters.
* Special case : when using ZSTD_initCStream_usingDict(), init will transparently create an internal CDict.
* Use ZSTD_estimateCDictSize() to estimate its size, and add for total CStream size */
ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< same as ZSTD_sizeof_CCtx */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); /**< same as ZSTD_sizeof_CCtx */
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8. This result in the creation of an internal CDict */

4
tests/paramgrill.c
View File

@ -388,8 +388,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params, 0/*streaming*/); size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize(params);
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params, 0); size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize(winners[cLevel].params);
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);