sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-08 00:04:02 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3362 from facebook/compressBound

Browse Source 
check potential overflow of compressBound()
This commit is contained in:
Yann Collet 2022-12-16 14:22:22 -08:00 committed by GitHub
commit 51355e1f70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 137 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lib/compress/zstd_compress.c
View File

@ -59,14 +59,17 @@
* Helper functions * Helper functions
***************************************/ ***************************************/
/* ZSTD_compressBound() /* ZSTD_compressBound()
* Note that the result from this function is only compatible with the "normal" * Note that the result from this function is only valid for
* full-block strategy. * the one-pass compression functions.
* When there are a lot of small blocks due to frequent flush in streaming mode * When employing the streaming mode,
* the overhead of headers can make the compressed data to be larger than the * if flushes are frequently altering the size of blocks,
* return value of ZSTD_compressBound(). * the overhead from block headers can make the compressed data larger
* than the return value of ZSTD_compressBound().
*/ */
size_t ZSTD_compressBound(size_t srcSize) { size_t ZSTD_compressBound(size_t srcSize) {
return ZSTD_COMPRESSBOUND(srcSize); size_t const r = ZSTD_COMPRESSBOUND(srcSize);
if (r==0) return ERROR(srcSize_wrong);
return r;
} }

26
lib/zstd.h
View File

@ -211,8 +211,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
/*====== Helper functions ======*/ /*====== Helper functions ======*/
#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ /* ZSTD_compressBound() :
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ * maximum compressed size in worst case single-pass scenario.
* When invoking `ZSTD_compress()` or any other one-pass compression function,
* it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
* as it eliminates one potential failure scenario,
* aka not enough room in dst buffer to write the compressed frame.
* Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
* In which case, ZSTD_compressBound() will return an error code
* which can be tested using ZSTD_isError().
*
* ZSTD_COMPRESSBOUND() :
* same as ZSTD_compressBound(), but as a macro.
* It can be used to produce constants, which can be useful for static allocation,
* for example to size a static array on stack.
* Will produce constant value 0 if srcSize too large.
*/
#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) > ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
/* ZSTD_isError() :
* Most ZSTD_* functions returning a size_t value can be tested for error,
* using ZSTD_isError().
* @return 1 if error, 0 otherwise
*/
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */

187
tests/fuzzer.c
View File

@ -82,8 +82,8 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
void FUZ_bug976(void); void FUZ_bug976(void);
void FUZ_bug976(void) void FUZ_bug976(void)
{ /* these constants shall not depend on MIN() macro */ { /* these constants shall not depend on MIN() macro */
assert(ZSTD_HASHLOG_MAX < 31); DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
assert(ZSTD_CHAINLOG_MAX < 31); DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
} }
@ -118,23 +118,24 @@ static U32 FUZ_highbit32(U32 v32)
/*============================================= /*=============================================
* Test macros * Test macros
=============================================*/ =============================================*/
#define CHECK_Z(f) { \ #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
size_t const err = f; \
if (ZSTD_isError(err)) { \ #define CHECK_Z(f) { \
DISPLAY("Error => %s : %s ", \ size_t const err = f; \
#f, ZSTD_getErrorName(err)); \ if (ZSTD_isError(err)) { \
exit(1); \ DISPLAY("Error => %s : %s ", \
#f, ZSTD_getErrorName(err)); \
exit(1); \
} } } }
#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; } #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
#define CHECK(fn) { CHECK_NEWV(__err, fn); }
#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
#define CHECK_OP(op, lhs, rhs) { \ #define CHECK_OP(op, lhs, rhs) { \
if (!((lhs) op (rhs))) { \ if (!((lhs) op (rhs))) { \
DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
goto _output_error; \ exit(1); \
} \ } \
} }
#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
@ -338,6 +339,7 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
} }
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
typedef struct { typedef struct {
ZSTD_CCtx* cctx; ZSTD_CCtx* cctx;
ZSTD_threadPool* pool; ZSTD_threadPool* pool;
@ -461,6 +463,28 @@ _output_error:
* Unit tests * Unit tests
=============================================*/ =============================================*/
static void test_compressBound(int tnb)
{
DISPLAYLEVEL(3, "test%3i : compressBound : ", tnb);
/* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
* for a large range of known valid values */
DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
{ int s;
for (s=0; s<30; s++) {
size_t const w = (size_t)1 << s;
CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
} }
// Ensure error if srcSize too big
{ size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
}
DISPLAYLEVEL(3, "OK \n");
}
static int basicUnitTests(U32 const seed, double compressibility) static int basicUnitTests(U32 const seed, double compressibility)
{ {
size_t const CNBuffSize = 5 MB; size_t const CNBuffSize = 5 MB;
@ -507,6 +531,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "%u (OK) \n", vn); DISPLAYLEVEL(3, "%u (OK) \n", vn);
} }
test_compressBound(testNb++);
DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
{ {
ZSTD_compressionParameters params; ZSTD_compressionParameters params;
@ -1579,7 +1605,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
CHECK( ZSTD_compressBegin(staticCCtx, 1) ); CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++); DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
@ -1647,8 +1673,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
testResult = 1; testResult = 1;
goto _end; goto _end;
} }
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) ); CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
@ -1678,9 +1704,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) ); CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
CHECK_VAR(cSize, ZSTD_compress2(mtctx, CHECK_VAR(cSize, ZSTD_compress2(mtctx,
compressedBuffer, compressedBufferSize, compressedBuffer, compressedBufferSize,
CNBuffer, CNBuffSize) ); CNBuffer, CNBuffSize) );
@ -1699,11 +1725,11 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
char out[32]; char out[32];
if (cctx == NULL || dctx == NULL) goto _output_error; if (cctx == NULL || dctx == NULL) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) ); CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize); DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) ); CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
{ char const* outPtr = out; { char const* outPtr = out;
ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 }; ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
ZSTD_outBuffer outBuffer = { NULL, 0, 0 }; ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
@ -1718,9 +1744,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++) DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx(); { ZSTD_CCtx* cctx = ZSTD_createCCtx();
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize); CHECK_Z(cSize);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@ -1728,13 +1754,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx(); { ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t cSize1, cSize2; size_t cSize1, cSize2;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize1); CHECK_Z(cSize1);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize2); CHECK_Z(cSize2);
CHECK_LT(cSize1, cSize2); CHECK_LT(cSize1, cSize2);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
} }
@ -1745,10 +1771,10 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
* ZSTDMT is forced to not take the shortcut. * ZSTDMT is forced to not take the shortcut.
*/ */
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@ -1758,22 +1784,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
int const jobSize = 512 KB; int const jobSize = 512 KB;
int value; int value;
/* Check that the overlap log and job size are unset. */ /* Check that the overlap log and job size are unset. */
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 0); CHECK_EQ(value, 0);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 0); CHECK_EQ(value, 0);
/* Set and check the overlap log and job size. */ /* Set and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) ); CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5); CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, jobSize); CHECK_EQ(value, jobSize);
/* Set the number of workers and check the overlap log and job size. */ /* Set the number of workers and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5); CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, jobSize); CHECK_EQ(value, jobSize);
ZSTD_freeCCtxParams(params); ZSTD_freeCCtxParams(params);
} }
@ -1884,8 +1910,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
@ -1945,8 +1971,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = CNBuffSize / 3; { size_t const testSize = CNBuffSize / 3;
CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) ); CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize) ); (const char*)CNBuffer + dictSize, testSize) );
@ -2780,7 +2806,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const wrongSrcSize = (srcSize + 1000); size_t const wrongSrcSize = (srcSize + 1000);
ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
{ size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
if (!ZSTD_isError(result)) goto _output_error; if (!ZSTD_isError(result)) goto _output_error;
if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
@ -2800,7 +2826,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
CNBuffer, srcSize, compressionLevel); CNBuffer, srcSize, compressionLevel);
if (ZSTD_isError(cSize_1pass)) goto _output_error; if (ZSTD_isError(cSize_1pass)) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
{ size_t const compressionResult = ZSTD_compress2(cctx, { size_t const compressionResult = ZSTD_compress2(cctx,
compressedBuffer, compressedBufferSize, compressedBuffer, compressedBufferSize,
CNBuffer, srcSize); CNBuffer, srcSize);
@ -2819,13 +2845,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx(); { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
assert(cctx != NULL); assert(cctx != NULL);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
{ size_t const compressedSize = ZSTD_compress2(cctx, { size_t const compressedSize = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize), compressedBuffer, ZSTD_compressBound(inputSize),
CNBuffer, inputSize); CNBuffer, inputSize);
CHECK(compressedSize); CHECK_Z(compressedSize);
cSize = compressedSize; cSize = compressedSize;
xxh64 = XXH64(compressedBuffer, compressedSize, 0); xxh64 = XXH64(compressedBuffer, compressedSize, 0);
} }
@ -2835,13 +2861,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx(); { ZSTD_CCtx* cctx = ZSTD_createCCtx();
DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
{ size_t const result = ZSTD_compress2(cctx, { size_t const result = ZSTD_compress2(cctx,
compressedBuffer, ZSTD_compressBound(inputSize), compressedBuffer, ZSTD_compressBound(inputSize),
CNBuffer, inputSize); CNBuffer, inputSize);
CHECK(result); CHECK_Z(result);
if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
@ -2856,7 +2882,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
{ ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
CHECK(bounds.error); CHECK_Z(bounds.error);
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@ -2890,7 +2916,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* basic block compression */ /* basic block compression */
DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@ -2908,7 +2934,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_frameHeader zfh; { ZSTD_frameHeader zfh;
size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
if (zfhrt != 0) goto _output_error; if (zfhrt != 0) goto _output_error;
@ -2930,7 +2956,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* basic block compression */ /* basic block compression */
DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++); DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_inBuffer in = { CNBuffer, 0, 0 }; { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 }; ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
@ -2942,7 +2968,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++); DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
/* one shot */ /* one shot */
{ size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
if (result != 0) goto _output_error; if (result != 0) goto _output_error;
@ -2977,13 +3003,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
int check; int check;
if (ZSTD_isError(bounds.error)) if (ZSTD_isError(bounds.error))
continue; continue;
CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1)); CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound; value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2)); CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check)); CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
if (check != value2) goto _output_error; if (check != value2) goto _output_error;
CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check)); CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
if (check != value1) goto _output_error; if (check != value1) goto _output_error;
} }
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
@ -3000,21 +3026,21 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* basic block compression */ /* basic block compression */
DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, 5) ); CHECK_Z( ZSTD_compressBegin(cctx, 5) );
CHECK( ZSTD_getBlockSize(cctx) >= blockSize); CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) ); CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin(dctx) ); CHECK_Z( ZSTD_decompressBegin(dctx) );
{ CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
if (r != blockSize) goto _output_error; } if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
/* very long stream of block compression */ /* very long stream of block compression */
DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
CHECK( ZSTD_getBlockSize(cctx) >= blockSize); CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
{ U64 const toCompress = 5000000000ULL; /* > 4 GB */ { U64 const toCompress = 5000000000ULL; /* > 4 GB */
U64 compressed = 0; U64 compressed = 0;
while (compressed < toCompress) { while (compressed < toCompress) {
@ -3027,7 +3053,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* dictionary block compression */ /* dictionary block compression */
DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize)); CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */ RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
{ CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */ { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
@ -3038,7 +3064,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
{ CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) ); { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
if (r != blockSize) { if (r != blockSize) {
DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
@ -3057,8 +3083,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
if (cdict==NULL) goto _output_error; if (cdict==NULL) goto _output_error;
CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) ); CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
ZSTD_freeCDict(cdict); ZSTD_freeCDict(cdict);
} }
DISPLAYLEVEL(3, "OK \n"); DISPLAYLEVEL(3, "OK \n");
@ -3227,7 +3253,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
size_t nbSeq = 1; size_t nbSeq = 1;
while (nbSeq <= maxNbSeq) { while (nbSeq <= maxNbSeq) {
CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
/* Check every sequence for the first 100, then skip more rapidly. */ /* Check every sequence for the first 100, then skip more rapidly. */
if (nbSeq < 100) { if (nbSeq < 100) {
++nbSeq; ++nbSeq;
@ -3256,7 +3282,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const bound = ZSTD_compressBound(CNBuffSize); size_t const bound = ZSTD_compressBound(CNBuffSize);
size_t size = 1; size_t size = 1;
while (size <= CNBuffSize) { while (size <= CNBuffSize) {
CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
/* Check every size for the first 100, then skip more rapidly. */ /* Check every size for the first 100, then skip more rapidly. */
if (size < 100) { if (size < 100) {
++size; ++size;
@ -3291,7 +3317,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
void* const outputBuffer = malloc(outputSize); void* const outputBuffer = malloc(outputSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_CCtx* const cctx = ZSTD_createCCtx();
if (!outputBuffer || !cctx) goto _output_error; if (!outputBuffer || !cctx) goto _output_error;
CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
free(outputBuffer); free(outputBuffer);
ZSTD_freeCCtx(cctx); ZSTD_freeCCtx(cctx);
} }
@ -3628,7 +3654,7 @@ static int longUnitTests(U32 const seed, double compressibility)
while (approxIndex <= (maxIndex / 4) * 3) { while (approxIndex <= (maxIndex / 4) * 3) {
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
approxIndex += in.pos; approxIndex += in.pos;
CHECK(in.pos == in.size); CHECK_Z(in.pos == in.size);
in.pos = 0; in.pos = 0;
out.pos = 0; out.pos = 0;
} }
@ -3654,7 +3680,7 @@ static int longUnitTests(U32 const seed, double compressibility)
while (approxIndex <= maxIndex) { while (approxIndex <= maxIndex) {
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
approxIndex += in.pos; approxIndex += in.pos;
CHECK(in.pos == in.size); CHECK_Z(in.pos == in.size);
in.pos = 0; in.pos = 0;
out.pos = 0; out.pos = 0;
} }
@ -3737,7 +3763,7 @@ static int longUnitTests(U32 const seed, double compressibility)
RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
CHECK(cctx_params != NULL); CHECK_Z(cctx_params != NULL);
for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) { for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize); DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
@ -3780,11 +3806,6 @@ _end:
free(compressedBuffer); free(compressedBuffer);
free(decodedBuffer); free(decodedBuffer);
return testResult; return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
} }