sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

minor code refactor

Browse Source
This commit is contained in:
Yann Collet 2016-03-19 12:12:07 +01:00
parent f3120413cc
commit d1d210f3fb
5 changed files with 109 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
lib/bitstream.h
View File

@ -56,10 +56,9 @@ extern "C" {
/*-****************************************** /*-******************************************
* bitStream encoding API (write forward) * bitStream encoding API (write forward)
********************************************/ ********************************************/
/*! /* bitStream can mix input from multiple sources.
* bitStream can mix input from multiple sources. * A critical property of these streams is that they encode and decode in **reverse** direction.
* A critical property of these streams is that they encode and decode in **reverse** direction. * So the first bit sequence you add will be the last to be read, like a LIFO stack.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/ */
typedef struct typedef struct
{ {
@ -75,10 +74,9 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
/*! /*Start by initCStream, providing the size of buffer to write into.
* Start by initCStream, providing the size of buffer to write into.
* bitStream will never write outside of this buffer. * bitStream will never write outside of this buffer.
* @dstCapacity must be >= sizeof(size_t), otherwise @return will be an error code. * `dstCapacity` must be >= sizeof(size_t), otherwise @return will be an error code.
* *
* bits are first added to a local register. * bits are first added to a local register.
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
@ -90,7 +88,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
* *
* Last operation is to close the bitStream. * Last operation is to close the bitStream.
* The function returns the final size of CStream in bytes. * The function returns the final size of CStream in bytes.
* If data couldn't fit into @dstBuffer, it will return a 0 ( == not storable) * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
*/ */
@ -117,8 +115,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
/*! /*Start by invoking BIT_initDStream().
* Start by invoking BIT_initDStream().
* A chunk of the bitStream is then stored into a local register. * A chunk of the bitStream is then stored into a local register.
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
* You can then retrieve bitFields stored into the local register, **in reverse order**. * You can then retrieve bitFields stored into the local register, **in reverse order**.
@ -190,7 +187,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
bitC->bitPos += nbBits; bitC->bitPos += nbBits;
} }
/*! BIT_addBitsFast /*! BIT_addBitsFast() :
* works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
{ {
@ -198,7 +195,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBi
bitC->bitPos += nbBits; bitC->bitPos += nbBits;
} }
/*! BIT_flushBitsFast /*! BIT_flushBitsFast() :
* unsafe version; does not check buffer overflow */ * unsafe version; does not check buffer overflow */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
{ {
@ -219,8 +216,8 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
} }
/*! BIT_closeCStream /*! BIT_closeCStream() :
* @result : size of CStream, in bytes, or 0 if it cannot fit into dstBuffer */ * @return : size of CStream, in bytes, or 0 if it cannot fit into dstBuffer */
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
{ {
char* endPtr; char* endPtr;
@ -241,12 +238,12 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
/*-******************************************************** /*-********************************************************
* bitStream decoding * bitStream decoding
**********************************************************/ **********************************************************/
/*!BIT_initDStream /*!BIT_initDStream() :
* Initialize a BIT_DStream_t. * Initialize a BIT_DStream_t.
* @bitD : a pointer to an already allocated BIT_DStream_t structure * `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* @srcBuffer must point at the beginning of a bitStream * `srcBuffer` must point at the beginning of a bitStream.
* @srcSize must be the exact size of the bitStream * `srcSize` must be the exact size of the bitStream.
* @result : size of stream (== srcSize) or an errorCode if a problem is detected * @return : size of stream (== srcSize) or an errorCode if a problem is detected
*/ */
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
{ {
@ -284,24 +281,24 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
return srcSize; return srcSize;
} }
/*!BIT_lookBits /*!BIT_lookBits() :
* Provides next n bits from local register * Provides next n bits from local register.
* local register is not modified (bits are still present for next read/look) * local register is not modified (bits are still present for next read/look).
* On 32-bits, maxNbBits==25 * On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==57 * On 64-bits, maxNbBits==56.
* @return : value extracted * @return : value extracted
*/ */
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
{ {
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
} }
/*! BIT_lookBitsFast : /*! BIT_lookBitsFast*() :
* unsafe version; only works only if nbBits >= 1 */ * unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
{ {
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
} }
@ -310,7 +307,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
bitD->bitsConsumed += nbBits; bitD->bitsConsumed += nbBits;
} }
/*!BIT_readBits /*!BIT_readBits() :
* Read next n bits from local register. * Read next n bits from local register.
* pay attention to not read more than nbBits contained into local register. * pay attention to not read more than nbBits contained into local register.
* @return : extracted value. * @return : extracted value.
@ -322,7 +319,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
return value; return value;
} }
/*!BIT_readBitsFast : /*!BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */ * unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
{ {
@ -360,7 +357,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
} }
} }
/*! BIT_endOfDStream /*! BIT_endOfDStream() :
* @return Tells if DStream has reached its exact end * @return Tells if DStream has reached its exact end
*/ */
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)

2
lib/fse.c
View File

@ -466,7 +466,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
bitStream >>= 2; bitStream >>= 2;
} }
{ {
const short max = (short)((2*threshold-1)-remaining); short const max = (short)((2*threshold-1)-remaining);
short count; short count;
if ((bitStream & (threshold-1)) < (U32)max) { if ((bitStream & (threshold-1)) < (U32)max) {

24
lib/zstd_decompress.c
View File

@ -515,11 +515,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog, static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
U32 max = (1<<rawBits)-1;
switch(type) switch(type)
{ {
case FSE_ENCODING_RLE : case FSE_ENCODING_RLE :
if (!srcSize) return ERROR(srcSize_wrong); if (!srcSize) return ERROR(srcSize_wrong);
FSE_buildDTable_rle(DTable, *(const BYTE*)src); FSE_buildDTable_rle(DTable, (*(const BYTE*)src) & max); /* if *src > max, data is corrupted */
return 1; return 1;
case FSE_ENCODING_RAW : case FSE_ENCODING_RAW :
FSE_buildDTable_raw(DTable, rawBits); FSE_buildDTable_raw(DTable, rawBits);
@ -528,7 +529,7 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32
return 0; return 0;
default : /* impossible */ default : /* impossible */
case FSE_ENCODING_DYNAMIC : case FSE_ENCODING_DYNAMIC :
{ U32 tableLog, max = (1<<rawBits)-1; { U32 tableLog;
S16 norm[MaxSeq+1]; S16 norm[MaxSeq+1];
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
if (FSE_isError(headerSize)) return ERROR(GENERIC); if (FSE_isError(headerSize)) return ERROR(GENERIC);
@ -539,8 +540,6 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32
} }
size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
const void* src, size_t srcSize) const void* src, size_t srcSize)
@ -641,13 +640,13 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
} }
/* Offset */ /* Offset */
{ { static const U32 offsetPrefix[MaxOff+1] = {
static const U32 offsetPrefix[MaxOff+1] = { 1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40,
1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 0x100, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000,
0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000,
0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 }; 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 };
const U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ U32 const offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
const U32 nbBits = offsetCode ? offsetCode-1 : 0; U32 const nbBits = offsetCode ? offsetCode-1 : 0;
offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits); offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits);
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset; if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset;
@ -727,8 +726,7 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
return sequenceLength; return sequenceLength;
} }
/* span extDict & currentPrefixSegment */ /* span extDict & currentPrefixSegment */
{ { size_t const length1 = dictEnd - match;
size_t length1 = dictEnd - match;
memmove(oLitEnd, match, length1); memmove(oLitEnd, match, length1);
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;

2
programs/.gitignore vendored
View File

@ -5,6 +5,8 @@ fullbench
fullbench32 fullbench32
fuzzer fuzzer
fuzzer32 fuzzer32
zbufftest
zbufftest32
datagen datagen
paramgrill paramgrill

178
programs/fuzzer.c
View File

@ -153,8 +153,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
if (!CNBuffer || !compressedBuffer || !decodedBuffer) if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
{
DISPLAY("Not enough memory, aborting\n"); DISPLAY("Not enough memory, aborting\n");
testResult = 1; testResult = 1;
goto _end; goto _end;
@ -174,11 +173,9 @@ static int basicUnitTests(U32 seed, double compressibility)
if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error; if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error;
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
{ { size_t i;
size_t i;
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) {
{
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;; if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
@ -197,8 +194,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* Dictionary and Duplication tests */ /* Dictionary and Duplication tests */
{ { ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx(); ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx();
const size_t dictSize = 500; const size_t dictSize = 500;
@ -269,8 +265,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* block API tests */ /* block API tests */
{ { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx();
const size_t blockSize = 100 KB; const size_t blockSize = 100 KB;
const size_t dictSize = 16 KB; const size_t dictSize = 16 KB;
@ -312,8 +307,7 @@ static int basicUnitTests(U32 seed, double compressibility)
} }
/* long rle test */ /* long rle test */
{ { size_t sampleSize = 0;
size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++); DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState); RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
@ -344,28 +338,26 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "OK \n");
/* nbSeq limit test */ /* nbSeq limit test */
{ #define _3BYTESTESTLENGTH 131000
#define _3BYTESTESTLENGTH 131000 #define NB3BYTESSEQLOG 9
#define NB3BYTESSEQLOG 9 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
#define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
#define NB3BYTESSEQMASK (NB3BYTESSEQ-1) { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
BYTE _3BytesSeqs[NB3BYTESSEQ][3];
U32 r = 1; U32 r = 1;
int i;
for (i=0; i < NB3BYTESSEQ; i++) { { int i; for (i=0; i < NB3BYTESSEQ; i++) {
_3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&r) & 255); _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&r) & 255);
_3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&r) & 255); _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&r) & 255);
_3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&r) & 255); _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&r) & 255);
} }}
for (i=0; i < _3BYTESTESTLENGTH; ) { { int i; for (i=0; i < _3BYTESTESTLENGTH; ) {
U32 id = FUZ_rand(&r) & NB3BYTESSEQMASK; U32 id = FUZ_rand(&r) & NB3BYTESSEQMASK;
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
i += 3; i += 3;
} }}
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19); result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19);
@ -398,8 +390,7 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
const BYTE* b1 = (const BYTE*)buf1; const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2; const BYTE* b2 = (const BYTE*)buf2;
size_t i; size_t i;
for (i=0; i<max; i++) for (i=0; i<max; i++) {
{
if (b1[i] != b2[i]) break; if (b1[i] != b2[i]) break;
} }
return i; return i;
@ -458,8 +449,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
FUZ_rand(&coreSeed); FUZ_rand(&coreSeed);
/* test loop */ /* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) {
{
size_t sampleSize, sampleStart, maxTestSize, totalTestSize; size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize; size_t cSize, dSize, dSupSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n; U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
@ -479,16 +469,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
lseed = coreSeed ^ prime1; lseed = coreSeed ^ prime1;
buffNb = FUZ_rand(&lseed) & 127; buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2; if (buffNb & 7) buffNb=2;
else else {
{
buffNb >>= 3; buffNb >>= 3;
if (buffNb & 7) if (buffNb & 7) {
{
const U32 tnb[2] = { 1, 3 }; const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3]; buffNb = tnb[buffNb >> 3];
} } else {
else
{
const U32 tnb[2] = { 0, 4 }; const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3]; buffNb = tnb[buffNb >> 3];
} }
@ -506,7 +492,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
crcOrig = XXH64(sampleBuffer, sampleSize, 0); crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression test */ /* compression test */
//cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* high levels only for small samples, for manageable speed */
cLevelMod = MIN( ZSTD_maxCLevel(), (U32)MAX(1, 55 - 3*(int)sampleSizeLog) ); /* high levels only for small samples, for manageable speed */ cLevelMod = MIN( ZSTD_maxCLevel(), (U32)MAX(1, 55 - 3*(int)sampleSizeLog) ); /* high levels only for small samples, for manageable speed */
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
@ -517,12 +502,11 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing; const size_t tooSmallSize = cSize - missing;
static const U32 endMark = 0x4DC2B1A9; static const U32 endMark = 0x4DC2B1A9;
U32 endCheck;
memcpy(dstBuffer+tooSmallSize, &endMark, 4); memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
memcpy(&endCheck, dstBuffer+tooSmallSize, 4); { U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
} }
/* decompression header test */ /* decompression header test */
@ -542,8 +526,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
free(sampleBuffer); /* no longer useful after this point */ free(sampleBuffer); /* no longer useful after this point */
/* truncated src decompression test */ /* truncated src decompression test */
{ { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing; const size_t tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !"); CHECK(cBufferTooSmall == NULL, "not enough memory !");
@ -554,8 +537,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
} }
/* too small dst decompression test */ /* too small dst decompression test */
if (sampleSize > 3) if (sampleSize > 3) {
{
const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = sampleSize - missing; const size_t tooSmallSize = sampleSize - missing;
static const BYTE token = 0xA9; static const BYTE token = 0xA9;
@ -566,39 +548,32 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
} }
/* noisy src decompression test */ /* noisy src decompression test */
if (cSize > 6) if (cSize > 6) {
{ /* insert noise into src */
const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
size_t pos = 4; /* preserve magic number (too easy to detect) */ size_t pos = 4; /* preserve magic number (too easy to detect) */
U32 nbBits = FUZ_rand(&lseed) % maxNbBits; for (;;) {
size_t mask = (1<<nbBits) - 1; /* keep some original src */
size_t skipLength = FUZ_rand(&lseed) & mask; { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
pos += skipLength; size_t const mask = (1<<nbBits) - 1;
size_t const skipLength = FUZ_rand(&lseed) & mask;
while (pos < cSize) pos += skipLength;
{ }
/* add noise */ if (pos <= cSize) break;
size_t noiseStart, noiseLength; /* add noise */
nbBits = FUZ_rand(&lseed) % maxNbBits; { U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
if (nbBits>0) nbBits--; size_t mask, noiseStart, noiseLength;
mask = (1<<nbBits) - 1; if (nbBits>0) nbBits--;
noiseLength = (FUZ_rand(&lseed) & mask) + 1; mask = (1<<nbBits) - 1;
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos; noiseLength = (FUZ_rand(&lseed) & mask) + 1;
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
pos += noiseLength; memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength;
/* keep some original src */ } } }
nbBits = FUZ_rand(&lseed) % maxNbBits;
mask = (1<<nbBits) - 1;
skipLength = FUZ_rand(&lseed) & mask;
pos += skipLength;
}
/* decompress noisy source */ /* decompress noisy source */
{ U32 const noiseSrc = FUZ_rand(&lseed) % 5; { U32 const endMark = 0xA9B1C3D6;
U32 const endMark = 0xA9B1C3D6;
srcBuffer = cNoiseBuffer[noiseSrc];
memcpy(dstBuffer+sampleSize, &endMark, 4); memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */ /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */
@ -606,8 +581,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
{ U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); }
} } } /* noisy src decompression test */
}
/* Streaming compression of scattered segments test */ /* Streaming compression of scattered segments test */
XXH64_reset(xxh64, 0); XXH64_reset(xxh64, 0);
@ -629,8 +603,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
errorCode = ZSTD_copyCCtx(ctx, refCtx); errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0; totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++) for (n=0; n<nbChunks; n++) {
{
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog; sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1); sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
@ -659,8 +632,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
totalCSize = 0; totalCSize = 0;
totalGenSize = 0; totalGenSize = 0;
while (totalCSize < cSize) while (totalCSize < cSize) {
{
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx); size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize)); CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
@ -675,7 +647,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize); errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)", CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]); (U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
} }
DISPLAY("\r%u fuzzer tests completed \n", testNb-1); DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
@ -699,10 +670,10 @@ _output_error:
} }
/********************************************************* /*_*******************************************************
* Command line * Command line
*********************************************************/ *********************************************************/
int FUZ_usage(char* programName) int FUZ_usage(const char* programName)
{ {
DISPLAY( "Usage :\n"); DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName); DISPLAY( " %s [args]\n", programName);
@ -719,7 +690,7 @@ int FUZ_usage(char* programName)
} }
int main(int argc, char** argv) int main(int argc, const char** argv)
{ {
U32 seed=0; U32 seed=0;
int seedset=0; int seedset=0;
@ -729,23 +700,18 @@ int main(int argc, char** argv)
int proba = FUZ_COMPRESSIBILITY_DEFAULT; int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0; int result=0;
U32 mainPause = 0; U32 mainPause = 0;
char* programName; const char* programName;
/* Check command line */ /* Check command line */
programName = argv[0]; programName = argv[0];
for(argNb=1; argNb<argc; argNb++) for (argNb=1; argNb<argc; argNb++) {
{ const char* argument = argv[argNb];
char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */ if(!argument) continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */ /* Handle commands. Aggregated commands are allowed */
if (argument[0]=='-') if (argument[0]=='-') {
{
argument++; argument++;
while (*argument!=0) {
while (*argument!=0)
{
switch(*argument) switch(*argument)
{ {
case 'h': case 'h':
@ -766,8 +732,7 @@ int main(int argc, char** argv)
case 'i': case 'i':
argument++; g_testTime=0; argument++; g_testTime=0;
nbTests=0; nbTests=0;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9')) {
{
nbTests *= 10; nbTests *= 10;
nbTests += *argument - '0'; nbTests += *argument - '0';
argument++; argument++;
@ -777,8 +742,7 @@ int main(int argc, char** argv)
case 'T': case 'T':
argument++; argument++;
nbTests=0; g_testTime=0; nbTests=0; g_testTime=0;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9')) {
{
g_testTime *= 10; g_testTime *= 10;
g_testTime += *argument - '0'; g_testTime += *argument - '0';
argument++; argument++;
@ -792,8 +756,7 @@ int main(int argc, char** argv)
argument++; argument++;
seed=0; seed=0;
seedset=1; seedset=1;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9')) {
{
seed *= 10; seed *= 10;
seed += *argument - '0'; seed += *argument - '0';
argument++; argument++;
@ -803,8 +766,7 @@ int main(int argc, char** argv)
case 't': case 't':
argument++; argument++;
testNb=0; testNb=0;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9')) {
{
testNb *= 10; testNb *= 10;
testNb += *argument - '0'; testNb += *argument - '0';
argument++; argument++;
@ -814,8 +776,7 @@ int main(int argc, char** argv)
case 'P': /* compressibility % */ case 'P': /* compressibility % */
argument++; argument++;
proba=0; proba=0;
while ((*argument>='0') && (*argument<='9')) while ((*argument>='0') && (*argument<='9')) {
{
proba *= 10; proba *= 10;
proba += *argument - '0'; proba += *argument - '0';
argument++; argument++;
@ -826,10 +787,7 @@ int main(int argc, char** argv)
default: default:
return FUZ_usage(programName); return FUZ_usage(programName);
} } } } } /* for (argNb=1; argNb<argc; argNb++) */
}
}
}
/* Get Seed */ /* Get Seed */
DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION); DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
@ -838,11 +796,11 @@ int main(int argc, char** argv)
DISPLAY("Seed = %u\n", seed); DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ if (testNb==0)
result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
if (!result) if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
if (mainPause) if (mainPause) {
{
int unused; int unused;
DISPLAY("Press Enter \n"); DISPLAY("Press Enter \n");
unused = getchar(); unused = getchar();