diff --git a/lib/bitstream.h b/lib/bitstream.h index 006a927f0..44c0241fa 100644 --- a/lib/bitstream.h +++ b/lib/bitstream.h @@ -56,10 +56,9 @@ extern "C" { /*-****************************************** * bitStream encoding API (write forward) ********************************************/ -/*! -* bitStream can mix input from multiple sources. -* A critical property of these streams is that they encode and decode in **reverse** direction. -* So the first bit sequence you add will be the last to be read, like a LIFO stack. +/* bitStream can mix input from multiple sources. +* A critical property of these streams is that they encode and decode in **reverse** direction. +* So the first bit sequence you add will be the last to be read, like a LIFO stack. */ typedef struct { @@ -75,10 +74,9 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); -/*! -* Start by initCStream, providing the size of buffer to write into. +/*Start by initCStream, providing the size of buffer to write into. * bitStream will never write outside of this buffer. -* @dstCapacity must be >= sizeof(size_t), otherwise @return will be an error code. +* `dstCapacity` must be >= sizeof(size_t), otherwise @return will be an error code. * * bits are first added to a local register. * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. @@ -90,7 +88,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); * * Last operation is to close the bitStream. * The function returns the final size of CStream in bytes. -* If data couldn't fit into @dstBuffer, it will return a 0 ( == not storable) +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) */ @@ -117,8 +115,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); -/*! -* Start by invoking BIT_initDStream(). +/*Start by invoking BIT_initDStream(). * A chunk of the bitStream is then stored into a local register. * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). * You can then retrieve bitFields stored into the local register, **in reverse order**. @@ -190,7 +187,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) bitC->bitPos += nbBits; } -/*! BIT_addBitsFast +/*! BIT_addBitsFast() : * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) { @@ -198,7 +195,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBi bitC->bitPos += nbBits; } -/*! BIT_flushBitsFast +/*! BIT_flushBitsFast() : * unsafe version; does not check buffer overflow */ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) { @@ -219,8 +216,8 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ } -/*! BIT_closeCStream - * @result : size of CStream, in bytes, or 0 if it cannot fit into dstBuffer */ +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, or 0 if it cannot fit into dstBuffer */ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) { char* endPtr; @@ -241,12 +238,12 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) /*-******************************************************** * bitStream decoding **********************************************************/ -/*!BIT_initDStream +/*!BIT_initDStream() : * Initialize a BIT_DStream_t. -* @bitD : a pointer to an already allocated BIT_DStream_t structure -* @srcBuffer must point at the beginning of a bitStream -* @srcSize must be the exact size of the bitStream -* @result : size of stream (== srcSize) or an errorCode if a problem is detected +* `bitD` : a pointer to an already allocated BIT_DStream_t structure. +* `srcBuffer` must point at the beginning of a bitStream. +* `srcSize` must be the exact size of the bitStream. +* @return : size of stream (== srcSize) or an errorCode if a problem is detected */ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) { @@ -284,24 +281,24 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si return srcSize; } -/*!BIT_lookBits - * Provides next n bits from local register - * local register is not modified (bits are still present for next read/look) - * On 32-bits, maxNbBits==25 - * On 64-bits, maxNbBits==57 +/*!BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified (bits are still present for next read/look). + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. * @return : value extracted */ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits) { - const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); } -/*! BIT_lookBitsFast : +/*! BIT_lookBitsFast*() : * unsafe version; only works only if nbBits >= 1 */ MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits) { - const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); } @@ -310,7 +307,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) bitD->bitsConsumed += nbBits; } -/*!BIT_readBits +/*!BIT_readBits() : * Read next n bits from local register. * pay attention to not read more than nbBits contained into local register. * @return : extracted value. @@ -322,7 +319,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) return value; } -/*!BIT_readBitsFast : +/*!BIT_readBitsFast() : * unsafe version; only works only if nbBits >= 1 */ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) { @@ -360,7 +357,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) } } -/*! BIT_endOfDStream +/*! BIT_endOfDStream() : * @return Tells if DStream has reached its exact end */ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) diff --git a/lib/fse.c b/lib/fse.c index a445f3285..291e64192 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -466,7 +466,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t bitStream >>= 2; } { - const short max = (short)((2*threshold-1)-remaining); + short const max = (short)((2*threshold-1)-remaining); short count; if ((bitStream & (threshold-1)) < (U32)max) { diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index d8d7837a5..5cf9c17c3 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -515,11 +515,12 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog, const void* src, size_t srcSize) { + U32 max = (1< max, data is corrupted */ return 1; case FSE_ENCODING_RAW : FSE_buildDTable_raw(DTable, rawBits); @@ -528,7 +529,7 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 rawBits, U32 return 0; default : /* impossible */ case FSE_ENCODING_DYNAMIC : - { U32 tableLog, max = (1<stateOffb)); /* <= maxOff, by table construction */ - const U32 nbBits = offsetCode ? offsetCode-1 : 0; + { static const U32 offsetPrefix[MaxOff+1] = { + 1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, + 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, + 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000, + 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 }; + U32 const offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + U32 const nbBits = offsetCode ? offsetCode-1 : 0; offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits); if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); if (offsetCode==0) offset = litLength ? seq->offset : seqState->prevOffset; @@ -727,8 +726,7 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, return sequenceLength; } /* span extDict & currentPrefixSegment */ - { - size_t length1 = dictEnd - match; + { size_t const length1 = dictEnd - match; memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; diff --git a/programs/.gitignore b/programs/.gitignore index 525037b94..a1d72aad1 100644 --- a/programs/.gitignore +++ b/programs/.gitignore @@ -5,6 +5,8 @@ fullbench fullbench32 fuzzer fuzzer32 +zbufftest +zbufftest32 datagen paramgrill diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 72d71ea98..8bd2a1866 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -153,8 +153,7 @@ static int basicUnitTests(U32 seed, double compressibility) CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)); decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); - if (!CNBuffer || !compressedBuffer || !decodedBuffer) - { + if (!CNBuffer || !compressedBuffer || !decodedBuffer) { DISPLAY("Not enough memory, aborting\n"); testResult = 1; goto _end; @@ -174,11 +173,9 @@ static int basicUnitTests(U32 seed, double compressibility) if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error; DISPLAYLEVEL(4, "OK \n"); - { - size_t i; + { size_t i; DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); - for (i=0; i>= 3; - if (buffNb & 7) - { + if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; buffNb = tnb[buffNb >> 3]; - } - else - { + } else { const U32 tnb[2] = { 0, 4 }; buffNb = tnb[buffNb >> 3]; } @@ -506,7 +492,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit crcOrig = XXH64(sampleBuffer, sampleSize, 0); /* compression test */ - //cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* high levels only for small samples, for manageable speed */ cLevelMod = MIN( ZSTD_maxCLevel(), (U32)MAX(1, 55 - 3*(int)sampleSizeLog) ); /* high levels only for small samples, for manageable speed */ cLevel = (FUZ_rand(&lseed) % cLevelMod) +1; cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); @@ -517,12 +502,11 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; static const U32 endMark = 0x4DC2B1A9; - U32 endCheck; memcpy(dstBuffer+tooSmallSize, &endMark, 4); errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); - memcpy(&endCheck, dstBuffer+tooSmallSize, 4); - CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); + { U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4); + CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } } /* decompression header test */ @@ -542,8 +526,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit free(sampleBuffer); /* no longer useful after this point */ /* truncated src decompression test */ - { - const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ + { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = cSize - missing; void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */ CHECK(cBufferTooSmall == NULL, "not enough memory !"); @@ -554,8 +537,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit } /* too small dst decompression test */ - if (sampleSize > 3) - { + if (sampleSize > 3) { const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ const size_t tooSmallSize = sampleSize - missing; static const BYTE token = 0xA9; @@ -566,39 +548,32 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit } /* noisy src decompression test */ - if (cSize > 6) - { - const U32 maxNbBits = FUZ_highbit32((U32)(cSize-4)); - size_t pos = 4; /* preserve magic number (too easy to detect) */ - U32 nbBits = FUZ_rand(&lseed) % maxNbBits; - size_t mask = (1<0) nbBits--; - mask = (1< cSize ) noiseLength = cSize-pos; - noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); - memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); - pos += noiseLength; - - /* keep some original src */ - nbBits = FUZ_rand(&lseed) % maxNbBits; - mask = (1< 6) { + /* insert noise into src */ + { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); + size_t pos = 4; /* preserve magic number (too easy to detect) */ + for (;;) { + /* keep some original src */ + { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; + size_t const mask = (1<0) nbBits--; + mask = (1< cSize ) noiseLength = cSize-pos; + noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); + memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); + pos += noiseLength; + } } } /* decompress noisy source */ - { U32 const noiseSrc = FUZ_rand(&lseed) % 5; - U32 const endMark = 0xA9B1C3D6; - srcBuffer = cNoiseBuffer[noiseSrc]; + { U32 const endMark = 0xA9B1C3D6; memcpy(dstBuffer+sampleSize, &endMark, 4); errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); /* result *may* be an unlikely success, but even then, it must strictly respect dest buffer boundaries */ @@ -606,8 +581,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize); { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); } - } - } + } } /* noisy src decompression test */ /* Streaming compression of scattered segments test */ XXH64_reset(xxh64, 0); @@ -629,8 +603,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit errorCode = ZSTD_copyCCtx(ctx, refCtx); CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); totalTestSize = 0; cSize = 0; - for (n=0; n='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; nbTests += *argument - '0'; argument++; @@ -777,8 +742,7 @@ int main(int argc, char** argv) case 'T': argument++; nbTests=0; g_testTime=0; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { g_testTime *= 10; g_testTime += *argument - '0'; argument++; @@ -792,8 +756,7 @@ int main(int argc, char** argv) argument++; seed=0; seedset=1; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { seed *= 10; seed += *argument - '0'; argument++; @@ -803,8 +766,7 @@ int main(int argc, char** argv) case 't': argument++; testNb=0; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { testNb *= 10; testNb += *argument - '0'; argument++; @@ -814,8 +776,7 @@ int main(int argc, char** argv) case 'P': /* compressibility % */ argument++; proba=0; - while ((*argument>='0') && (*argument<='9')) - { + while ((*argument>='0') && (*argument<='9')) { proba *= 10; proba += *argument - '0'; argument++; @@ -826,10 +787,7 @@ int main(int argc, char** argv) default: return FUZ_usage(programName); - } - } - } - } + } } } } /* for (argNb=1; argNb