mirror of
https://github.com/facebook/zstd.git
synced 2025-12-10 00:06:51 -05:00
commit
ea0b5fc193
12
.travis.yml
12
.travis.yml
@ -20,23 +20,25 @@ matrix:
|
|||||||
- env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
|
- env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
|
||||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
|
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
|
||||||
|
|
||||||
|
- env: Cmd='make staticAnalyze'
|
||||||
|
|
||||||
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
|
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
|
||||||
- env: Cmd='make gcc6install libc6install
|
- env: Cmd='make gcc6install libc6install
|
||||||
&& make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
|
&& make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
|
||||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
|
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
|
||||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
||||||
|
|
||||||
- env: Cmd='make arminstall && make armfuzz'
|
|
||||||
- env: Cmd='make arminstall && make aarch64fuzz'
|
|
||||||
- env: Cmd='make ppcinstall && make ppcfuzz'
|
|
||||||
- env: Cmd='make ppcinstall && make ppc64fuzz'
|
|
||||||
|
|
||||||
- env: Cmd='make -j uasanregressiontest
|
- env: Cmd='make -j uasanregressiontest
|
||||||
&& make clean && make -j msanregressiontest'
|
&& make clean && make -j msanregressiontest'
|
||||||
|
|
||||||
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
|
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
|
||||||
&& make clean && make -C tests test-fuzzer-stackmode'
|
&& make clean && make -C tests test-fuzzer-stackmode'
|
||||||
|
|
||||||
|
- env: Cmd='make arminstall && make armfuzz'
|
||||||
|
- env: Cmd='make arminstall && make aarch64fuzz'
|
||||||
|
- env: Cmd='make ppcinstall && make ppcfuzz'
|
||||||
|
- env: Cmd='make ppcinstall && make ppc64fuzz'
|
||||||
|
|
||||||
- env: Cmd='make lz4install && make -C tests test-lz4
|
- env: Cmd='make lz4install && make -C tests test-lz4
|
||||||
&& make clean && make -C tests test-pool
|
&& make clean && make -C tests test-pool
|
||||||
&& make clean && bash tests/libzstd_partial_builds.sh'
|
&& make clean && bash tests/libzstd_partial_builds.sh'
|
||||||
|
|||||||
22
Makefile
22
Makefile
@ -30,10 +30,9 @@ default: lib-release zstd-release
|
|||||||
all: allmost examples manual contrib
|
all: allmost examples manual contrib
|
||||||
|
|
||||||
.PHONY: allmost
|
.PHONY: allmost
|
||||||
allmost: allzstd
|
allmost: allzstd zlibwrapper
|
||||||
$(MAKE) -C $(ZWRAPDIR) all
|
|
||||||
|
|
||||||
#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||||
.PHONY: allzstd
|
.PHONY: allzstd
|
||||||
allzstd: lib
|
allzstd: lib
|
||||||
$(MAKE) -C $(PRGDIR) all
|
$(MAKE) -C $(PRGDIR) all
|
||||||
@ -44,8 +43,8 @@ all32:
|
|||||||
$(MAKE) -C $(PRGDIR) zstd32
|
$(MAKE) -C $(PRGDIR) zstd32
|
||||||
$(MAKE) -C $(TESTDIR) all32
|
$(MAKE) -C $(TESTDIR) all32
|
||||||
|
|
||||||
.PHONY: lib lib-release
|
.PHONY: lib lib-release libzstd.a
|
||||||
lib lib-release:
|
lib lib-release :
|
||||||
@$(MAKE) -C $(ZSTDDIR) $@
|
@$(MAKE) -C $(ZSTDDIR) $@
|
||||||
|
|
||||||
.PHONY: zstd zstd-release
|
.PHONY: zstd zstd-release
|
||||||
@ -59,8 +58,8 @@ zstdmt:
|
|||||||
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||||
|
|
||||||
.PHONY: zlibwrapper
|
.PHONY: zlibwrapper
|
||||||
zlibwrapper:
|
zlibwrapper: lib
|
||||||
$(MAKE) -C $(ZWRAPDIR) test
|
$(MAKE) -C $(ZWRAPDIR) all
|
||||||
|
|
||||||
.PHONY: test
|
.PHONY: test
|
||||||
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
|
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
|
||||||
@ -114,7 +113,7 @@ clean:
|
|||||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
|
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
|
||||||
|
|
||||||
HOST_OS = POSIX
|
HOST_OS = POSIX
|
||||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||||
|
|
||||||
.PHONY: list
|
.PHONY: list
|
||||||
list:
|
list:
|
||||||
@ -351,7 +350,10 @@ bmi32build: clean
|
|||||||
$(CC) -v
|
$(CC) -v
|
||||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||||
|
|
||||||
staticAnalyze: clean
|
# static analyzer test uses clang's scan-build
|
||||||
|
# does not analyze zlibWrapper, due to detected issues in zlib source code
|
||||||
|
staticAnalyze: SCANBUILD ?= scan-build
|
||||||
|
staticAnalyze:
|
||||||
$(CC) -v
|
$(CC) -v
|
||||||
CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
|
CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
|
||||||
endif
|
endif
|
||||||
|
|||||||
@ -101,7 +101,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
free(buffOut);
|
free(buffOut);
|
||||||
}
|
}
|
||||||
|
|
||||||
static const char* createOutFilename_orDie(const char* filename)
|
static char* createOutFilename_orDie(const char* filename)
|
||||||
{
|
{
|
||||||
size_t const inL = strlen(filename);
|
size_t const inL = strlen(filename);
|
||||||
size_t const outL = inL + 5;
|
size_t const outL = inL + 5;
|
||||||
@ -109,7 +109,7 @@ static const char* createOutFilename_orDie(const char* filename)
|
|||||||
memset(outSpace, 0, outL);
|
memset(outSpace, 0, outL);
|
||||||
strcat(outSpace, filename);
|
strcat(outSpace, filename);
|
||||||
strcat(outSpace, ".zst");
|
strcat(outSpace, ".zst");
|
||||||
return (const char*)outSpace;
|
return (char*)outSpace;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, const char** argv) {
|
int main(int argc, const char** argv) {
|
||||||
@ -124,8 +124,9 @@ int main(int argc, const char** argv) {
|
|||||||
{ const char* const inFileName = argv[1];
|
{ const char* const inFileName = argv[1];
|
||||||
unsigned const frameSize = (unsigned)atoi(argv[2]);
|
unsigned const frameSize = (unsigned)atoi(argv[2]);
|
||||||
|
|
||||||
const char* const outFileName = createOutFilename_orDie(inFileName);
|
char* const outFileName = createOutFilename_orDie(inFileName);
|
||||||
compressFile_orDie(inFileName, outFileName, 5, frameSize);
|
compressFile_orDie(inFileName, outFileName, 5, frameSize);
|
||||||
|
free(outFileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@ -56,6 +56,7 @@
|
|||||||
|
|
||||||
#include <stdlib.h> /* malloc, free */
|
#include <stdlib.h> /* malloc, free */
|
||||||
#include <stdio.h> /* FILE* */
|
#include <stdio.h> /* FILE* */
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
#define XXH_STATIC_LINKING_ONLY
|
#define XXH_STATIC_LINKING_ONLY
|
||||||
#define XXH_NAMESPACE ZSTD_
|
#define XXH_NAMESPACE ZSTD_
|
||||||
@ -112,7 +113,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
|
|||||||
|
|
||||||
static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
|
static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
|
||||||
{
|
{
|
||||||
buffWrapper_t* buff = (buffWrapper_t*) opaque;
|
buffWrapper_t* const buff = (buffWrapper_t*) opaque;
|
||||||
unsigned long long newOffset;
|
unsigned long long newOffset;
|
||||||
switch (origin) {
|
switch (origin) {
|
||||||
case SEEK_SET:
|
case SEEK_SET:
|
||||||
@ -124,6 +125,8 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
|
|||||||
case SEEK_END:
|
case SEEK_END:
|
||||||
newOffset = (unsigned long long)buff->size - offset;
|
newOffset = (unsigned long long)buff->size - offset;
|
||||||
break;
|
break;
|
||||||
|
default:
|
||||||
|
assert(0); /* not possible */
|
||||||
}
|
}
|
||||||
if (newOffset > buff->size) {
|
if (newOffset > buff->size) {
|
||||||
return -1;
|
return -1;
|
||||||
|
|||||||
@ -158,7 +158,8 @@ int main(int argc, const char** argv)
|
|||||||
}
|
}
|
||||||
|
|
||||||
freeResources(ress);
|
freeResources(ress);
|
||||||
/* success */
|
free(ofnBuffer);
|
||||||
|
|
||||||
printf("compressed %i files \n", argc-1);
|
printf("compressed %i files \n", argc-1);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@ -73,7 +73,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
ZSTD_CStream* const cstream = ZSTD_createCStream();
|
ZSTD_CStream* const cstream = ZSTD_createCStream();
|
||||||
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
|
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
|
||||||
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
|
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
|
||||||
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
|
if (ZSTD_isError(initResult)) {
|
||||||
|
fprintf(stderr, "ZSTD_initCStream() error : %s \n",
|
||||||
|
ZSTD_getErrorName(initResult));
|
||||||
|
exit(11);
|
||||||
|
}
|
||||||
|
|
||||||
size_t read, toRead = buffInSize;
|
size_t read, toRead = buffInSize;
|
||||||
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
|
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
|
||||||
@ -81,7 +85,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
while (input.pos < input.size) {
|
while (input.pos < input.size) {
|
||||||
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
||||||
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
|
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
|
||||||
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
|
if (ZSTD_isError(toRead)) {
|
||||||
|
fprintf(stderr, "ZSTD_compressStream() error : %s \n",
|
||||||
|
ZSTD_getErrorName(toRead));
|
||||||
|
exit(12);
|
||||||
|
}
|
||||||
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
|
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
|
||||||
fwrite_orDie(buffOut, output.pos, fout);
|
fwrite_orDie(buffOut, output.pos, fout);
|
||||||
}
|
}
|
||||||
@ -100,15 +108,15 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static const char* createOutFilename_orDie(const char* filename)
|
static char* createOutFilename_orDie(const char* filename)
|
||||||
{
|
{
|
||||||
size_t const inL = strlen(filename);
|
size_t const inL = strlen(filename);
|
||||||
size_t const outL = inL + 5;
|
size_t const outL = inL + 5;
|
||||||
void* outSpace = malloc_orDie(outL);
|
void* const outSpace = malloc_orDie(outL);
|
||||||
memset(outSpace, 0, outL);
|
memset(outSpace, 0, outL);
|
||||||
strcat(outSpace, filename);
|
strcat(outSpace, filename);
|
||||||
strcat(outSpace, ".zst");
|
strcat(outSpace, ".zst");
|
||||||
return (const char*)outSpace;
|
return (char*)outSpace;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, const char** argv)
|
int main(int argc, const char** argv)
|
||||||
@ -124,8 +132,10 @@ int main(int argc, const char** argv)
|
|||||||
|
|
||||||
const char* const inFilename = argv[1];
|
const char* const inFilename = argv[1];
|
||||||
|
|
||||||
const char* const outFilename = createOutFilename_orDie(inFilename);
|
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||||
compressFile_orDie(inFilename, outFilename, 1);
|
compressFile_orDie(inFilename, outFilename, 1);
|
||||||
|
|
||||||
|
free(outFilename); /* not strictly required, since program execution stops there,
|
||||||
|
* but some static analyzer main complain otherwise */
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -79,8 +79,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
|||||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||||
|
|
||||||
#define ZSTD_FRAMEIDSIZE 4
|
#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
|
||||||
static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */
|
|
||||||
|
|
||||||
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||||
|
|||||||
@ -83,7 +83,9 @@
|
|||||||
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
|
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
|
||||||
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
|
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
|
||||||
*/
|
*/
|
||||||
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||||
|
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
||||||
|
void* workSpace, size_t wkspSize)
|
||||||
{
|
{
|
||||||
U32 const tableSize = 1 << tableLog;
|
U32 const tableSize = 1 << tableLog;
|
||||||
U32 const tableMask = tableSize - 1;
|
U32 const tableMask = tableSize - 1;
|
||||||
@ -101,10 +103,14 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
|
|||||||
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||||
tableU16[-2] = (U16) tableLog;
|
tableU16[-2] = (U16) tableLog;
|
||||||
tableU16[-1] = (U16) maxSymbolValue;
|
tableU16[-1] = (U16) maxSymbolValue;
|
||||||
assert(tableLog < 16); /* required for the threshold strategy to work */
|
assert(tableLog < 16); /* required for threshold strategy to work */
|
||||||
|
|
||||||
/* For explanations on how to distribute symbol values over the table :
|
/* For explanations on how to distribute symbol values over the table :
|
||||||
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||||
|
|
||||||
|
#ifdef __clang_analyzer__
|
||||||
|
memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
||||||
|
#endif
|
||||||
|
|
||||||
/* symbol start positions */
|
/* symbol start positions */
|
||||||
{ U32 u;
|
{ U32 u;
|
||||||
@ -124,13 +130,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
|
|||||||
U32 symbol;
|
U32 symbol;
|
||||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||||
int nbOccurences;
|
int nbOccurences;
|
||||||
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
|
int const freq = normalizedCounter[symbol];
|
||||||
|
for (nbOccurences=0; nbOccurences<freq; nbOccurences++) {
|
||||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||||
position = (position + step) & tableMask;
|
position = (position + step) & tableMask;
|
||||||
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
|
while (position > highThreshold)
|
||||||
|
position = (position + step) & tableMask; /* Low proba area */
|
||||||
} }
|
} }
|
||||||
|
|
||||||
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
|
assert(position==0); /* Must have initialized all positions */
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build table */
|
/* Build table */
|
||||||
@ -201,9 +209,10 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
|
|||||||
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
static size_t
|
||||||
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||||
unsigned writeIsSafe)
|
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
||||||
|
unsigned writeIsSafe)
|
||||||
{
|
{
|
||||||
BYTE* const ostart = (BYTE*) header;
|
BYTE* const ostart = (BYTE*) header;
|
||||||
BYTE* out = ostart;
|
BYTE* out = ostart;
|
||||||
@ -212,13 +221,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||||||
const int tableSize = 1 << tableLog;
|
const int tableSize = 1 << tableLog;
|
||||||
int remaining;
|
int remaining;
|
||||||
int threshold;
|
int threshold;
|
||||||
U32 bitStream;
|
U32 bitStream = 0;
|
||||||
int bitCount;
|
int bitCount = 0;
|
||||||
unsigned charnum = 0;
|
unsigned symbol = 0;
|
||||||
int previous0 = 0;
|
unsigned const alphabetSize = maxSymbolValue + 1;
|
||||||
|
int previousIs0 = 0;
|
||||||
|
|
||||||
bitStream = 0;
|
|
||||||
bitCount = 0;
|
|
||||||
/* Table Size */
|
/* Table Size */
|
||||||
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
|
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
|
||||||
bitCount += 4;
|
bitCount += 4;
|
||||||
@ -228,48 +236,53 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||||||
threshold = tableSize;
|
threshold = tableSize;
|
||||||
nbBits = tableLog+1;
|
nbBits = tableLog+1;
|
||||||
|
|
||||||
while (remaining>1) { /* stops at 1 */
|
while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
|
||||||
if (previous0) {
|
if (previousIs0) {
|
||||||
unsigned start = charnum;
|
unsigned start = symbol;
|
||||||
while (!normalizedCounter[charnum]) charnum++;
|
while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
|
||||||
while (charnum >= start+24) {
|
if (symbol == alphabetSize) break; /* incorrect distribution */
|
||||||
|
while (symbol >= start+24) {
|
||||||
start+=24;
|
start+=24;
|
||||||
bitStream += 0xFFFFU << bitCount;
|
bitStream += 0xFFFFU << bitCount;
|
||||||
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend-2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE) bitStream;
|
out[0] = (BYTE) bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out+=2;
|
out+=2;
|
||||||
bitStream>>=16;
|
bitStream>>=16;
|
||||||
}
|
}
|
||||||
while (charnum >= start+3) {
|
while (symbol >= start+3) {
|
||||||
start+=3;
|
start+=3;
|
||||||
bitStream += 3 << bitCount;
|
bitStream += 3 << bitCount;
|
||||||
bitCount += 2;
|
bitCount += 2;
|
||||||
}
|
}
|
||||||
bitStream += (charnum-start) << bitCount;
|
bitStream += (symbol-start) << bitCount;
|
||||||
bitCount += 2;
|
bitCount += 2;
|
||||||
if (bitCount>16) {
|
if (bitCount>16) {
|
||||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend - 2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE)bitStream;
|
out[0] = (BYTE)bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out += 2;
|
out += 2;
|
||||||
bitStream >>= 16;
|
bitStream >>= 16;
|
||||||
bitCount -= 16;
|
bitCount -= 16;
|
||||||
} }
|
} }
|
||||||
{ int count = normalizedCounter[charnum++];
|
{ int count = normalizedCounter[symbol++];
|
||||||
int const max = (2*threshold-1)-remaining;
|
int const max = (2*threshold-1) - remaining;
|
||||||
remaining -= count < 0 ? -count : count;
|
remaining -= count < 0 ? -count : count;
|
||||||
count++; /* +1 for extra accuracy */
|
count++; /* +1 for extra accuracy */
|
||||||
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
if (count>=threshold)
|
||||||
|
count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
||||||
bitStream += count << bitCount;
|
bitStream += count << bitCount;
|
||||||
bitCount += nbBits;
|
bitCount += nbBits;
|
||||||
bitCount -= (count<max);
|
bitCount -= (count<max);
|
||||||
previous0 = (count==1);
|
previousIs0 = (count==1);
|
||||||
if (remaining<1) return ERROR(GENERIC);
|
if (remaining<1) return ERROR(GENERIC);
|
||||||
while (remaining<threshold) { nbBits--; threshold>>=1; }
|
while (remaining<threshold) { nbBits--; threshold>>=1; }
|
||||||
}
|
}
|
||||||
if (bitCount>16) {
|
if (bitCount>16) {
|
||||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend - 2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE)bitStream;
|
out[0] = (BYTE)bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out += 2;
|
out += 2;
|
||||||
@ -277,19 +290,23 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||||||
bitCount -= 16;
|
bitCount -= 16;
|
||||||
} }
|
} }
|
||||||
|
|
||||||
|
if (remaining != 1)
|
||||||
|
return ERROR(GENERIC); /* incorrect normalized distribution */
|
||||||
|
assert(symbol <= alphabetSize);
|
||||||
|
|
||||||
/* flush remaining bitStream */
|
/* flush remaining bitStream */
|
||||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend - 2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE)bitStream;
|
out[0] = (BYTE)bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out+= (bitCount+7) /8;
|
out+= (bitCount+7) /8;
|
||||||
|
|
||||||
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC);
|
|
||||||
|
|
||||||
return (out-ostart);
|
return (out-ostart);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
||||||
|
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||||
{
|
{
|
||||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
|
||||||
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
|
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
|
||||||
@ -297,7 +314,7 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
|
|||||||
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
|
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
|
||||||
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
|
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
|
||||||
|
|
||||||
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
|
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -1147,7 +1147,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
|||||||
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
||||||
zc->workSpaceSize = neededSpace;
|
zc->workSpaceSize = neededSpace;
|
||||||
zc->workSpaceOversizedDuration = 0;
|
zc->workSpaceOversizedDuration = 0;
|
||||||
ptr = zc->workSpace;
|
|
||||||
|
|
||||||
/* Statically sized space.
|
/* Statically sized space.
|
||||||
* entropyWorkspace never moves,
|
* entropyWorkspace never moves,
|
||||||
|
|||||||
@ -970,7 +970,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
|
|||||||
U32 seqPos = cur;
|
U32 seqPos = cur;
|
||||||
|
|
||||||
DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
|
DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
|
||||||
last_pos, cur);
|
last_pos, cur); (void)last_pos;
|
||||||
assert(storeEnd < ZSTD_OPT_NUM);
|
assert(storeEnd < ZSTD_OPT_NUM);
|
||||||
DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
||||||
storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
|
storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
|
||||||
|
|||||||
@ -320,7 +320,8 @@ static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq)
|
|||||||
|
|
||||||
static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem)
|
static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem)
|
||||||
{
|
{
|
||||||
ZSTDMT_seqPool* seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem);
|
ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem);
|
||||||
|
if (seqPool == NULL) return NULL;
|
||||||
ZSTDMT_setNbSeq(seqPool, 0);
|
ZSTDMT_setNbSeq(seqPool, 0);
|
||||||
return seqPool;
|
return seqPool;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -185,7 +185,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
|
|||||||
static size_t ZSTD_startingInputLength(ZSTD_format_e format)
|
static size_t ZSTD_startingInputLength(ZSTD_format_e format)
|
||||||
{
|
{
|
||||||
size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
|
size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
|
||||||
ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize :
|
ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE :
|
||||||
ZSTD_frameHeaderSize_prefix;
|
ZSTD_frameHeaderSize_prefix;
|
||||||
ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
|
ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
|
||||||
/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
|
/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
|
||||||
@ -278,7 +278,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
|||||||
* Note 3 : Skippable Frame Identifiers are considered valid. */
|
* Note 3 : Skippable Frame Identifiers are considered valid. */
|
||||||
unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
||||||
{
|
{
|
||||||
if (size < ZSTD_frameIdSize) return 0;
|
if (size < ZSTD_FRAMEIDSIZE) return 0;
|
||||||
{ U32 const magic = MEM_readLE32(buffer);
|
{ U32 const magic = MEM_readLE32(buffer);
|
||||||
if (magic == ZSTD_MAGICNUMBER) return 1;
|
if (magic == ZSTD_MAGICNUMBER) return 1;
|
||||||
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
|
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
|
||||||
@ -330,7 +330,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
|||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t const minInputSize = ZSTD_startingInputLength(format);
|
size_t const minInputSize = ZSTD_startingInputLength(format);
|
||||||
|
|
||||||
|
memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
|
||||||
if (srcSize < minInputSize) return minInputSize;
|
if (srcSize < minInputSize) return minInputSize;
|
||||||
|
if (src==NULL) return ERROR(GENERIC); /* invalid parameter */
|
||||||
|
|
||||||
if ( (format != ZSTD_f_zstd1_magicless)
|
if ( (format != ZSTD_f_zstd1_magicless)
|
||||||
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
|
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
|
||||||
@ -339,7 +341,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
|||||||
if (srcSize < ZSTD_skippableHeaderSize)
|
if (srcSize < ZSTD_skippableHeaderSize)
|
||||||
return ZSTD_skippableHeaderSize; /* magic number + frame length */
|
return ZSTD_skippableHeaderSize; /* magic number + frame length */
|
||||||
memset(zfhPtr, 0, sizeof(*zfhPtr));
|
memset(zfhPtr, 0, sizeof(*zfhPtr));
|
||||||
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize);
|
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
|
||||||
zfhPtr->frameType = ZSTD_skippableFrame;
|
zfhPtr->frameType = ZSTD_skippableFrame;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@ -451,7 +453,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
|||||||
size_t skippableSize;
|
size_t skippableSize;
|
||||||
if (srcSize < ZSTD_skippableHeaderSize)
|
if (srcSize < ZSTD_skippableHeaderSize)
|
||||||
return ERROR(srcSize_wrong);
|
return ERROR(srcSize_wrong);
|
||||||
skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize)
|
skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE)
|
||||||
+ ZSTD_skippableHeaderSize;
|
+ ZSTD_skippableHeaderSize;
|
||||||
if (srcSize < skippableSize) {
|
if (srcSize < skippableSize) {
|
||||||
return ZSTD_CONTENTSIZE_ERROR;
|
return ZSTD_CONTENTSIZE_ERROR;
|
||||||
@ -540,6 +542,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
|||||||
static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
|
static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
|
||||||
const void* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
||||||
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||||
memcpy(dst, src, srcSize);
|
memcpy(dst, src, srcSize);
|
||||||
return srcSize;
|
return srcSize;
|
||||||
@ -1763,7 +1766,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|||||||
#endif
|
#endif
|
||||||
if ( (srcSize >= ZSTD_skippableHeaderSize)
|
if ( (srcSize >= ZSTD_skippableHeaderSize)
|
||||||
&& (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
|
&& (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
|
||||||
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize);
|
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE);
|
||||||
} else {
|
} else {
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
const BYTE* const ipstart = ip;
|
const BYTE* const ipstart = ip;
|
||||||
@ -1797,7 +1800,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|||||||
if (zfh.checksumFlag) { /* Final frame content checksum */
|
if (zfh.checksumFlag) { /* Final frame content checksum */
|
||||||
if (remainingSize < 4) return ERROR(srcSize_wrong);
|
if (remainingSize < 4) return ERROR(srcSize_wrong);
|
||||||
ip += 4;
|
ip += 4;
|
||||||
remainingSize -= 4;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return ip - ipstart;
|
return ip - ipstart;
|
||||||
@ -1932,7 +1934,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|||||||
size_t skippableSize;
|
size_t skippableSize;
|
||||||
if (srcSize < ZSTD_skippableHeaderSize)
|
if (srcSize < ZSTD_skippableHeaderSize)
|
||||||
return ERROR(srcSize_wrong);
|
return ERROR(srcSize_wrong);
|
||||||
skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize)
|
skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE)
|
||||||
+ ZSTD_skippableHeaderSize;
|
+ ZSTD_skippableHeaderSize;
|
||||||
if (srcSize < skippableSize) return ERROR(srcSize_wrong);
|
if (srcSize < skippableSize) return ERROR(srcSize_wrong);
|
||||||
|
|
||||||
@ -2057,7 +2059,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
case ZSTDds_getFrameHeaderSize :
|
case ZSTDds_getFrameHeaderSize :
|
||||||
assert(src != NULL);
|
assert(src != NULL);
|
||||||
if (dctx->format == ZSTD_f_zstd1) { /* allows header */
|
if (dctx->format == ZSTD_f_zstd1) { /* allows header */
|
||||||
assert(srcSize >= ZSTD_frameIdSize); /* to read skippable magic number */
|
assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
|
||||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||||
memcpy(dctx->headerBuffer, src, srcSize);
|
memcpy(dctx->headerBuffer, src, srcSize);
|
||||||
dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */
|
dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */
|
||||||
@ -2167,7 +2169,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
assert(src != NULL);
|
assert(src != NULL);
|
||||||
assert(srcSize <= ZSTD_skippableHeaderSize);
|
assert(srcSize <= ZSTD_skippableHeaderSize);
|
||||||
memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */
|
memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */
|
||||||
dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize); /* note : dctx->expected can grow seriously large, beyond local buffer size */
|
dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
|
||||||
dctx->stage = ZSTDds_skipFrame;
|
dctx->stage = ZSTDds_skipFrame;
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
@ -2268,7 +2270,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
|
|||||||
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
||||||
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
||||||
} }
|
} }
|
||||||
dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
|
dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
|
||||||
|
|
||||||
/* load entropy tables */
|
/* load entropy tables */
|
||||||
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
|
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
|
||||||
@ -2381,7 +2383,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e
|
|||||||
return 0; /* pure content mode */
|
return 0; /* pure content mode */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
|
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
|
||||||
|
|
||||||
/* load entropy tables */
|
/* load entropy tables */
|
||||||
CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
|
CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
|
||||||
@ -2398,6 +2400,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
|||||||
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
||||||
ddict->dictBuffer = NULL;
|
ddict->dictBuffer = NULL;
|
||||||
ddict->dictContent = dict;
|
ddict->dictContent = dict;
|
||||||
|
if (!dict) dictSize = 0;
|
||||||
} else {
|
} else {
|
||||||
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
||||||
ddict->dictBuffer = internalBuffer;
|
ddict->dictBuffer = internalBuffer;
|
||||||
@ -2510,7 +2513,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
|||||||
{
|
{
|
||||||
if (dictSize < 8) return 0;
|
if (dictSize < 8) return 0;
|
||||||
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
|
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
|
||||||
return MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
|
return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*! ZSTD_getDictID_fromDDict() :
|
/*! ZSTD_getDictID_fromDDict() :
|
||||||
@ -2855,7 +2858,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
|
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
|
||||||
|
|
||||||
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||||
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize);
|
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
|
||||||
zds->stage = ZSTDds_skipFrame;
|
zds->stage = ZSTDds_skipFrame;
|
||||||
} else {
|
} else {
|
||||||
CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
|
CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
|
||||||
|
|||||||
@ -1637,7 +1637,7 @@ construct_SA(const unsigned char *T, int *SA,
|
|||||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||||
k = SA + BUCKET_B(c2 = c0, c1);
|
k = SA + BUCKET_B(c2 = c0, c1);
|
||||||
}
|
}
|
||||||
assert(k < j);
|
assert(k < j); assert(k != NULL);
|
||||||
*k-- = s;
|
*k-- = s;
|
||||||
} else {
|
} else {
|
||||||
assert(((s == 0) && (T[s] == c1)) || (s < 0));
|
assert(((s == 0) && (T[s] == c1)) || (s < 0));
|
||||||
@ -1701,7 +1701,7 @@ construct_BWT(const unsigned char *T, int *SA,
|
|||||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||||
k = SA + BUCKET_B(c2 = c0, c1);
|
k = SA + BUCKET_B(c2 = c0, c1);
|
||||||
}
|
}
|
||||||
assert(k < j);
|
assert(k < j); assert(k != NULL);
|
||||||
*k-- = s;
|
*k-- = s;
|
||||||
} else if(s != 0) {
|
} else if(s != 0) {
|
||||||
*j = ~s;
|
*j = ~s;
|
||||||
@ -1785,7 +1785,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
|
|||||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||||
k = SA + BUCKET_B(c2 = c0, c1);
|
k = SA + BUCKET_B(c2 = c0, c1);
|
||||||
}
|
}
|
||||||
assert(k < j);
|
assert(k < j); assert(k != NULL);
|
||||||
*k-- = s;
|
*k-- = s;
|
||||||
} else if(s != 0) {
|
} else if(s != 0) {
|
||||||
*j = ~s;
|
*j = ~s;
|
||||||
|
|||||||
@ -698,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|||||||
short litLengthNCount[MaxLL+1];
|
short litLengthNCount[MaxLL+1];
|
||||||
U32 repOffset[MAXREPOFFSET];
|
U32 repOffset[MAXREPOFFSET];
|
||||||
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
||||||
EStats_ress_t esr;
|
EStats_ress_t esr = { NULL, NULL, NULL };
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
||||||
size_t pos = 0, errorCode;
|
size_t pos = 0, errorCode;
|
||||||
|
|||||||
@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
|
|||||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||||
|
|
||||||
/* Init, lay down lowprob symbols */
|
/* Init, lay down lowprob symbols */
|
||||||
|
memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
||||||
DTableH.tableLog = (U16)tableLog;
|
DTableH.tableLog = (U16)tableLog;
|
||||||
for (s=0; s<=maxSymbolValue; s++)
|
for (s=0; s<=maxSymbolValue; s++)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
|
|||||||
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||||
|
|
||||||
/* Init, lay down lowprob symbols */
|
/* Init, lay down lowprob symbols */
|
||||||
|
memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
||||||
DTableH.tableLog = (U16)tableLog;
|
DTableH.tableLog = (U16)tableLog;
|
||||||
for (s=0; s<=maxSymbolValue; s++) {
|
for (s=0; s<=maxSymbolValue; s++) {
|
||||||
if (normalizedCounter[s]==-1) {
|
if (normalizedCounter[s]==-1) {
|
||||||
@ -2845,6 +2846,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|||||||
|
|
||||||
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
||||||
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||||
memcpy(dst, src, srcSize);
|
memcpy(dst, src, srcSize);
|
||||||
return srcSize;
|
return srcSize;
|
||||||
|
|||||||
@ -3041,6 +3041,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|||||||
|
|
||||||
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
||||||
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||||
memcpy(dst, src, srcSize);
|
memcpy(dst, src, srcSize);
|
||||||
return srcSize;
|
return srcSize;
|
||||||
@ -4006,7 +4007,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|||||||
if (ZSTDv06_isError(hSize)) return hSize;
|
if (ZSTDv06_isError(hSize)) return hSize;
|
||||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
||||||
zbd->lhSize += iend-ip; ip = iend; notDone = 0;
|
zbd->lhSize += iend-ip;
|
||||||
*dstCapacityPtr = 0;
|
*dstCapacityPtr = 0;
|
||||||
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3150,10 +3150,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
|
|
||||||
if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
|
if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
|
||||||
|
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||||
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
||||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
||||||
if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
|
if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
|
||||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
|
||||||
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||||
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||||
return 0;
|
return 0;
|
||||||
@ -3175,11 +3175,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
U32 windowSize = 0;
|
U32 windowSize = 0;
|
||||||
U32 dictID = 0;
|
U32 dictID = 0;
|
||||||
U64 frameContentSize = 0;
|
U64 frameContentSize = 0;
|
||||||
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
|
if ((fhdByte & 0x08) != 0) /* reserved bits, which must be zero */
|
||||||
|
return ERROR(frameParameter_unsupported);
|
||||||
if (!directMode) {
|
if (!directMode) {
|
||||||
BYTE const wlByte = ip[pos++];
|
BYTE const wlByte = ip[pos++];
|
||||||
U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
|
U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
|
||||||
if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
if (windowLog > ZSTDv07_WINDOWLOG_MAX)
|
||||||
|
return ERROR(frameParameter_unsupported);
|
||||||
windowSize = (1U << windowLog);
|
windowSize = (1U << windowLog);
|
||||||
windowSize += (windowSize >> 3) * (wlByte&7);
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||||
}
|
}
|
||||||
@ -3201,7 +3203,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||||
}
|
}
|
||||||
if (!windowSize) windowSize = (U32)frameContentSize;
|
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||||
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
if (windowSize > windowSizeMax)
|
||||||
|
return ERROR(frameParameter_unsupported);
|
||||||
fparamsPtr->frameContentSize = frameContentSize;
|
fparamsPtr->frameContentSize = frameContentSize;
|
||||||
fparamsPtr->windowSize = windowSize;
|
fparamsPtr->windowSize = windowSize;
|
||||||
fparamsPtr->dictID = dictID;
|
fparamsPtr->dictID = dictID;
|
||||||
@ -3220,11 +3223,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
- frame header not completely provided (`srcSize` too small) */
|
- frame header not completely provided (`srcSize` too small) */
|
||||||
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
|
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
{ ZSTDv07_frameParams fparams;
|
ZSTDv07_frameParams fparams;
|
||||||
size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
|
size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
|
||||||
if (frResult!=0) return 0;
|
if (frResult!=0) return 0;
|
||||||
return fparams.frameContentSize;
|
return fparams.frameContentSize;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -27,6 +27,7 @@
|
|||||||
#include <string.h> /* memset */
|
#include <string.h> /* memset */
|
||||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||||
#include <errno.h> /* errno */
|
#include <errno.h> /* errno */
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
#include "mem.h" /* read */
|
#include "mem.h" /* read */
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
@ -165,6 +166,7 @@ static U32 DiB_rand(U32* src)
|
|||||||
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
|
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
|
||||||
U32 seed = 0xFD2FB528;
|
U32 seed = 0xFD2FB528;
|
||||||
unsigned i;
|
unsigned i;
|
||||||
|
assert(nbFiles >= 1);
|
||||||
for (i = nbFiles - 1; i > 0; --i) {
|
for (i = nbFiles - 1; i > 0; --i) {
|
||||||
unsigned const j = DiB_rand(&seed) % (i + 1);
|
unsigned const j = DiB_rand(&seed) % (i + 1);
|
||||||
const char* const tmp = fileNamesTable[j];
|
const char* const tmp = fileNamesTable[j];
|
||||||
@ -310,7 +312,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||||||
/* Load input buffer */
|
/* Load input buffer */
|
||||||
DISPLAYLEVEL(3, "Shuffling input files\n");
|
DISPLAYLEVEL(3, "Shuffling input files\n");
|
||||||
DiB_shuffle(fileNamesTable, nbFiles);
|
DiB_shuffle(fileNamesTable, nbFiles);
|
||||||
nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
|
DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
|
||||||
|
|
||||||
{ size_t dictSize;
|
{ size_t dictSize;
|
||||||
if (params) {
|
if (params) {
|
||||||
@ -319,6 +321,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||||||
srcBuffer, sampleSizes, fs.nbSamples,
|
srcBuffer, sampleSizes, fs.nbSamples,
|
||||||
*params);
|
*params);
|
||||||
} else if (optimizeCover) {
|
} else if (optimizeCover) {
|
||||||
|
assert(coverParams != NULL);
|
||||||
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
|
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
|
||||||
srcBuffer, sampleSizes, fs.nbSamples,
|
srcBuffer, sampleSizes, fs.nbSamples,
|
||||||
coverParams);
|
coverParams);
|
||||||
@ -327,6 +330,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
|
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
assert(coverParams != NULL);
|
||||||
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
|
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
|
||||||
sampleSizes, fs.nbSamples, *coverParams);
|
sampleSizes, fs.nbSamples, *coverParams);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -33,7 +33,7 @@
|
|||||||
*/
|
*/
|
||||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||||
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
||||||
ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
|
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
|
||||||
int optimizeCover);
|
int optimizeCover);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@ -179,13 +179,9 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
|
|||||||
(U32)(count.totalMalloc >> 10));
|
(U32)(count.totalMalloc >> 10));
|
||||||
}
|
}
|
||||||
|
|
||||||
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
|
||||||
|
void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
|
||||||
{
|
{
|
||||||
size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
|
|
||||||
size_t const outSize = ZSTD_compressBound(inSize);
|
|
||||||
void* const inBuffer = malloc(inSize);
|
|
||||||
void* const outBuffer = malloc(outSize);
|
|
||||||
|
|
||||||
/* test only played in verbose mode, as they are long */
|
/* test only played in verbose mode, as they are long */
|
||||||
if (g_displayLevel<3) return 0;
|
if (g_displayLevel<3) return 0;
|
||||||
|
|
||||||
@ -270,6 +266,28 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||||
|
{
|
||||||
|
size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
|
||||||
|
size_t const outSize = ZSTD_compressBound(inSize);
|
||||||
|
void* const inBuffer = malloc(inSize);
|
||||||
|
void* const outBuffer = malloc(outSize);
|
||||||
|
int result;
|
||||||
|
|
||||||
|
/* Create compressible noise */
|
||||||
|
if (!inBuffer || !outBuffer) {
|
||||||
|
DISPLAY("Not enough memory, aborting \n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
result = FUZ_mallocTests_internal(seed, compressibility, part,
|
||||||
|
inBuffer, inSize, outBuffer, outSize);
|
||||||
|
|
||||||
|
free(inBuffer);
|
||||||
|
free(outBuffer);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||||
@ -1535,7 +1553,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
||||||
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
||||||
BYTE* cNoiseBuffer[5];
|
BYTE* cNoiseBuffer[5];
|
||||||
BYTE* srcBuffer; /* jumping pointer */
|
|
||||||
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
||||||
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
||||||
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
||||||
@ -1544,7 +1561,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
U32 result = 0;
|
U32 result = 0;
|
||||||
U32 testNb = 0;
|
U32 testNb = 0;
|
||||||
U32 coreSeed = seed, lseed = 0;
|
U32 coreSeed = seed;
|
||||||
UTIL_time_t const startClock = UTIL_getTime();
|
UTIL_time_t const startClock = UTIL_getTime();
|
||||||
U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
|
U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
|
||||||
int const cLevelLimiter = bigTests ? 3 : 2;
|
int const cLevelLimiter = bigTests ? 3 : 2;
|
||||||
@ -1565,13 +1582,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||||
srcBuffer = cNoiseBuffer[2];
|
|
||||||
|
|
||||||
/* catch up testNb */
|
/* catch up testNb */
|
||||||
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
|
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
|
||||||
|
|
||||||
/* main test loop */
|
/* main test loop */
|
||||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
|
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
|
||||||
|
BYTE* srcBuffer; /* jumping pointer */
|
||||||
|
U32 lseed;
|
||||||
size_t sampleSize, maxTestSize, totalTestSize;
|
size_t sampleSize, maxTestSize, totalTestSize;
|
||||||
size_t cSize, totalCSize, totalGenSize;
|
size_t cSize, totalCSize, totalGenSize;
|
||||||
U64 crcOrig;
|
U64 crcOrig;
|
||||||
@ -1802,11 +1820,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
||||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||||
if (crcDest!=crcOrig) {
|
CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
|
||||||
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
|
(U32)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (U32)totalTestSize);
|
||||||
CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
}
|
||||||
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
|
|
||||||
} }
|
|
||||||
} /* for ( ; (testNb <= nbTests) */
|
} /* for ( ; (testNb <= nbTests) */
|
||||||
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
|
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
|
||||||
|
|
||||||
|
|||||||
@ -212,7 +212,7 @@ static void loadFile(void* buffer, const char* fileName, size_t fileSize)
|
|||||||
static void fileCheck(const char* fileName, int testCCtxParams)
|
static void fileCheck(const char* fileName, int testCCtxParams)
|
||||||
{
|
{
|
||||||
size_t const fileSize = getFileSize(fileName);
|
size_t const fileSize = getFileSize(fileName);
|
||||||
void* buffer = malloc(fileSize);
|
void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
|
||||||
if (!buffer) {
|
if (!buffer) {
|
||||||
fprintf(stderr, "not enough memory \n");
|
fprintf(stderr, "not enough memory \n");
|
||||||
exit(4);
|
exit(4);
|
||||||
|
|||||||
@ -600,7 +600,6 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
||||||
DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
|
DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
|
||||||
if (ZSTD_isError(initError)) goto _output_error;
|
if (ZSTD_isError(initError)) goto _output_error;
|
||||||
cSize = 0;
|
|
||||||
outBuff.dst = compressedBuffer;
|
outBuff.dst = compressedBuffer;
|
||||||
outBuff.size = compressedBufferSize;
|
outBuff.size = compressedBufferSize;
|
||||||
outBuff.pos = 0;
|
outBuff.pos = 0;
|
||||||
@ -718,7 +717,6 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
|
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
|
||||||
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
|
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
|
||||||
if (ZSTD_isError(initError)) goto _output_error;
|
if (ZSTD_isError(initError)) goto _output_error;
|
||||||
cSize = 0;
|
|
||||||
outBuff.dst = compressedBuffer;
|
outBuff.dst = compressedBuffer;
|
||||||
outBuff.size = compressedBufferSize;
|
outBuff.size = compressedBufferSize;
|
||||||
outBuff.pos = 0;
|
outBuff.pos = 0;
|
||||||
|
|||||||
@ -573,10 +573,10 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
|||||||
do {
|
do {
|
||||||
testmem = (BYTE*)malloc((size_t)requiredMem);
|
testmem = (BYTE*)malloc((size_t)requiredMem);
|
||||||
requiredMem -= step;
|
requiredMem -= step;
|
||||||
} while (!testmem);
|
} while (!testmem && requiredMem); /* do not allocate zero bytes */
|
||||||
|
|
||||||
free(testmem);
|
free(testmem);
|
||||||
return (size_t)(requiredMem);
|
return (size_t)(requiredMem+1); /* avoid zero */
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||||
@ -734,7 +734,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||||
if (benchedSize < totalSizeToLoad)
|
if (benchedSize < totalSizeToLoad)
|
||||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||||
srcBuffer = malloc(benchedSize);
|
srcBuffer = malloc(benchedSize + !benchedSize);
|
||||||
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
||||||
|
|
||||||
/* Load input buffer */
|
/* Load input buffer */
|
||||||
|
|||||||
@ -111,7 +111,7 @@ local gzFile gz_open(path, fd, mode)
|
|||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
/* allocate gzFile structure to return */
|
/* allocate gzFile structure to return */
|
||||||
state = (gz_statep)(gz_state*)malloc(sizeof(gz_state));
|
state.state = (gz_state*)malloc(sizeof(gz_state));
|
||||||
if (state.state == NULL)
|
if (state.state == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
state.state->size = 0; /* no buffers allocated yet */
|
state.state->size = 0; /* no buffers allocated yet */
|
||||||
@ -266,7 +266,7 @@ local gzFile gz_open(path, fd, mode)
|
|||||||
gz_reset(state);
|
gz_reset(state);
|
||||||
|
|
||||||
/* return stream */
|
/* return stream */
|
||||||
return (gzFile)state.file;
|
return state.file;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* -- see zlib.h -- */
|
/* -- see zlib.h -- */
|
||||||
|
|||||||
@ -6,6 +6,8 @@
|
|||||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
#include "gzguts.h"
|
#include "gzguts.h"
|
||||||
|
|
||||||
/* Local functions */
|
/* Local functions */
|
||||||
@ -24,7 +26,7 @@ local int gz_init(state)
|
|||||||
z_streamp strm = &(state.state->strm);
|
z_streamp strm = &(state.state->strm);
|
||||||
|
|
||||||
/* allocate input buffer (double size for gzprintf) */
|
/* allocate input buffer (double size for gzprintf) */
|
||||||
state.state->in = (unsigned char *)malloc(state.state->want << 1);
|
state.state->in = (unsigned char*)malloc(state.state->want << 1);
|
||||||
if (state.state->in == NULL) {
|
if (state.state->in == NULL) {
|
||||||
gz_error(state, Z_MEM_ERROR, "out of memory");
|
gz_error(state, Z_MEM_ERROR, "out of memory");
|
||||||
return -1;
|
return -1;
|
||||||
@ -33,7 +35,7 @@ local int gz_init(state)
|
|||||||
/* only need output buffer and deflate state if compressing */
|
/* only need output buffer and deflate state if compressing */
|
||||||
if (!state.state->direct) {
|
if (!state.state->direct) {
|
||||||
/* allocate output buffer */
|
/* allocate output buffer */
|
||||||
state.state->out = (unsigned char *)malloc(state.state->want);
|
state.state->out = (unsigned char*)malloc(state.state->want);
|
||||||
if (state.state->out == NULL) {
|
if (state.state->out == NULL) {
|
||||||
free(state.state->in);
|
free(state.state->in);
|
||||||
gz_error(state, Z_MEM_ERROR, "out of memory");
|
gz_error(state, Z_MEM_ERROR, "out of memory");
|
||||||
@ -284,6 +286,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
|
|||||||
gz_statep state;
|
gz_statep state;
|
||||||
|
|
||||||
/* get internal structure */
|
/* get internal structure */
|
||||||
|
assert(size != 0);
|
||||||
if (file == NULL)
|
if (file == NULL)
|
||||||
return 0;
|
return 0;
|
||||||
state = (gz_statep)file;
|
state = (gz_statep)file;
|
||||||
@ -294,7 +297,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
|
|||||||
|
|
||||||
/* compute bytes to read -- error on overflow */
|
/* compute bytes to read -- error on overflow */
|
||||||
len = nitems * size;
|
len = nitems * size;
|
||||||
if (size && len / size != nitems) {
|
if (size && (len / size != nitems)) {
|
||||||
gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
|
gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user