diff --git a/Makefile b/Makefile index 7d9ff4f8d..19b12d0ef 100644 --- a/Makefile +++ b/Makefile @@ -26,13 +26,21 @@ endif default: lib zstd .PHONY: all -all: - $(MAKE) -C $(ZSTDDIR) $@ - $(MAKE) -C $(PRGDIR) $@ zstd32 - $(MAKE) -C $(TESTDIR) $@ all32 - $(MAKE) -C $(ZWRAPDIR) $@ +all: allmost CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@ +.PHONY: allmost +allmost: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(PRGDIR) all + $(MAKE) -C $(TESTDIR) all + $(MAKE) -C $(ZWRAPDIR) all + +.PHONY: all32 +all32: + $(MAKE) -C $(PRGDIR) zstd32 + $(MAKE) -C $(TESTDIR) all32 + .PHONY: lib lib: @$(MAKE) -C $(ZSTDDIR) @@ -151,13 +159,13 @@ gnu90test: clean CFLAGS="-std=gnu90" $(MAKE) all c99test: clean - CFLAGS="-std=c99" $(MAKE) all + CFLAGS="-std=c99" $(MAKE) allmost gnu99test: clean CFLAGS="-std=gnu99" $(MAKE) all c11test: clean - CFLAGS="-std=c11" $(MAKE) all + CFLAGS="-std=c11" $(MAKE) allmost bmix64test: clean CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test diff --git a/lib/common/mem.h b/lib/common/mem.h index 32c63dd17..aff044de1 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -39,7 +39,7 @@ extern "C" { #endif /* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 19e8287e2..e976cd26d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1766,6 +1766,18 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator); } + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is simply referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, 1, allocator); +} + + size_t ZSTD_freeDDict(ZSTD_DDict* ddict) { if (ddict==NULL) return 0; /* support free on NULL */ diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 921e37886..ac22e8705 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -306,13 +306,13 @@ static dictItem ZDICT_analyzePos( } while (length >=MINMATCHLENGTH); /* look backward */ - length = MINMATCHLENGTH; - while ((length >= MINMATCHLENGTH) & (start > 0)) { - length = ZDICT_count(b + pos, b + suffix[start - 1]); - if (length >= LLIMIT) length = LLIMIT - 1; - lengthList[length]++; - if (length >= MINMATCHLENGTH) start--; - } + length = MINMATCHLENGTH; + while ((length >= MINMATCHLENGTH) & (start > 0)) { + length = ZDICT_count(b + pos, b + suffix[start - 1]); + if (length >= LLIMIT) length = LLIMIT - 1; + lengthList[length]++; + if (length >= MINMATCHLENGTH) start--; + } /* largest useful length */ memset(cumulLength, 0, sizeof(cumulLength)); diff --git a/lib/zstd.h b/lib/zstd.h index 187ee5c28..333feff7d 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -348,8 +348,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output #define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +#define ZSTD_FRAMEHEADERSIZE_MIN 6 static const size_t ZSTD_frameHeaderSize_prefix = 5; -static const size_t ZSTD_frameHeaderSize_min = 6; +static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index f389c3a4f..ee413c3ae 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -82,7 +82,7 @@ local int partcompress(FILE *in, z_streamp def) flush = Z_SYNC_FLUSH; do { - def->avail_in = fread(raw, 1, RAWLEN, in); + def->avail_in = (uInt)fread(raw, 1, RAWLEN, in); if (ferror(in)) return Z_ERRNO; def->next_in = raw; @@ -148,7 +148,7 @@ int main(int argc, char **argv) /* get requested output size */ if (argc != 2) quit("need one argument: size of output block"); - ret = strtol(argv[1], argv + 1, 10); + ret = (int)strtol(argv[1], argv + 1, 10); if (argv[1][0] != 0) quit("argument must be a number"); if (ret < 8) /* 8 is minimum zlib stream size */ diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 49a2632e1..e5c54438b 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -315,10 +315,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ } def.next_in = (z_const void*) blockTable[blockNb].srcPtr; - def.avail_in = blockTable[blockNb].srcSize; + def.avail_in = (uInt)blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; - def.avail_out = blockTable[blockNb].cRoom; + def.avail_out = (uInt)blockTable[blockNb].cRoom; def.total_out = 0; ret = deflate(&def, Z_FINISH); if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize); @@ -346,10 +346,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); } def.next_in = (z_const void*) blockTable[blockNb].srcPtr; - def.avail_in = blockTable[blockNb].srcSize; + def.avail_in = (uInt)blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; - def.avail_out = blockTable[blockNb].cRoom; + def.avail_out = (uInt)blockTable[blockNb].cRoom; def.total_out = 0; ret = deflate(&def, Z_FINISH); if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure"); @@ -451,10 +451,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ret = inflateReset(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); inf.next_in = (z_const void*) blockTable[blockNb].cPtr; - inf.avail_in = blockTable[blockNb].cSize; + inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; - inf.avail_out = blockTable[blockNb].srcSize; + inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.total_out = 0; ret = inflate(&inf, Z_FINISH); if (ret == Z_NEED_DICT) { @@ -483,10 +483,10 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ret = inflateInit(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); inf.next_in = (z_const void*) blockTable[blockNb].cPtr; - inf.avail_in = blockTable[blockNb].cSize; + inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; - inf.avail_out = blockTable[blockNb].srcSize; + inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.total_out = 0; ret = inflate(&inf, Z_FINISH); if (ret == Z_NEED_DICT) { diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c index bf1a3cc6c..f251e2fe4 100644 --- a/zlibWrapper/gzread.c +++ b/zlibWrapper/gzread.c @@ -30,7 +30,7 @@ local int gz_load(state, buf, len, have) *have = 0; do { - ret = read(state.state->fd, buf + *have, len - *have); + ret = (int)read(state.state->fd, buf + *have, len - *have); if (ret <= 0) break; *have += ret; @@ -469,7 +469,7 @@ int ZEXPORT gzungetc(c, file) if (state.state->x.have == 0) { state.state->x.have = 1; state.state->x.next = state.state->out + (state.state->size << 1) - 1; - state.state->x.next[0] = c; + state.state->x.next[0] = (unsigned char)c; state.state->x.pos--; state.state->past = 0; return c; @@ -491,7 +491,7 @@ int ZEXPORT gzungetc(c, file) } state.state->x.have++; state.state->x.next--; - state.state->x.next[0] = c; + state.state->x.next[0] = (unsigned char)c; state.state->x.pos--; state.state->past = 0; return c; diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c index 13c8fb264..6f3c9658b 100644 --- a/zlibWrapper/gzwrite.c +++ b/zlibWrapper/gzwrite.c @@ -84,7 +84,7 @@ local int gz_comp(state, flush) /* write directly if requested */ if (state.state->direct) { - got = write(state.state->fd, strm->next_in, strm->avail_in); + got = (int)write(state.state->fd, strm->next_in, strm->avail_in); if (got < 0 || (unsigned)got != strm->avail_in) { gz_error(state, Z_ERRNO, zstrerror()); return -1; @@ -101,7 +101,7 @@ local int gz_comp(state, flush) if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && (flush != Z_FINISH || ret == Z_STREAM_END))) { have = (unsigned)(strm->next_out - state.state->x.next); - if (have && ((got = write(state.state->fd, state.state->x.next, have)) < 0 || + if (have && ((got = (int)write(state.state->fd, state.state->x.next, have)) < 0 || (unsigned)got != have)) { gz_error(state, Z_ERRNO, zstrerror()); return -1; @@ -278,7 +278,7 @@ int ZEXPORT gzputc(file, c) strm->next_in = state.state->in; have = (unsigned)((strm->next_in + strm->avail_in) - state.state->in); if (have < state.state->size) { - state.state->in[have] = c; + state.state->in[have] = (unsigned char)c; strm->avail_in++; state.state->x.pos++; return c & 0xff; @@ -286,7 +286,7 @@ int ZEXPORT gzputc(file, c) } /* no room in buffer or not initialized, use gz_write() */ - buf[0] = c; + buf[0] = (unsigned char)c; if (gzwrite(file, buf, 1) != 1) return -1; return c & 0xff; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index d26663142..238510523 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -61,7 +61,7 @@ ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } static void* ZWRAP_allocFunction(void* opaque, size_t size) { z_streamp strm = (z_streamp) opaque; - void* address = strm->zalloc(strm->opaque, 1, size); + void* address = strm->zalloc(strm->opaque, 1, (uInt)size); /* printf("ZWRAP alloc %p, %d \n", address, (int)size); */ return address; } @@ -130,7 +130,7 @@ int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, { LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc); if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR; - + if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); size_t errorCode; @@ -156,7 +156,7 @@ int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message) ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; strm->msg = message; if (zwc == NULL) return Z_STREAM_ERROR; - + return ZWRAPC_finishWithError(zwc, strm, 0); } @@ -165,7 +165,7 @@ int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize) { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; if (zwc == NULL) return Z_STREAM_ERROR; - + zwc->pledgedSrcSize = pledgedSrcSize; zwc->comprState = ZWRAP_useInit; return Z_OK; @@ -232,7 +232,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) LOG_WRAPPERC("- deflateReset\n"); if (!g_ZWRAP_useZSTDcompression) return deflateReset(strm); - + ZWRAP_deflateReset_keepDict(strm); { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; @@ -284,7 +284,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) if (zwc->zbc == NULL) { int res; zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); - if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : 0); if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset; @@ -321,9 +321,9 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) strm->avail_in -= zwc->inBuffer.pos; } - if (flush == Z_FULL_FLUSH + if (flush == Z_FULL_FLUSH #if ZLIB_VERNUM >= 0x1240 - || flush == Z_TREES + || flush == Z_TREES #endif || flush == Z_BLOCK) return ZWRAPC_finishWithErrorMsg(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); @@ -424,7 +424,7 @@ typedef struct { } ZWRAP_DCtx; -int ZWRAP_isUsingZSTDdecompression(z_streamp strm) +int ZWRAP_isUsingZSTDdecompression(z_streamp strm) { if (strm == NULL) return 0; return (strm->reserved == ZWRAP_ZSTD_STREAM); @@ -458,7 +458,7 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_frameHeaderSize_min); /* if compilation fails here, assertion is false */ + MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_FRAMEHEADERSIZE_MIN); /* if compilation fails here, assertion is false */ ZWRAP_initDCtx(zwd); return zwd; } @@ -488,7 +488,7 @@ int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message) ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; strm->msg = message; if (zwd == NULL) return Z_STREAM_ERROR; - + return ZWRAPD_finishWithError(zwd, strm, 0); } @@ -528,7 +528,7 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { return inflateInit2_(strm, windowBits, version, stream_size); } - + { int ret = z_inflateInit_ (strm, version, stream_size); LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits); @@ -552,7 +552,7 @@ int ZWRAP_inflateReset_keepDict(z_streamp strm) ZWRAP_initDCtx(zwd); zwd->decompState = ZWRAP_useReset; } - + strm->total_in = 0; strm->total_out = 0; return Z_OK; @@ -569,7 +569,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) if (ret != Z_OK) return ret; } { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (zwd == NULL) return Z_STREAM_ERROR; + if (zwd == NULL) return Z_STREAM_ERROR; zwd->decompState = ZWRAP_useInit; } return Z_OK; @@ -608,7 +608,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); - zwd->decompState = ZWRAP_useReset; + zwd->decompState = ZWRAP_useReset; if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; @@ -787,9 +787,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) strm->total_in += zwd->inBuffer.pos; strm->next_in += zwd->inBuffer.pos; strm->avail_in -= zwd->inBuffer.pos; - if (errorCode == 0) { - LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); - zwd->decompState = ZWRAP_streamEnd; + if (errorCode == 0) { + LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + zwd->decompState = ZWRAP_streamEnd; return Z_STREAM_END; } }