diff --git a/.travis.yml b/.travis.yml index be8a90afe..58728d89c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,63 @@ compiler: gcc matrix: fast_finish: true include: + # OS X Mavericks + - os: osx + env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install" + # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install" + - os: linux + sudo: false + language: cpp + install: + - export CXX="g++-4.8" CC="gcc-4.8" + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 + env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" + - os: linux + sudo: false + env: PLATFORM="Ubuntu 12.04 container" CMD="make asan" + # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest' + env: PLATFORM="Ubuntu 12.04" CMD="make armtest" + addons: + apt: + packages: + - gcc-arm-linux-gnueabi + - libc6-dev-armel-cross + - linux-libc-dev-armel-cross + - binfmt-support + - qemu + - qemu-user-static + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest" + - os: linux + sudo: required + env: PLATFORM="Ubuntu 12.04" CMD="make asan32" + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - libc6-dev-i386 + - gcc-multilib + # Ubuntu 14.04 LTS Server Edition 64 bit + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest" addons: apt: packages: @@ -13,7 +67,25 @@ matrix: - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" + env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest" + addons: + apt: + packages: + - libc6-dev-i386 + - g++-multilib + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32" + addons: + apt: + packages: + - libc6-dev-i386 + - gcc-multilib + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest" addons: apt: sources: diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index a612f2593..ea025b951 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -1,8 +1,8 @@ # Makefile for example of using zstd wrapper for zlib # -# make - compiles statically and dynamically linked examples -# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on -# make test test_d - runs statically and dynamically linked examples +# make - compiles examples +# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on +# make test - runs examples # Paths to static and dynamic zlib and zstd libraries @@ -22,7 +22,7 @@ LDFLAGS = $(LOC) RM = rm -f -all: clean fitblk example example_d zwrapbench +all: clean fitblk example zwrapbench test: example fitblk example_zstd fitblk_zstd zwrapbench ./example @@ -34,11 +34,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench ./zwrapbench -qb1e5 ../zstd_compression_format.md ./zwrapbench -qb1e5B1K ../zstd_compression_format.md -test_d: example_d - ./example_d - +#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 -valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench @echo "\n ---- valgrind tests ----" $(VALGRIND) ./example @@ -83,5 +80,5 @@ $(ZSTDLIBDIR)/libzstd.so: $(MAKE) -C $(ZSTDLIBDIR) all clean: - -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench @echo Cleaning completed diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 7a825899c..76f075108 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -548,6 +548,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR; errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); + zwd->decompState = Z_NEED_DICT; if (strm->total_in == ZSTD_HEADERSIZE) { zwd->inBuffer.src = zwd->headerBuf; @@ -571,6 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) { + ZWRAP_DCtx* zwd; int res; if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); @@ -581,60 +583,79 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->avail_in > 0) { size_t errorCode, srcSize; - ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; - if (zwd == NULL) return Z_STREAM_ERROR; + zwd = (ZWRAP_DCtx*) strm->state; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + if (zwd == NULL) return Z_STREAM_ERROR; if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; - if (strm->total_in < ZLIB_HEADERSIZE) - { - srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); - memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); - strm->total_in += srcSize; - strm->next_in += srcSize; - strm->avail_in -= srcSize; - if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK; + if (strm->total_in < ZLIB_HEADERSIZE) { + if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) { + if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) { + if (zwd->windowBits) + errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); + else + errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); - if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { - z_stream strm2; - strm2.next_in = strm->next_in; - strm2.avail_in = strm->avail_in; - strm2.next_out = strm->next_out; - strm2.avail_out = strm->avail_out; + strm->reserved = 0; /* mark as zlib stream */ + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) goto error; - if (zwd->windowBits) - errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); - else - errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); - LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); - if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); + if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); + else res = inflate(strm, flush); + LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); + return res; + } + } else { + srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); + memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); + strm->total_in += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK; - /* inflate header */ - strm->next_in = (unsigned char*)zwd->headerBuf; - strm->avail_in = ZLIB_HEADERSIZE; - strm->avail_out = 0; - errorCode = inflate(strm, Z_NO_FLUSH); - LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); - if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); - if (strm->avail_in > 0) goto error; + if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { + z_stream strm2; + strm2.next_in = strm->next_in; + strm2.avail_in = strm->avail_in; + strm2.next_out = strm->next_out; + strm2.avail_out = strm->avail_out; - strm->next_in = strm2.next_in; - strm->avail_in = strm2.avail_in; - strm->next_out = strm2.next_out; - strm->avail_out = strm2.avail_out; + if (zwd->windowBits) + errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); + else + errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); + LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); + if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); - strm->reserved = 0; /* mark as zlib stream */ - errorCode = ZWRAP_freeDCtx(zwd); - if (ZSTD_isError(errorCode)) goto error; + /* inflate header */ + strm->next_in = (unsigned char*)zwd->headerBuf; + strm->avail_in = ZLIB_HEADERSIZE; + strm->avail_out = 0; + errorCode = inflate(strm, Z_NO_FLUSH); + LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); + if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); + if (strm->avail_in > 0) goto error; - if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); - else res = inflate(strm, flush); - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); - return res; + strm->next_in = strm2.next_in; + strm->avail_in = strm2.avail_in; + strm->next_out = strm2.next_out; + strm->avail_out = strm2.avail_out; + + strm->reserved = 0; /* mark as zlib stream */ + errorCode = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(errorCode)) goto error; + + if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); + else res = inflate(strm, flush); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); + return res; + } } } + if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } + if (!zwd->zbd) { zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; } @@ -642,31 +663,36 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (strm->total_in < ZSTD_HEADERSIZE) { - srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); - memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); - strm->total_in += srcSize; - strm->next_in += srcSize; - strm->avail_in -= srcSize; - if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; + if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) { + if (zwd->decompState != Z_NEED_DICT) { + errorCode = ZSTD_initDStream(zwd->zbd); + if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } + } + } else { + srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); + memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); + strm->total_in += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; - errorCode = ZSTD_initDStream(zwd->zbd); - if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } + errorCode = ZSTD_initDStream(zwd->zbd); + if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } - if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } - - zwd->inBuffer.src = zwd->headerBuf; - zwd->inBuffer.size = ZSTD_HEADERSIZE; - zwd->inBuffer.pos = 0; - zwd->outBuffer.dst = strm->next_out; - zwd->outBuffer.size = 0; - zwd->outBuffer.pos = 0; - errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); - if (ZSTD_isError(errorCode)) { - LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); - goto error; + zwd->inBuffer.src = zwd->headerBuf; + zwd->inBuffer.size = ZSTD_HEADERSIZE; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = 0; + zwd->outBuffer.pos = 0; + errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + if (ZSTD_isError(errorCode)) { + LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); + goto error; + } + if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ } - if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */ } zwd->inBuffer.src = strm->next_in; @@ -694,13 +720,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->decompState = Z_STREAM_END; return Z_STREAM_END; } - goto finish; -error: - return ZWRAPD_finishWithError(zwd, strm, 0); } -finish: LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); return Z_OK; + +error: + return ZWRAPD_finishWithError(zwd, strm, 0); }