faster inflate() autodetection of zlib/zstd

.travis.yml

@@ -3,9 +3,63 @@ compiler: gcc
 matrix:
   fast_finish: true
   include:
+    # OS X Mavericks
+    - os: osx
+      env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install"
+    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
+    - os: linux
+      sudo: false
+      language: cpp
+      install:
+        - export CXX="g++-4.8" CC="gcc-4.8"
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - gcc-4.8
+            - g++-4.8
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
+    - os: linux
+      sudo: false
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
+    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest'
+      env: PLATFORM="Ubuntu 12.04" CMD="make armtest"
+      addons:
+        apt:
+          packages:
+            - gcc-arm-linux-gnueabi
+            - libc6-dev-armel-cross
+            - linux-libc-dev-armel-cross
+            - binfmt-support
+            - qemu
+            - qemu-user-static
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
+    - os: linux
+      sudo: required
+      env: PLATFORM="Ubuntu 12.04" CMD="make asan32"
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - libc6-dev-i386
+            - gcc-multilib
+    # Ubuntu 14.04 LTS Server Edition 64 bit
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest"
       addons:
         apt:
           packages:
@@ -13,7 +67,25 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
+      env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - g++-multilib 
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32"
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+            - gcc-multilib
+    - os: linux
+      dist: trusty
+      sudo: required
+      env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
       addons:
         apt:
           sources:

zlibWrapper/Makefile

@@ -1,8 +1,8 @@
 # Makefile for example of using zstd wrapper for zlib
 #
-# make - compiles statically and dynamically linked examples
+# make - compiles examples
-# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on
+# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
-# make test test_d - runs statically and dynamically linked examples
+# make test - runs examples
 
 
 # Paths to static and dynamic zlib and zstd libraries
@@ -22,7 +22,7 @@ LDFLAGS = $(LOC)
 RM = rm -f
 
 
-all: clean fitblk example example_d zwrapbench
+all: clean fitblk example zwrapbench
 
 test: example fitblk example_zstd fitblk_zstd zwrapbench
 	./example
@@ -34,11 +34,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench
 	./zwrapbench -qb1e5 ../zstd_compression_format.md
 	./zwrapbench -qb1e5B1K ../zstd_compression_format.md
 
-test_d: example_d
+#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
-	./example_d
-
 valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
-valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
 valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
 	@echo "\n ---- valgrind tests ----"
 	$(VALGRIND) ./example
@@ -83,5 +80,5 @@ $(ZSTDLIBDIR)/libzstd.so:
 	$(MAKE) -C $(ZSTDLIBDIR) all
 
 clean:
-	-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench
+	-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench
 	@echo Cleaning completed

zlibWrapper/zstd_zlibwrapper.c

@@ -548,6 +548,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
         if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
         errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
         if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
+        zwd->decompState = Z_NEED_DICT; 
 
         if (strm->total_in == ZSTD_HEADERSIZE) {
             zwd->inBuffer.src = zwd->headerBuf;
@@ -571,6 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
 
 ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 {
+    ZWRAP_DCtx* zwd;
     int res;
     if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
         LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
@@ -581,14 +583,30 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 
     if (strm->avail_in > 0) {
         size_t errorCode, srcSize;
-        ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+        zwd = (ZWRAP_DCtx*) strm->state;
-        if (zwd == NULL) return Z_STREAM_ERROR;
         LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
 
+        if (zwd == NULL) return Z_STREAM_ERROR;
         if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
 
-        if (strm->total_in < ZLIB_HEADERSIZE)
+        if (strm->total_in < ZLIB_HEADERSIZE) {
-        {
+            if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
+                if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
+                    if (zwd->windowBits)
+                        errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
+                    else
+                        errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
+
+                    strm->reserved = 0; /* mark as zlib stream */
+                    errorCode = ZWRAP_freeDCtx(zwd);
+                    if (ZSTD_isError(errorCode)) goto error;
+
+                    if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
+                    else res = inflate(strm, flush);
+                    LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+                    return res;
+                }
+            } else {
                 srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
                 memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
                 strm->total_in += srcSize;
@@ -634,6 +652,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                     return res;
                 }
             }
+        }
+
+        if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
 
         if (!zwd->zbd) {
             zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
@@ -642,6 +663,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
 
         if (strm->total_in < ZSTD_HEADERSIZE)
         {
+            if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
+                if (zwd->decompState != Z_NEED_DICT) {
+                    errorCode = ZSTD_initDStream(zwd->zbd);
+                    if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+                }
+            } else {
                 srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
                 memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
                 strm->total_in += srcSize;
@@ -652,8 +679,6 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                 errorCode = ZSTD_initDStream(zwd->zbd);
                 if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
 
-            if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
-
                 zwd->inBuffer.src = zwd->headerBuf;
                 zwd->inBuffer.size = ZSTD_HEADERSIZE;
                 zwd->inBuffer.pos = 0;
@@ -668,6 +693,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
                 }
                 if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
             }
+        }
 
         zwd->inBuffer.src = strm->next_in;
         zwd->inBuffer.size = strm->avail_in;
@@ -694,13 +720,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
             zwd->decompState = Z_STREAM_END; 
             return Z_STREAM_END;
         }
-        goto finish;
-error:
-        return ZWRAPD_finishWithError(zwd, strm, 0);
     }
-finish:
     LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
     return Z_OK;
+
+error:
+    return ZWRAPD_finishWithError(zwd, strm, 0);
 }