Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

.travis.yml

@@ -1,146 +1,34 @@
 language: c
+sudo: required
+dist: trusty
 matrix:
   fast_finish: true
   include:
     # OS X Mavericks
-    - env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
+    - env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
       os: osx
 
-
+    # Ubuntu 14.04 LTS Server Edition 64 bit
-    # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
+    - env: Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
-    - env: Ubu=12.04cont Cmd="make usan"
-      os: linux
-      sudo: false
-
-    - env: Ubu=12.04cont Cmd="make asan"
-      os: linux
-      sudo: false
-
-
-    - env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"'
-      os: linux
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - llvm-toolchain-trusty-4.0
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-            - clang-4.0
-
-
-    # Standard Ubuntu 12.04 LTS Server Edition 64 bit
-    - env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..'
-      os: linux
-      sudo: required
       install:
         - export CXX="g++-6" CC="gcc-6"
-        - export LDFLAGS="-fuse-ld=gold"
+    - env: Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest'
-        - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
+    - env: Cmd="make arminstall armtest && make clean && make aarch64test"
-      addons:
+    - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - gcc-6
-            - g++-6
-
-
-    # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean
-                       && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
-      os: linux
-      dist: trusty
-      sudo: required
-      install:
-        - export CXX="g++-4.8" CC="gcc-4.8"
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - g++-multilib
-            - gcc-4.8
-            - gcc-4.8-multilib
-            - g++-4.8
-            - g++-4.8-multilib
-
-    - env: Ubu=14.04 Cmd="make armtest"
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - qemu-system-arm
-            - qemu-user-static
-            - gcc-arm-linux-gnueabi
-            - libc6-dev-armel-cross
-
-    - env: Ubu=14.04 Cmd="make aarch64test"
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - qemu-system-arm
-            - qemu-user-static
-            - gcc-aarch64-linux-gnu
-            - libc6-dev-arm64-cross
-
-    - env: Ubu=14.04 Cmd='make ppctest'
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - qemu-system-ppc
-            - qemu-user-static
-            - gcc-powerpc-linux-gnu
-
-    - env: Ubu=14.04 Cmd='make ppc64test'
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - qemu-system-ppc
-            - qemu-user-static
-            - gcc-powerpc-linux-gnu
-
 
     # other feature branches => short tests
-    - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
+    - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
-      os: linux
+    - env: Cmd="make libc6install && make -C tests test32"
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - valgrind
-
-    - env: Ubu=14.04 Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make -C tests test32 versionsTest"
-      os: linux
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          packages:
-            - libc6-dev-i386
-            - gcc-multilib
-
-
 
 script:
   - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-
   #  cron & master          => long tests, as this is the final step towards a Release
-  #  dev                    => normal tests
+  #  dev && pull requests   => normal tests
-  #  other feature branches => short tests (number > 10)
+  #  other feature branches => short tests (number > 5)
   - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
-        FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1;
+        FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
     else
-        if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then
+        if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then
             sh -c "$Cmd" || travis_terminate 1;
         fi
     fi

Makefile

@@ -128,6 +128,12 @@ ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
+arm-ppc-compilation:
+	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
+
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
 
@@ -143,6 +149,34 @@ asan32: clean
 uasan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
 
+uasan-%: clean
+	LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $*
+
+apt-install:
+	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
+
+apt-add-repo:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+
+ppcinstall:
+	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
+
+arminstall:
+	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
+
+valgrindinstall: 
+	APT_PACKAGES="valgrind" $(MAKE) apt-install
+
+libc6install:
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
+
+gcc6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
+
+gpp6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
+
 endif
 
 

contrib/pzstd/Makefile

@@ -85,6 +85,23 @@ endif
 .PHONY: default
 default: all
 
+.PHONY: test-pzstd
+test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd: clean googletest pzstd tests check
+
+.PHONY: test-pzstd32
+test-pzstd32: clean googletest32 all32 check
+
+.PHONY: test-pzstd-tsan
+test-pzstd-tsan: LDFLAGS=-fuse-ld=gold
+test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-tsan: clean googletest tsan check
+
+.PHONY: test-pzstd-asan
+test-pzstd-asan: LDFLAGS=-fuse-ld=gold
+test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-asan: clean asan check
+
 .PHONY: check
 check:
 	$(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
@@ -117,7 +134,7 @@ debug: pzstd$(EXT) tests roundtrip
 
 .PHONY: tsan
 tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
-tsan: PZSTD_LDFLAGS   += -fsanitize=thread -pie
+tsan: PZSTD_LDFLAGS   += -fsanitize=thread
 tsan: debug
 
 .PHONY: asan

examples/Makefile

@@ -9,7 +9,7 @@
 
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
-LDFLAGS+= -lzstd
+LDFLAGS += -lzstd
 
 .PHONY: default all clean test
 
@@ -52,16 +52,23 @@ clean:
 test: all
 	cp README.md tmp
 	cp Makefile tmp2
-	@echo starting simple compression
+	@echo -- Simple compression tests
 	./simple_compression tmp
 	./simple_decompression tmp.zst
 	./streaming_decompression tmp.zst > /dev/null
-	@echo starting streaming compression
+	@echo -- Streaming compression tests
 	./streaming_compression tmp
 	./streaming_decompression tmp.zst > /dev/null
-	@echo starting multiple streaming compression
+	@echo -- Edge cases detection
+	! ./streaming_decompression tmp    # invalid input, must fail
+	! ./simple_decompression tmp       # invalid input, must fail
+	! ./simple_decompression tmp.zst   # unknown input size, must fail
+	touch tmpNull                      # create 0-size file
+	./simple_compression tmpNull
+	./simple_decompression tmpNull.zst # 0-size frame : must work
+	@echo -- Multiple streaming tests
 	./multiple_streaming_compression *.c
-	@echo starting dictionary compression
+	@echo -- Dictionary compression tests
 	./dictionary_compression tmp2 tmp README.md
 	./dictionary_decompression tmp2.zst tmp.zst README.md
 	$(RM) tmp* *.zst

examples/dictionary_decompression.c

@@ -78,10 +78,14 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
     size_t cSize;
     void* const cBuff = loadFile_orDie(fname, &cSize);
     unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
-    if (rSize==0) {
+    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
+        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
+        exit(5);
+    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
         fprintf(stderr, "%s : original size unknown \n", fname);
         exit(6);
     }
+
     void* const rBuff = malloc_orDie((size_t)rSize);
 
     ZSTD_DCtx* const dctx = ZSTD_createDCtx();

examples/simple_decompression.c

@@ -20,7 +20,7 @@ static off_t fsize_orDie(const char *filename)
     struct stat st;
     if (stat(filename, &st) == 0) return st.st_size;
     /* error */
-    printf("stat: %s : %s \n", filename, strerror(errno));
+    fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno));
     exit(1);
 }
 
@@ -29,16 +29,16 @@ static FILE* fopen_orDie(const char *filename, const char *instruction)
     FILE* const inFile = fopen(filename, instruction);
     if (inFile) return inFile;
     /* error */
-    printf("fopen: %s : %s \n", filename, strerror(errno));
+    fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno));
     exit(2);
 }
 
 static void* malloc_orDie(size_t size)
 {
-    void* const buff = malloc(size);
+    void* const buff = malloc(size + !size);   /* avoid allocating size of 0 : may return NULL (implementation dependent) */
     if (buff) return buff;
     /* error */
-    printf("malloc: %s \n", strerror(errno));
+    fprintf(stderr, "malloc: %s \n", strerror(errno));
     exit(3);
 }
 
@@ -49,7 +49,7 @@ static void* loadFile_orDie(const char* fileName, size_t* size)
     void* const buffer = malloc_orDie(buffSize);
     size_t const readSize = fread(buffer, 1, buffSize, inFile);
     if (readSize != (size_t)buffSize) {
-        printf("fread: %s : %s \n", fileName, strerror(errno));
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
         exit(4);
     }
     fclose(inFile);   /* can't fail (read only) */
@@ -63,16 +63,21 @@ static void decompress(const char* fname)
     size_t cSize;
     void* const cBuff = loadFile_orDie(fname, &cSize);
     unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
-    if (rSize==0) {
+    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
-        printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
+        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
         exit(5);
+    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
+        fprintf(stderr,
+                "%s : original size unknown. Use streaming decompression instead.\n", fname);
+        exit(6);
     }
+
     void* const rBuff = malloc_orDie((size_t)rSize);
 
     size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
 
     if (dSize != rSize) {
-        printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
+        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
         exit(7);
     }
 

lib/compress/fse_compress.c

@@ -506,6 +506,7 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
 
 static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
 {
+    short const NOT_YET_ASSIGNED = -2;
     U32 s;
     U32 distributed = 0;
     U32 ToDistribute;
@@ -531,7 +532,8 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
             total -= count[s];
             continue;
         }
-        norm[s]=-2;
+
+        norm[s]=NOT_YET_ASSIGNED;
     }
     ToDistribute = (1 << tableLog) - distributed;
 
@@ -539,7 +541,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
         /* risk of rounding to zero */
         lowOne = (U32)((total * 3) / (ToDistribute * 2));
         for (s=0; s<=maxSymbolValue; s++) {
-            if ((norm[s] == -2) && (count[s] <= lowOne)) {
+            if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
                 norm[s] = 1;
                 distributed++;
                 total -= count[s];
@@ -559,12 +561,19 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
         return 0;
     }
 
+    if (total == 0) {
+        /* all of the symbols were low enough for the lowOne or lowThreshold */
+        for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
+            if (norm[s] > 0) ToDistribute--, norm[s]++;
+        return 0;
+    }
+
     {   U64 const vStepLog = 62 - tableLog;
         U64 const mid = (1ULL << (vStepLog-1)) - 1;
         U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total;   /* scale on remaining */
         U64 tmpTotal = mid;
         for (s=0; s<=maxSymbolValue; s++) {
-            if (norm[s]==-2) {
+            if (norm[s]==NOT_YET_ASSIGNED) {
                 U64 const end = tmpTotal + (count[s] * rStep);
                 U32 const sStart = (U32)(tmpTotal >> vStepLog);
                 U32 const sEnd = (U32)(end >> vStepLog);

lib/compress/zstd_compress.c

@@ -2786,7 +2786,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
 
         if (!cdict || !cctx) {
             ZSTD_free(cdict, customMem);
-            ZSTD_free(cctx, customMem);
+            ZSTD_freeCCtx(cctx);
             return NULL;
         }
 
@@ -2804,8 +2804,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
         {   size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
             if (ZSTD_isError(errorCode)) {
                 ZSTD_free(cdict->dictBuffer, customMem);
-                ZSTD_free(cctx, customMem);
                 ZSTD_free(cdict, customMem);
+                ZSTD_freeCCtx(cctx);
                 return NULL;
         }   }
 

programs/Makefile

@@ -148,7 +148,7 @@ generate_res:
 	windres/generate_res.bat
 
 clean:
-	$(MAKE) -C ../lib clean
+	$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
 	@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
         zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \

programs/zstdcli.c

@@ -399,7 +399,7 @@ int main(int argCount, const char* argv[])
                 while (argument[0]!=0) {
                     if (lastCommand) {
                         DISPLAY("error : command must be followed by argument \n");
-                        return 1;
+                        CLEAN_RETURN(1);
                     }
 #ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
@@ -555,7 +555,7 @@ int main(int argCount, const char* argv[])
         filenameTable[filenameIdx++] = argument;
     }
 
-    if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; }  /* forgotten argument */
+    if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); }  /* forgotten argument */
 
     /* Welcome message (if verbose) */
     DISPLAYLEVEL(3, WELCOME_MESSAGE);

tests/Makefile

@@ -170,11 +170,11 @@ namespaceTest:
 	if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
 	$(RM) $@
 
-versionsTest:
+versionsTest: clean
 	$(PYTHON) test-zstd-versions.py
 
 clean:
-	$(MAKE) -C ../lib clean
+	$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) -fR $(TESTARTEFACT)
 	@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
         $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \

tests/legacy.c

@@ -65,6 +65,7 @@ int testSimpleAPI(void)
         return 1;
     }
 
+    free(output);
     DISPLAY("Simple API OK\n");
     return 0;
 }
@@ -118,6 +119,8 @@ int testStreamingAPI(void)
         }
     }
 
+    free(outBuff);
+    ZSTD_freeDStream(stream);
     DISPLAY("Streaming API OK\n");
     return 0;
 }

tests/zstreamtest.c

@@ -496,6 +496,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
             /* Bug will cause checksum to fail */
             if (ZSTD_isError(r)) goto _output_error;
         }
+
+        ZSTD_freeDStream(zds);
     }
     DISPLAYLEVEL(3, "OK \n");