Merge branch 'dev' of github.com:facebook/zstd into dev

NEWS

@@ -1,8 +1,12 @@
 v1.3.2
+new : long range mode, using --long command, by Stella Lau (@stellamplau)
 license : changed /examples license to BSD + GPLv2
 license : fix a few header files to reflect new license (#825)
-fix : 32-bits build can now decode large offsets (levels 21+)
+fix : multi-threading compression works with custom allocators
 fix : a rare compression bug when compression generates very large distances (only possible at --ultra -22)
+fix : 32-bits build can now decode large offsets (levels 21+)
+cli : new : can split input file for dictionary training, using command -B#
+cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
 build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
 build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
 

programs/fileio.c

@@ -105,9 +105,6 @@ static clock_t g_time = 0;
 #define MIN(a,b)    ((a) < (b) ? (a) : (b))
 
 
-/*-*************************************
-*  Errors
-***************************************/
 /*-*************************************
 *  Debug
 ***************************************/
@@ -1023,8 +1020,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
  ***************************************************************************/
 typedef struct {
     void*  srcBuffer;
-    size_t srcBufferLoaded;
     size_t srcBufferSize;
+    size_t srcBufferLoaded;
     void*  dstBuffer;
     size_t dstBufferSize;
     ZSTD_DStream* dctx;
@@ -1560,7 +1557,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 
     /* Close file */
     if (fclose(srcFile)) {
-        DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));  /* error should never happen */
+        DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));  /* error should not happen */
         return 1;
     }
     if ( g_removeSrcFile /* --rm */
@@ -1590,7 +1587,8 @@ static int FIO_decompressDstFile(dRess_t ress,
     ress.dstFile = FIO_openDstFile(dstFileName);
     if (ress.dstFile==0) return 1;
 
-    if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf))
+    if ( strcmp(srcFileName, stdinmark)
+      && UTIL_getFileStat(srcFileName, &statbuf) )
         stat_result = 1;
     result = FIO_decompressSrcFile(ress, dstFileName, srcFileName);
 
@@ -1601,10 +1599,14 @@ static int FIO_decompressDstFile(dRess_t ress,
 
     if ( (result != 0)  /* operation failure */
       && strcmp(dstFileName, nulmark)      /* special case : don't remove() /dev/null (#316) */
-       && remove(dstFileName) /* remove artefact */ )
+      && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */
-        result=1;   /* don't do anything special if remove() fails */
+        remove(dstFileName);  /* remove decompression artefact; note don't do anything special if remove() fails */
-    else if (strcmp (dstFileName, stdoutmark) && stat_result)
+    else {  /* operation success */
-        UTIL_setFileStat(dstFileName, &statbuf);
+        if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */
+          && strcmp(dstFileName, nulmark)    /* special case : don't chmod /dev/null */
+          && stat_result )                   /* file permissions correctly extracted from src */
+            UTIL_setFileStat(dstFileName, &statbuf);  /* transfer file permissions from src into dst */
+    }
     return result;
 }
 

programs/zstdcli.c

@@ -313,8 +313,8 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
         if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
-        if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
-        if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         return 0;
     }
 

tests/playTests.sh

@@ -161,6 +161,8 @@ roundTripTest -g512K
 roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6"
 roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
 roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6"
+roundTripTest -g512K " --long --zstd=ldmHashLog=20,ldmSearchLength=64,ldmBucketSizeLog=1,ldmHashEveryLog=7"
+roundTripTest -g512K " --long --zstd=ldmhlog=20,ldmslen=64,ldmblog=1,ldmhevery=7"
 roundTripTest -g512K 19