Merge remote-tracking branch 'upstream/dev' into dev

* upstream/dev:
  Streaming : dictionary compression on multiple files / segments can correctly provide srcSize into header (when provided) using pledgedSrcSize.
  cli : better adaptation to small files
  fixed : cli : forgotten mandatory argument
  Implemented "command must be followed by argument" protection suggested by @terrelln (#375)
  minor refactor for clarity
  minor build comment
  gmake necessary on *BSD systems.
  added error check
  added error checking
  fixed cmake test
  completed change from projects to build
  changed projects to build
  bumped version number
  streaming compression example can handle situations where input buffer size is manually set to a small value.
  added comments on buffer sizes guarantees

Makefile

@@ -9,6 +9,7 @@
 
 PRGDIR   = programs
 ZSTDDIR  = lib
+BUILDIR  = build
 ZWRAPDIR = zlibWrapper
 TESTDIR  = tests
 
@@ -121,9 +122,9 @@ endif
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
 cmaketest:
 	cmake --version
-	$(RM) -r projects/cmake/build
-	mkdir projects/cmake/build
-	cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
+	$(RM) -r $(BUILDIR)/cmake/build
+	mkdir $(BUILDIR)/cmake/build
+	cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 
 c90test: clean
 	CFLAGS="-std=c90" $(MAKE) all  # will fail, due to // and long long

NEWS

@@ -1,7 +1,7 @@
-v1.0.1
+v1.1.0
 New : contrib/pzstd, parallel version of zstd, by Nick Terrell
 added : NetBSD install target (#338)
-Improved : variable compression speed improvements on batches of small files.
+Improved : speed improvements for batches of small files.
 Fixed : CLI -d output to stdout by default when input is stdin (#322)
 Fixed : CLI correctly detects console on Mac OS-X
 Fixed : CLI supports recursive mode `-r` on Mac OS-X

README.md

@@ -73,6 +73,35 @@ Hence, deploying one dictionary per type of data will provide the greatest benef
 
 `zstd --decompress FILE.zst -D dictionaryName`
 
+### Build
+
+Once you have the repository cloned, there are multiple ways provided to build Zstandard.
+
+#### Makefile
+
+If your system is compatible with a standard `make` (or `gmake`) binary generator,
+you can simply run it at the root directory.
+It will generate `zstd` within root directory.
+
+Other available options include :
+- `make install` : create and install zstd binary, library and man page
+- `make test` : create and run `zstd` and test tools on local platform
+
+#### cmake
+
+A `cmake` project generator is provided within `build/cmake`.
+It can generate Makefiles or other build scripts
+to create `zstd` binary, and `libzstd` dynamic and static libraries.
+
+#### Visual (Windows)
+
+Going into `build` directory, you will find additional possibilities :
+- Projects for Visual Studio 2005, 2008 and 2010
+  + VS2010 project is compatible with VS2012, VS2013 and VS2015
+- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,
+  which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
+
+
 ### Status
 
 Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amounts of data in multiple formats and use cases.

appveyor.yml

@@ -76,51 +76,51 @@ build_script:
       ECHO *** &&
       ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
       ECHO *** &&
-      msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
+      msbuild "build\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\projects\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
-      DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
-      MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe tests\
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe tests\
     )
 
 test_script:

build/README.md

@@ -8,7 +8,7 @@ The following projects are included with the zstd distribution:
 - `VS2005` - Visual Studio 2005 project
 - `VS2008` - Visual Studio 2008 project
 - `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
-- `build` - command line scripts prepared for Visual Studio compilation without IDE
+- `VS_scripts` - command line scripts prepared for Visual Studio compilation without IDE
 
 
 #### How to compile zstd with Visual Studio

build/cmake/lib/CMakeLists.txt

@@ -108,7 +108,7 @@ IF (ZSTD_LEGACY_SUPPORT)
 ENDIF (ZSTD_LEGACY_SUPPORT)
 
 IF (MSVC)
-    SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib)
+    SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/zstdlib)
     SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc)
 ENDIF (MSVC)
 

examples/streaming_compression.c

@@ -65,29 +65,31 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
 {
     FILE* const fin  = fopen_orDie(fname, "rb");
     FILE* const fout = fopen_orDie(outName, "wb");
-    size_t const buffInSize = ZSTD_CStreamInSize();;
+    size_t const buffInSize = ZSTD_CStreamInSize();    /* can always read one full block */
     void*  const buffIn  = malloc_orDie(buffInSize);
-    size_t const buffOutSize = ZSTD_CStreamOutSize();;
+    size_t const buffOutSize = ZSTD_CStreamOutSize();  /* can always flush a full block */
     void*  const buffOut = malloc_orDie(buffOutSize);
 
     ZSTD_CStream* const cstream = ZSTD_createCStream();
     if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
     size_t const initResult = ZSTD_initCStream(cstream, cLevel);
-    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error \n"); exit(11); }
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
 
     size_t read, toRead = buffInSize;
     while( (read = fread_orDie(buffIn, toRead, fin)) ) {
         ZSTD_inBuffer input = { buffIn, read, 0 };
         while (input.pos < input.size) {
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
-            toRead = ZSTD_compressStream(cstream, &output , &input);
+            toRead = ZSTD_compressStream(cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
+            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            if (toRead > buffInSize) toRead = buffInSize;   /* Safely handle when `buffInSize` is manually changed to a smaller value */
             fwrite_orDie(buffOut, output.pos, fout);
         }
     }
 
     ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
     size_t const remainingToFlush = ZSTD_endStream(cstream, &output);   /* close frame */
-    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(12); }
+    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
     fwrite_orDie(buffOut, output.pos, fout);
 
     ZSTD_freeCStream(cstream);

examples/streaming_decompression.c

@@ -80,6 +80,7 @@ static void decompressFile_orDie(const char* fname)
         while (input.pos < input.size) {
             ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
             toRead = ZSTD_decompressStream(dstream, &output , &input);  /* toRead : size of next compressed block */
+            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_decompressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
             fwrite_orDie(buffOut, output.pos, fout);
         }
     }

lib/compress/zstd_compress.c

@@ -322,13 +322,12 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
 *   Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
 *   Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()).
 *   @return : 0, or an error code */
-size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
+size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
 {
     if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
 
     memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
-    ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, ZSTDcrp_noMemset);
-    dstCCtx->params.fParams.contentSizeFlag = 0;   /* content size different from the one set during srcCCtx init */
+    ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset);
 
     /* copy tables */
     {   size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
@@ -2730,7 +2729,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
 size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
 {
     if (cdict==NULL) return 0;   /* support free on NULL */
-    {   ZSTD_customMem cMem = cdict->refContext->customMem;
+    {   ZSTD_customMem const cMem = cdict->refContext->customMem;
         ZSTD_freeCCtx(cdict->refContext);
         ZSTD_free(cdict->dictContent, cMem);
         ZSTD_free(cdict, cMem);
@@ -2740,7 +2739,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
 
 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
 {
-    if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
+    if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
     else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
     return 0;
 }

lib/dictBuilder/zdict.c

@@ -563,7 +563,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params,
     size_t cSize;
 
     if (srcSize > blockSizeMax) srcSize = blockSizeMax;   /* protection vs large samples */
-    {  size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
+    {  size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0);
             if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
     }
     cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize);

lib/zstd.h

@@ -32,8 +32,8 @@ extern "C" {
 
 /*=======   Version   =======*/
 #define ZSTD_VERSION_MAJOR    1
-#define ZSTD_VERSION_MINOR    0
-#define ZSTD_VERSION_RELEASE  1
+#define ZSTD_VERSION_MINOR    1
+#define ZSTD_VERSION_RELEASE  0
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -447,7 +447,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
-ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
+ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
 
 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);

programs/Makefile

@@ -57,8 +57,8 @@ endif
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
 VOID = nul
-RES64_FILE = ..\projects\VS2010\zstd\generate_res\zstd64.res
-RES32_FILE = ..\projects\VS2010\zstd\generate_res\zstd32.res
+RES64_FILE = ..\build\VS2010\zstd\generate_res\zstd64.res
+RES32_FILE = ..\build\VS2010\zstd\generate_res\zstd32.res
 ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
     RES_FILE = $(RES64_FILE)
 else

programs/fileio.c

@@ -255,7 +255,7 @@ typedef struct {
     FILE* srcFile;
 } cRess_t;
 
-static cRess_t FIO_createCResources(const char* dictFileName, int cLevel)
+static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize)
 {
     cRess_t ress;
     memset(&ress, 0, sizeof(ress));
@@ -272,11 +272,11 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel)
     {   void* dictBuffer;
         size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName);
         if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer");
-        {   ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictBuffSize);
+        {   ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
             params.fParams.contentSizeFlag = 1;
             params.fParams.checksumFlag = g_checksumFlag;
             params.fParams.noDictIDFlag = !g_dictIDFlag;
-            {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, 0);
+            {   size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize);
                 if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode));
         }   }
         free(dictBuffer);
@@ -408,8 +408,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
                          const char* dictFileName, int compressionLevel)
 {
     clock_t const start = clock();
+    U64 const srcSize = UTIL_getFileSize(srcFileName);
 
-    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel);
+    cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
     int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName);
 
     double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
@@ -428,7 +429,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
     size_t dfnSize = FNSPACE;
     char*  dstFileName = (char*)malloc(FNSPACE);
     size_t const suffixSize = suffix ? strlen(suffix) : 0;
-    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel);
+    U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
+    cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize);
 
     /* init */
     if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName");

programs/zstdcli.c

@@ -23,7 +23,7 @@
 #endif
 
 #ifndef ZSTDCLI_CLEVEL_MAX
-#  define ZSTDCLI_CLEVEL_MAX 19
+#  define ZSTDCLI_CLEVEL_MAX 19   /* when not using --ultra */
 #endif
 
 
@@ -51,14 +51,12 @@
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
 #  include <io.h>       /* _isatty */
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
-#else
-#  if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__))  /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
+#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__))  /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
 #  include <unistd.h>   /* isatty */
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
 #else
 #  define IS_CONSOLE(stdStream) 0
 #endif
-#endif
 
 
 /*-************************************
@@ -195,7 +193,7 @@ static unsigned readU32FromChar(const char** stringPtr)
 
 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
 
-int main(int argCount, char** argv)
+int main(int argCount, const char* argv[])
 {
     int argNb,
         bench=0,
@@ -210,7 +208,8 @@ int main(int argCount, char** argv)
         nextArgumentIsMaxDict=0,
         nextArgumentIsDictID=0,
         nextArgumentIsFile=0,
-        ultra=0;
+        ultra=0,
+        lastCommand = 0;
     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
     int cLevelLast = 1;
     unsigned recursive = 0;
@@ -219,13 +218,12 @@ int main(int argCount, char** argv)
     const char* programName = argv[0];
     const char* outFileName = NULL;
     const char* dictFileName = NULL;
-    char* dynNameSpace = NULL;
     unsigned maxDictSize = g_defaultMaxDictSize;
     unsigned dictID = 0;
     int dictCLevel = g_defaultDictCLevel;
     unsigned dictSelect = g_defaultSelectivityLevel;
 #ifdef UTIL_HAS_CREATEFILELIST
-    const char** fileNamesTable = NULL;
+    const char** extendedFileList = NULL;
     char* fileNamesBuf = NULL;
     unsigned fileNamesNb;
 #endif
@@ -271,8 +269,8 @@ int main(int argCount, char** argv)
             if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
             if (!strcmp(argument, "--test")) { testmode=1; decode=1; continue; }
             if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
-            if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
-            if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
+            if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }
+            if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }
             if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
             if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
 
@@ -290,6 +288,10 @@ int main(int argCount, char** argv)
                 argument++;
 
                 while (argument[0]!=0) {
+                    if (lastCommand) {
+                        DISPLAY("error : command must be followed by argument \n");
+                        return 1;
+                    }
 #ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
                     if ((*argument>='0') && (*argument<='9')) {
@@ -312,7 +314,7 @@ int main(int argCount, char** argv)
                     case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
 
                         /* Use file content as dictionary */
-                    case 'D': nextEntryIsDictionary = 1; argument++; break;
+                    case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
 
                         /* Overwrite */
                     case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break;
@@ -333,7 +335,7 @@ int main(int argCount, char** argv)
                     case 't': testmode=1; decode=1; argument++; break;
 
                         /* destination file name */
-                    case 'o': nextArgumentIsOutFileName=1; argument++; break;
+                    case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
 
 #ifdef UTIL_HAS_CREATEFILELIST
                         /* recursive */
@@ -397,6 +399,7 @@ int main(int argCount, char** argv)
 
             if (nextArgumentIsMaxDict) {
                 nextArgumentIsMaxDict = 0;
+                lastCommand = 0;
                 maxDictSize = readU32FromChar(&argument);
                 if (toupper(*argument)=='K') maxDictSize <<= 10;
                 if (toupper(*argument)=='M') maxDictSize <<= 20;
@@ -405,6 +408,7 @@ int main(int argCount, char** argv)
 
             if (nextArgumentIsDictID) {
                 nextArgumentIsDictID = 0;
+                lastCommand = 0;
                 dictID = readU32FromChar(&argument);
                 continue;
             }
@@ -413,12 +417,14 @@ int main(int argCount, char** argv)
 
         if (nextEntryIsDictionary) {
             nextEntryIsDictionary = 0;
+            lastCommand = 0;
             dictFileName = argument;
             continue;
         }
 
         if (nextArgumentIsOutFileName) {
             nextArgumentIsOutFileName = 0;
+            lastCommand = 0;
             outFileName = argument;
             if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
             continue;
@@ -428,17 +434,19 @@ int main(int argCount, char** argv)
         filenameTable[filenameIdx++] = argument;
     }
 
+    if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; }  /* forgotten argument */
+
     /* Welcome message (if verbose) */
     DISPLAYLEVEL(3, WELCOME_MESSAGE);
 
 #ifdef UTIL_HAS_CREATEFILELIST
-    if (recursive) {
-        fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
-        if (fileNamesTable) {
+    if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
+        extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
+        if (extendedFileList) {
             unsigned u;
-            for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]);
+            for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
             free((void*)filenameTable);
-            filenameTable = fileNamesTable;
+            filenameTable = extendedFileList;
             filenameIdx = fileNamesNb;
         }
     }
@@ -495,15 +503,16 @@ int main(int argCount, char** argv)
 
     /* IO Stream/File */
     FIO_setNotificationLevel(displayLevel);
-#ifndef ZSTD_NOCOMPRESS
     if (!decode) {
-        if (filenameIdx==1 && outFileName)
+#ifndef ZSTD_NOCOMPRESS
+        if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
         else
           operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
-    } else
+#else
+        DISPLAY("Compression not supported\n");
 #endif
-    {  /* decompression */
+    } else {  /* decompression */
 #ifndef ZSTD_NODECOMPRESS
         if (testmode) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
         if (filenameIdx==1 && outFileName)
@@ -517,10 +526,9 @@ int main(int argCount, char** argv)
 
 _end:
     if (main_pause) waitEnter();
-    free(dynNameSpace);
 #ifdef UTIL_HAS_CREATEFILELIST
-    if (fileNamesTable)
-        UTIL_freeFileList(fileNamesTable, fileNamesBuf);
+    if (extendedFileList)
+        UTIL_freeFileList(extendedFileList, fileNamesBuf);
     else
 #endif
         free((void*)filenameTable);

tests/fuzzer.c

@@ -173,13 +173,13 @@ static int basicUnitTests(U32 seed, double compressibility)
         static const size_t dictSize = 551;
 
         DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
-        { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
+        { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
           if (!ZSTD_isError(copyResult)) goto _output_error; }   /* error must be detected */
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
         CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
-        CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
+        CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, CNBuffSize - dictSize) );
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
@@ -221,10 +221,10 @@ static int basicUnitTests(U32 seed, double compressibility)
                 p.fParams.contentSizeFlag = 1;
                 CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
             }
-            CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
+            CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
 
-            CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
-                                               (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
+            CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
+                                          (const char*)CNBuffer + dictSize, testSize),
                       cSize = r);
             {   ZSTD_frameParams fp;
                 if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
@@ -674,9 +674,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
                 errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
                 CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
             }
-            { size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
-              CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
-        }
+            {   size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx, 0);
+                CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
+        }   }
         XXH64_reset(&xxhState, 0);
         {   U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
             U32 n;

tests/playTests.sh

@@ -54,6 +54,17 @@ $ZSTD -99 -f tmp  # too large compression level, automatic sized down
 $ECHO "test : compress to stdout"
 $ZSTD tmp -c > tmpCompressed
 $ZSTD tmp --stdout > tmpCompressed       # long command format
+$ECHO "test : compress to named file"
+rm tmpCompressed
+$ZSTD tmp -o tmpCompressed
+ls tmpCompressed   # must work
+$ECHO "test : -o must be followed by filename (must fail)"
+$ZSTD tmp -of tmpCompressed && die "-o must be followed by filename "
+$ECHO "test : force write, correct order"
+$ZSTD tmp -fo tmpCompressed
+$ECHO "test : forgotten argument"
+cp tmp tmp2
+$ZSTD tmp2 -fo && die "-o must be followed by filename "
 $ECHO "test : implied stdout when input is stdin"
 $ECHO bob | $ZSTD | $ZSTD -d
 $ECHO "test : null-length file roundtrip"
@@ -183,18 +194,26 @@ $ECHO "- Create first dictionary"
 $ZSTD --train *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
 $ZSTD -f tmp -D tmpDict
-$ZSTD -d tmp.zst -D tmpDict -of result
+$ZSTD -d tmp.zst -D tmpDict -fo result
 diff $TESTFILE result
 $ECHO "- Create second (different) dictionary"
 $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
-$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
+$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
 $ECHO "- Create dictionary with short dictID"
 $ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1
 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "- Create dictionary with wrong dictID parameter order (must fail)"
+$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
+$ECHO "- Create dictionary with size limit"
+$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict 4K -v
+$ECHO "- Create dictionary with wrong parameter order (must fail)"
+$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
 $ECHO "- Compress without dictID"
 $ZSTD -f tmp -D tmpDict1 --no-dictID
-$ZSTD -d tmp.zst -D tmpDict -of result
+$ZSTD -d tmp.zst -D tmpDict -fo result
 diff $TESTFILE result
+$ECHO "- Compress with wrong argument order (must fail)"
+$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name "
 $ECHO "- Compress multiple files with dictionary"
 rm -rf dirTestDict
 mkdir dirTestDict