Integrated Legacy v0.5.x decoder (provided by @inikep)

lib/Makefile

@@ -52,7 +52,7 @@ LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR=$(PREFIX)/include
 
 ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c zdict.c divsufsort.c
-ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c
+ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
 
 ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0

lib/legacy/zstd_legacy.h

@@ -46,6 +46,7 @@ extern "C" {
 #include "zstd_v02.h"
 #include "zstd_v03.h"
 #include "zstd_v04.h"
+#include "zstd_v05.h"
 
 MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
 {
@@ -53,28 +54,32 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
 	{
 		case ZSTDv01_magicNumberLE :
 		case ZSTDv02_magicNumber :
-		case ZSTDv03_magicNumber : 
+		case ZSTDv03_magicNumber :
-		case ZSTDv04_magicNumber : return 1;
+		case ZSTDv04_magicNumber :
+		case ZSTDv05_MAGICNUMBER :
+            return 1;
 		default : return 0;
 	}
 }
 
 
 MEM_STATIC size_t ZSTD_decompressLegacy(
-                     void* dst, size_t maxOriginalSize,
+                     void* dst, size_t dstCapacity,
                const void* src, size_t compressedSize,
                      U32 magicNumberLE)
 {
 	switch(magicNumberLE)
 	{
 		case ZSTDv01_magicNumberLE :
-			return ZSTDv01_decompress(dst, maxOriginalSize, src, compressedSize);
+			return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
 		case ZSTDv02_magicNumber :
-			return ZSTDv02_decompress(dst, maxOriginalSize, src, compressedSize);
+			return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
 		case ZSTDv03_magicNumber :
-			return ZSTDv03_decompress(dst, maxOriginalSize, src, compressedSize);
+			return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
 		case ZSTDv04_magicNumber :
-			return ZSTDv04_decompress(dst, maxOriginalSize, src, compressedSize);
+			return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
+		case ZSTDv05_MAGICNUMBER :
+			return ZSTDv05_decompress(dst, dstCapacity, src, compressedSize);
 		default :
 		    return ERROR(prefix_unknown);
 	}

lib/legacy/zstd_v04.h

@@ -95,7 +95,7 @@ size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSi
 ***************************************/
 typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx;
 ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void);
-size_t      ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);
+size_t         ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);
 
 size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx);
 size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize);

lib/legacy/zstd_v05.h

@@ -0,0 +1,156 @@
+/*
+    zstd_v05 - decoder for 0.5 format
+    Header File
+    Copyright (C) 2014-2016, Yann Collet.
+
+    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+    - zstd source repository : https://github.com/Cyan4973/zstd
+*/
+#ifndef ZSTDv05_H
+#define ZSTDv05_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+*  Dependencies
+***************************************/
+#include <stddef.h>   /* size_t */
+
+
+
+/* *************************************
+*  Simple functions
+***************************************/
+/*! ZSTDv05_decompress() :
+    `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.
+    `dstCapacity` must be large enough, equal or larger than originalSize.
+    @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
+              or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */
+size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
+                     const void* src, size_t compressedSize);
+
+
+/* *************************************
+*  Helper functions
+***************************************/
+/* Error Management */
+unsigned    ZSTDv05_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
+const char* ZSTDv05_getErrorName(size_t code);     /*!< provides readable string for an error code */
+
+
+/* *************************************
+*  Explicit memory management
+***************************************/
+/** Decompression context */
+typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx;
+ZSTDv05_DCtx* ZSTDv05_createDCtx(void);
+size_t     ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx);      /*!< @return : errorCode */
+
+/** ZSTDv05_decompressDCtx() :
+*   Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */
+size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+
+/*-***********************
+*  Dictionary API
+*************************/
+/*! ZSTDv05_decompress_usingDict() :
+*   Decompression using a pre-defined Dictionary content (see dictBuilder).
+*   Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
+*   Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */
+size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx,
+                                             void* dst, size_t dstCapacity,
+                                       const void* src, size_t srcSize,
+                                       const void* dict,size_t dictSize);
+
+
+
+
+
+
+typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx;
+ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void);
+size_t      ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);
+
+size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx);
+size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize);
+
+size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx,
+                                            void* dst, size_t* dstCapacityPtr,
+                                      const void* src, size_t* srcSizePtr);
+
+/*-***************************************************************************
+*  Streaming decompression
+*
+*  A ZBUFFv05_DCtx object is required to track streaming operations.
+*  Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources.
+*  Use ZBUFFv05_decompressInit() to start a new decompression operation,
+*   or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary.
+*  Note that ZBUFFv05_DCtx objects can be reused multiple times.
+*
+*  Use ZBUFFv05_decompressContinue() repetitively to consume your input.
+*  *srcSizePtr and *dstCapacityPtr can be any size.
+*  The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
+*  Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
+*  The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst.
+*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency)
+*            or 0 when a frame is completely decoded
+*            or an error code, which can be tested using ZBUFFv05_isError().
+*
+*  Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize()
+*  output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
+*  input  : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
+* *******************************************************************************/
+
+
+/* *************************************
+*  Tool functions
+***************************************/
+unsigned ZBUFFv05_isError(size_t errorCode);
+const char* ZBUFFv05_getErrorName(size_t errorCode);
+
+/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
+*   These sizes are just hints, and tend to offer better latency */
+size_t ZBUFFv05_recommendedDInSize(void);
+size_t ZBUFFv05_recommendedDOutSize(void);
+
+
+
+/*-*************************************
+*  Constants
+***************************************/
+#define ZSTDv05_MAGICNUMBER 0xFD2FB525   /* v0.5 */
+
+
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif  /* ZSTDv0505_H */

lib/zstd_decompress.c

@@ -136,7 +136,7 @@ struct ZSTD_DCtx_s
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
 };  /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
 
-size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }
+size_t ZSTD_sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }   /* non published interface */
 
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
 {

programs/Makefile

@@ -61,7 +61,8 @@ ZSTD_FILES_LEGACY:=
 else
 ZSTD_LEGACY_SUPPORT:=1
 CPPFLAGS  += -I../lib/legacy -I./legacy
-ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c $(ZSTDDIR)/legacy/zstd_v04.c legacy/fileio_legacy.c
+ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
+                    $(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c
 endif
 
 

programs/legacy/fileio_legacy.c

@@ -1,6 +1,6 @@
 /*
-  fileio.c - File i/o handler
+  fileio_legacy.c - File i/o handler for legacy format
-  Copyright (C) Yann Collet 2013-2015
+  Copyright (C) Yann Collet 2015-2016
 
   GPL v2 License
 
@@ -19,12 +19,11 @@
   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
   You can contact the author at :
+  - zstd homepage : http://www.zstd.net
   - zstd source repository : https://github.com/Cyan4973/zstd
-  - Public forum : https://groups.google.com/forum/#!forum/lz4c
 */
 /*
-  Note : this is stand-alone program.
+  Note : this file is not part of ZSTD compression library.
-  It is not part of ZSTD compression library, it is a user program of ZSTD library.
   The license of ZSTD library is BSD.
   The license of this file is GPLv2.
 */
@@ -172,8 +171,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
 
     /* Main decompression Loop */
     toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
-    while (toRead)
+    while (toRead){
-    {
         size_t readSize, decodedSize;
 
         /* Fill input buffer */
@@ -187,8 +185,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
         decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
         if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
 
-        if (decodedSize)   /* not a header */
+        if (decodedSize) {  /* not a header */
-        {
             /* Write block */
             sizeCheck = fwrite(op, 1, decodedSize, foutput);
             if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@@ -232,8 +229,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
 
     /* Main decompression Loop */
     toRead = ZSTDv02_nextSrcSizeToDecompress(dctx);
-    while (toRead)
+    while (toRead) {
-    {
         size_t readSize, decodedSize;
 
         /* Fill input buffer */
@@ -247,8 +243,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
         decodedSize = ZSTDv02_decompressContinue(dctx, op, oend-op, inBuff, readSize);
         if (ZSTDv02_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
 
-        if (decodedSize)   /* not a header */
+        if (decodedSize) {   /* not a header */
-        {
             /* Write block */
             sizeCheck = fwrite(op, 1, decodedSize, foutput);
             if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@@ -292,8 +287,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
 
     /* Main decompression Loop */
     toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
-    while (toRead)
+    while (toRead) {
-    {
         size_t readSize, decodedSize;
 
         /* Fill input buffer */
@@ -307,8 +301,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
         decodedSize = ZSTDv03_decompressContinue(dctx, op, oend-op, inBuff, readSize);
         if (ZSTDv03_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
 
-        if (decodedSize)   /* not a header */
+        if (decodedSize) {   /* not a header */
-        {
             /* Write block */
             sizeCheck = fwrite(op, 1, decodedSize, foutput);
             if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@@ -329,7 +322,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
 }
 
 
-/*- v0.4.x -*/
+/*=====    v0.4.x    =====*/
 
 typedef struct {
     void*  srcBuffer;
@@ -380,8 +373,7 @@ unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
     ZBUFFv04_decompressInit(ress.dctx);
     ZBUFFv04_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
 
-    while (1)
+    while (1) {
-    {
         /* Decode */
         size_t sizeCheck;
         size_t inSize=readSize, decodedSize=ress.dstBufferSize;
@@ -404,11 +396,89 @@ unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
         if (readSize != toRead) EXM_THROW(35, "Read error");
     }
 
-    FIOv04_freeDResources(ress);
     return frameSize;
 }
 
 
+/*=====    v0.5.x    =====*/
+
+typedef struct {
+    void*  srcBuffer;
+    size_t srcBufferSize;
+    void*  dstBuffer;
+    size_t dstBufferSize;
+    void*  dictBuffer;
+    size_t dictBufferSize;
+    ZBUFFv05_DCtx* dctx;
+} dRessv05_t;
+
+static dRessv05_t FIOv05_createDResources(void)
+{
+    dRessv05_t ress;
+
+    /* init */
+    ress.dctx = ZBUFFv05_createDCtx();
+    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
+    ress.dictBuffer = NULL; ress.dictBufferSize=0;
+
+    /* Allocate Memory */
+    ress.srcBufferSize = ZBUFFv05_recommendedDInSize();
+    ress.srcBuffer = malloc(ress.srcBufferSize);
+    ress.dstBufferSize = ZBUFFv05_recommendedDOutSize();
+    ress.dstBuffer = malloc(ress.dstBufferSize);
+    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+
+    return ress;
+}
+
+static void FIOv05_freeDResources(dRessv05_t ress)
+{
+    size_t const errorCode = ZBUFFv05_freeDCtx(ress.dctx);
+    if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode));
+    free(ress.srcBuffer);
+    free(ress.dstBuffer);
+    free(ress.dictBuffer);
+}
+
+
+unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
+                                          FILE* foutput, FILE* finput)
+{
+    U64    frameSize = 0;
+    size_t readSize = 4;
+
+    MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER);
+    ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
+
+    while (1) {
+        /* Decode */
+        size_t sizeCheck;
+        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
+        size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
+        if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead));
+        readSize -= inSize;
+
+        /* Write block */
+        sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
+        if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
+        frameSize += decodedSize;
+        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
+
+        if (toRead == 0) break;
+        if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
+
+        /* Fill input buffer */
+        if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
+        readSize = fread(ress.srcBuffer, 1, toRead, finput);
+        if (readSize != toRead) EXM_THROW(35, "Read error");
+    }
+
+    return frameSize;
+}
+
+
+/*=====   General legacy dispatcher   =====*/
+
 unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 magicNumberLE)
 {
 	switch(magicNumberLE)
@@ -420,7 +490,17 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 ma
 		case ZSTDv03_magicNumber :
 			return FIOv03_decompressFrame(foutput, finput);
 		case ZSTDv04_magicNumber :
-			return FIOv04_decompressFrame(FIOv04_createDResources(), foutput, finput);
+		    {   dRessv04_t r = FIOv04_createDResources();
+                unsigned long long const s = FIOv04_decompressFrame(r, foutput, finput);
+                FIOv04_freeDResources(r);
+                return s;
+		    }
+		case ZSTDv05_MAGICNUMBER :
+		    {   dRessv05_t r = FIOv05_createDResources();
+                unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput);
+                FIOv05_freeDResources(r);
+                return s;
+		    }
 		default :
 		    return ERROR(prefix_unknown);
 	}