sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-17 00:07:08 -04:00
Code Issues Projects Releases Wiki Activity

Integrated Legacy v0.5.x decoder (provided by @inikep)

Browse Source
This commit is contained in:
Yann Collet 2016-04-09 09:42:27 +02:00
parent 7eff39f7ea
commit 029267ab3f
8 changed files with 5001 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/Makefile
View File

@ -52,7 +52,7 @@ LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c zdict.c divsufsort.c
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0

19
lib/legacy/zstd_legacy.h
View File

@ -46,6 +46,7 @@ extern "C" {
#include "zstd_v02.h"
#include "zstd_v03.h"
#include "zstd_v04.h"
#include "zstd_v05.h"
MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
{
@ -53,28 +54,32 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
{
case ZSTDv01_magicNumberLE :
case ZSTDv02_magicNumber :
case ZSTDv03_magicNumber :
case ZSTDv04_magicNumber : return 1;
case ZSTDv03_magicNumber :
case ZSTDv04_magicNumber :
case ZSTDv05_MAGICNUMBER :
return 1;
default : return 0;
}
}
MEM_STATIC size_t ZSTD_decompressLegacy(
void* dst, size_t maxOriginalSize,
void* dst, size_t dstCapacity,
const void* src, size_t compressedSize,
U32 magicNumberLE)
{
switch(magicNumberLE)
{
case ZSTDv01_magicNumberLE :
return ZSTDv01_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv02_magicNumber :
return ZSTDv02_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv03_magicNumber :
return ZSTDv03_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv04_magicNumber :
return ZSTDv04_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv05_MAGICNUMBER :
return ZSTDv05_decompress(dst, dstCapacity, src, compressedSize);
default :
return ERROR(prefix_unknown);
}

2
lib/legacy/zstd_v04.h
View File

@ -95,7 +95,7 @@ size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSi
***************************************/
typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx;
ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void);
size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);
size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);
size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx);
size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize);

4726
lib/legacy/zstd_v05.c Normal file
View File

File diff suppressed because it is too large Load Diff

156
lib/legacy/zstd_v05.h Normal file
View File

@ -0,0 +1,156 @@
/*
zstd_v05 - decoder for 0.5 format
Header File
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
*/
#ifndef ZSTDv05_H
#define ZSTDv05_H
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Dependencies
***************************************/
#include <stddef.h> /* size_t */
/* *************************************
* Simple functions
***************************************/
/*! ZSTDv05_decompress() :
`compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.
`dstCapacity` must be large enough, equal or larger than originalSize.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/* *************************************
* Helper functions
***************************************/
/* Error Management */
unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */
/* *************************************
* Explicit memory management
***************************************/
/** Decompression context */
typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx;
ZSTDv05_DCtx* ZSTDv05_createDCtx(void);
size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv05_decompressDCtx() :
* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */
size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-***********************
* Dictionary API
*************************/
/*! ZSTDv05_decompress_usingDict() :
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */
size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx;
ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void);
size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
/*-***************************************************************************
* Streaming decompression
*
* A ZBUFFv05_DCtx object is required to track streaming operations.
* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources.
* Use ZBUFFv05_decompressInit() to start a new decompression operation,
* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFFv05_DCtx objects can be reused multiple times.
*
* Use ZBUFFv05_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency)
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFFv05_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize()
* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
/* *************************************
* Tool functions
***************************************/
unsigned ZBUFFv05_isError(size_t errorCode);
const char* ZBUFFv05_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, and tend to offer better latency */
size_t ZBUFFv05_recommendedDInSize(void);
size_t ZBUFFv05_recommendedDOutSize(void);
/*-*************************************
* Constants
***************************************/
#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDv0505_H */

2
lib/zstd_decompress.c
View File

@ -136,7 +136,7 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }
size_t ZSTD_sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } /* non published interface */
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{

3
programs/Makefile
View File

@ -61,7 +61,8 @@ ZSTD_FILES_LEGACY:=
else
ZSTD_LEGACY_SUPPORT:=1
CPPFLAGS += -I../lib/legacy -I./legacy
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c $(ZSTDDIR)/legacy/zstd_v04.c legacy/fileio_legacy.c
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
$(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c
endif

124
programs/legacy/fileio_legacy.c
View File

@ -1,6 +1,6 @@
/*
fileio.c - File i/o handler
Copyright (C) Yann Collet 2013-2015
fileio_legacy.c - File i/o handler for legacy format
Copyright (C) Yann Collet 2015-2016
GPL v2 License
@ -19,12 +19,11 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net
- zstd source repository : https://github.com/Cyan4973/zstd
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
Note : this is stand-alone program.
It is not part of ZSTD compression library, it is a user program of ZSTD library.
Note : this file is not part of ZSTD compression library.
The license of ZSTD library is BSD.
The license of this file is GPLv2.
*/
@ -172,8 +171,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
/* Main decompression Loop */
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
while (toRead)
{
while (toRead){
size_t readSize, decodedSize;
/* Fill input buffer */
@ -187,8 +185,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@ -232,8 +229,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
/* Main decompression Loop */
toRead = ZSTDv02_nextSrcSizeToDecompress(dctx);
while (toRead)
{
while (toRead) {
size_t readSize, decodedSize;
/* Fill input buffer */
@ -247,8 +243,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
decodedSize = ZSTDv02_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv02_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@ -292,8 +287,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
/* Main decompression Loop */
toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
while (toRead)
{
while (toRead) {
size_t readSize, decodedSize;
/* Fill input buffer */
@ -307,8 +301,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
decodedSize = ZSTDv03_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv03_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@ -329,7 +322,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
}
/*- v0.4.x -*/
/*===== v0.4.x =====*/
typedef struct {
void* srcBuffer;
@ -380,8 +373,7 @@ unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
ZBUFFv04_decompressInit(ress.dctx);
ZBUFFv04_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1)
{
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
@ -404,11 +396,89 @@ unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
if (readSize != toRead) EXM_THROW(35, "Read error");
}
FIOv04_freeDResources(ress);
return frameSize;
}
/*===== v0.5.x =====*/
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZBUFFv05_DCtx* dctx;
} dRessv05_t;
static dRessv05_t FIOv05_createDResources(void)
{
dRessv05_t ress;
/* init */
ress.dctx = ZBUFFv05_createDCtx();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
ress.dictBuffer = NULL; ress.dictBufferSize=0;
/* Allocate Memory */
ress.srcBufferSize = ZBUFFv05_recommendedDInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFFv05_recommendedDOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
return ress;
}
static void FIOv05_freeDResources(dRessv05_t ress)
{
size_t const errorCode = ZBUFFv05_freeDCtx(ress.dctx);
if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
}
unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
FILE* foutput, FILE* finput)
{
U64 frameSize = 0;
size_t readSize = 4;
MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER);
ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead));
readSize -= inSize;
/* Write block */
sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
}
return frameSize;
}
/*===== General legacy dispatcher =====*/
unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 magicNumberLE)
{
switch(magicNumberLE)
@ -420,7 +490,17 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 ma
case ZSTDv03_magicNumber :
return FIOv03_decompressFrame(foutput, finput);
case ZSTDv04_magicNumber :
return FIOv04_decompressFrame(FIOv04_createDResources(), foutput, finput);
{ dRessv04_t r = FIOv04_createDResources();
unsigned long long const s = FIOv04_decompressFrame(r, foutput, finput);
FIOv04_freeDResources(r);
return s;
}
case ZSTDv05_MAGICNUMBER :
{ dRessv05_t r = FIOv05_createDResources();
unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput);
FIOv05_freeDResources(r);
return s;
}
default :
return ERROR(prefix_unknown);
}