sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-08 00:04:02 -04:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev

Browse Source
This commit is contained in:
inikep 2016-06-09 11:33:08 +02:00
commit 14947ed15c
16 changed files with 524 additions and 606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -23,6 +23,7 @@ projects/VS2010
projects/VS2012
projects/VS2013
projects/VS2015
build/bin
# IDEA solution files
*.idea

2
Makefile
View File

@ -130,7 +130,7 @@ asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c fails this test, for no obvious reason
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c used to fail this test for no obvious reason
asan32: clean
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"

1
NEWS
View File

@ -1,5 +1,6 @@
v0.7.0
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
New : Visual build scripts, by Christophe Chevalier
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
New : Frame checksum support
New : Support pass-through mode (when using `-df`)

7
build/build.VS2010.cmd Normal file
View File

@ -0,0 +1,7 @@
@echo off
rem build 32-bit
call "%~p0%build.generic.cmd" VS2010 Win32 Release v100
rem build 64-bit
call "%~p0%build.generic.cmd" VS2010 x64 Release v100

6
build/build.VS2012.cmd Normal file
View File

@ -0,0 +1,6 @@
@echo off
rem build 32-bit
call "%~p0%build.generic.cmd" VS2012 Win32 Release v110
rem build 64-bit
call "%~p0%build.generic.cmd" VS2012 x64 Release v110

7
build/build.VS2013.cmd Normal file
View File

@ -0,0 +1,7 @@
@echo off
rem build 32-bit
call "%~p0%build.generic.cmd" VS2013 Win32 Release v120
rem build 64-bit
call "%~p0%build.generic.cmd" VS2013 x64 Release v120

7
build/build.VS2015.cmd Normal file
View File

@ -0,0 +1,7 @@
@echo off
rem build 32-bit
call "%~p0%build.generic.cmd" VS2015 Win32 Release v140
rem build 64-bit
call "%~p0%build.generic.cmd" VS2015 x64 Release v140

51
build/build.generic.cmd Normal file
View File

@ -0,0 +1,51 @@
@echo off
IF "%1%" == "" GOTO display_help
SET vs_version=%1
SET vs_platform=%2
IF "%vs_platform%" == "" SET vs_platform=x64
SET vs_configuration=%3
IF "%vs_configuration%" == "" SET vs_configuration=Release
SET vs_toolset=%4
GOTO build
:display_help
echo Syntax: build.generic.cmd vs_version vs_platform vs_configuration vs_toolset
echo vs_version: VS installed version (VS2012, VS2013, VS2015, ...)
echo vs_platform: Platform (x64 or Win32)
echo vs_configuration: VS configuration (Release or Debug)
echo vs_toolset: Platform Toolset (v100, v110, v120, v140)
EXIT /B 1
:build
SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
IF %vs_version% == VS2013 SET msbuild="C:\Program Files (x86)\MSBuild\12.0\Bin\MSBuild.exe"
IF %vs_version% == VS2015 SET msbuild="C:\Program Files (x86)\MSBuild\14.0\Bin\MSBuild.exe"
rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ?
SET project="%~p0\..\projects\VS2010\zstd.sln"
SET msbuildparams=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%vs_platform% /p:Configuration=%vs_configuration%
IF NOT "%vs_toolset%" == "" SET msbuildparams=%msbuildparams% /p:PlatformToolset=%vs_toolset%
SET output=%~p0%bin
SET output="%output%/%vs_configuration%/%vs_platform%/"
SET msbuildparams=%msbuildparams% /p:OutDir=%output%
echo ### Building %vs_version% project for %vs_configuration% %vs_platform% (%vs_toolset%)...
echo ### Build Params: %msbuildparams%
%msbuild% %project% %msbuildparams%
IF ERRORLEVEL 1 EXIT /B 1
echo # Success
echo # OutDir: %output%
echo #

85
lib/common/huf.h
View File

@ -31,8 +31,8 @@
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef HUF_H
#define HUF_H
#ifndef HUF_H_298734234
#define HUF_H_298734234
#if defined (__cplusplus)
extern "C" {
@ -53,8 +53,9 @@ size_t HUF_decompress(void* dst, size_t dstSize,
/*
HUF_compress() :
Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
Note : `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB
'dst' buffer must be already allocated.
Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
`srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB
@return : size of compressed data (<= `dstCapacity`)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression).
@ -63,7 +64,7 @@ HUF_compress() :
HUF_decompress() :
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
into already allocated buffer 'dst', of minimum size 'dstSize'.
`dstSize` : must be the **exact** size of original (uncompressed) data.
`dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
because it knows size to regenerate.
@ -121,13 +122,12 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
/* static allocation of HUF's DTable */
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
typedef U16 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((maxTableLog)*0x101) }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)+1)] = { (((maxTableLog)+1)*0x101) }
/* ****************************************
@ -135,7 +135,6 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
/* ****************************************
@ -161,35 +160,6 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, un
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@ -204,6 +174,39 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
/*
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif /* HUF_STATIC_LINKING_ONLY */
@ -211,4 +214,4 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* sr
}
#endif
#endif /* HUF_H */
#endif /* HUF_H_298734234 */

105
lib/common/zstd.h
View File

@ -126,12 +126,14 @@ ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*!< @return : error
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-***********************
* Dictionary API
*************************/
/*-************************
* Simple dictionary API
***************************/
/*! ZSTD_compress_usingDict() :
* Compression using a pre-defined Dictionary content (see dictBuilder).
* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */
* Note 1 : This function load the dictionary, resulting in a significant startup time.
* Note 2 : `dict` must remain valid and unmodified during compression operation.
* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@ -140,14 +142,52 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
/*! ZSTD_decompress_usingDict() :
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */
* Dictionary must be identical to the one used during compression.
* Note 1 : This function load the dictionary, resulting in a significant startup time
* Note 2 : `dict` must remain valid and unmodified during compression operation.
* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
/*-**************************
* Advanced Dictionary API
****************************/
/*! ZSTD_createCDict() :
* Create a digested dictionary, ready to start compression operation without startup delay.
* `dict` can be released after creation */
typedef struct ZSTD_CDict_s ZSTD_CDict;
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
/*! ZSTD_compress_usingCDict() :
* Compression using a pre-digested Dictionary.
* Much faster than ZSTD_compress_usingDict() when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict);
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */
typedef struct ZSTD_DDict_s ZSTD_DDict;
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
/*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Much faster than ZSTD_decompress_usingDict() when same dictionary is used multiple times. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_DDict* ddict);
#ifdef ZSTD_STATIC_LINKING_ONLY
/* ====================================================================================
@ -158,7 +198,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
* ==================================================================================== */
/*--- Dependency ---*/
#include "mem.h"
#include "mem.h" /* U32 */
/*--- Constants ---*/
@ -187,22 +227,22 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f
/*--- Types ---*/
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */
typedef struct {
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == faster, more memory */
U32 searchLog; /* nb of searches : larger == more compression, slower */
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
U32 windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */
U32 chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /*< dispatch table : larger == faster, more memory */
U32 searchLog; /*< nb of searches : larger == more compression, slower */
U32 searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */
U32 targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
ZSTD_strategy strategy;
} ZSTD_compressionParameters;
typedef struct {
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
U32 checksumFlag; /* 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */
U32 checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
U32 noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@ -217,15 +257,16 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
/*-*************************************
* Advanced functions
* Advanced compression functions
***************************************/
/*! ZSTD_createCCtx_advanced() :
* Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTD_parameters params, ZSTD_customMem customMem);
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
@ -251,27 +292,11 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
const void* dict,size_t dictSize,
ZSTD_parameters params);
/*! ZSTD_compress_usingPreparedDCtx() :
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*- Advanced Decompression functions -*/
/*! ZSTD_decompress_usingPreparedDCtx() :
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*! ZSTD_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
/* **************************************

94
lib/compress/zstd_compress.c
View File

@ -2148,7 +2148,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
{ BYTE* const op = (BYTE*)dst;
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
U32 const checksumFlag = params.fParams.checksumFlag;
U32 const checksumFlag = params.fParams.checksumFlag>0;
U32 const windowSize = 1U << params.cParams.windowLog;
U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
@ -2385,7 +2385,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
/*! ZSTD_compressBegin_advanced() :
* @return : 0, or an error code */
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{
@ -2393,17 +2393,17 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
{ size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize);
if (ZSTD_isError(errorCode)) return errorCode; }
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, pledgedSrcSize);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
}
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
}
@ -2449,7 +2449,12 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
}
size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
/*! ZSTD_compress_usingPreparedCCtx() :
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
@ -2532,6 +2537,81 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
}
/* ===== Dictionary API ===== */
struct ZSTD_CDict_s {
void* dictContent;
size_t dictContentSize;
ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within zstd.h */
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict));
void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
if (!dictContent || !cdict || !cctx) {
customMem.customFree(customMem.opaque, dictContent);
customMem.customFree(customMem.opaque, cdict);
customMem.customFree(customMem.opaque, cctx);
return NULL;
}
memcpy(dictContent, dict, dictSize);
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
customMem.customFree(customMem.opaque, dictContent);
customMem.customFree(customMem.opaque, cdict);
customMem.customFree(customMem.opaque, cctx);
return NULL;
} }
cdict->dictContent = dictContent;
cdict->dictContentSize = dictSize;
cdict->refContext = cctx;
return cdict;
}
}
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params;
memset(&params, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
}
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
{
ZSTD_freeFunction const cFree = cdict->refContext->customMem.customFree;
void* const opaque = cdict->refContext->customMem.opaque;
ZSTD_freeCCtx(cdict->refContext);
cFree(opaque, cdict->dictContent);
cFree(opaque, cdict);
return 0;
}
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict)
{
return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext,
dst, dstCapacity,
src, srcSize);
}
/*-===== Pre-defined compression levels =====-*/
#define ZSTD_DEFAULT_CLEVEL 1

592
lib/decompress/huf_decompress.c
View File

@ -60,37 +60,26 @@
* Includes
****************************************************************/
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
/* **************************************************************
* Error Management
****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* *******************************************************
* HUF : Huffman block decompression
*********************************************************/
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
/*-***************************/
/* single-symbol decoding */
/*-***************************/
typedef struct { BYTE maxTableLog; BYTE currentTableLog; } DTableDesc;
size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
@ -101,16 +90,19 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
U32 nextRankStart;
void* const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
DTableDesc dtd;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */
memcpy(&dtd, DTable, sizeof(dtd));
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */
DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
if (tableLog > dtd.maxTableLog) return ERROR(tableLog_tooLarge); /* DTable is too small */
dtd.currentTableLog = (BYTE)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
memcpy(DTable, &dtd, sizeof(dtd));
/* Prepare ranks */
nextRankStart = 0;
@ -181,14 +173,18 @@ static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, B
size_t HUF_decompress1X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U16* DTable)
const HUF_DTable* DTable)
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
const U32 dtLog = DTable[0];
const void* dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1;
BIT_DStream_t bitD;
DTableDesc dtd;
U32 dtLog;
memcpy(&dtd, DTable, sizeof(dtd));
dtLog = dtd.currentTableLog;
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode; }
@ -219,7 +215,7 @@ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS
size_t HUF_decompress4X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U16* DTable)
const HUF_DTable* DTable)
{
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
@ -229,18 +225,16 @@ size_t HUF_decompress4X2_usingDTable(
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
size_t const length1 = MEM_readLE16(istart);
size_t const length2 = MEM_readLE16(istart+2);
size_t const length3 = MEM_readLE16(istart+4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
@ -254,17 +248,21 @@ size_t HUF_decompress4X2_usingDTable(
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
DTableDesc dtd;
U32 dtLog;
memcpy(&dtd, DTable, sizeof(dtd));
dtLog = dtd.currentTableLog;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode;
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
@ -315,11 +313,11 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
const BYTE* ip = (const BYTE*) cSrc;
size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
ip += errorCode;
cSrcSize -= errorCode;
size_t const hSize = HUF_readDTableX2 (DTable, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
@ -328,6 +326,9 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS
/* *************************/
/* double-symbols decoding */
/* *************************/
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
@ -413,7 +414,7 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
}
}
size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
@ -422,20 +423,23 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
U32* const rankStart = rankStart0+1;
rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
const U32 memLog = DTable[0];
DTableDesc dtd;
U32 maxTableLog;
size_t iSize;
void* dtPtr = DTable;
HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1;
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
memcpy(&dtd, DTable, sizeof(dtd));
maxTableLog = dtd.maxTableLog-1;
if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
@ -464,7 +468,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
/* Build rankVal */
{ U32* const rankVal0 = rankVal[0];
{ int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
U32 nextRankVal = 0;
U32 w;
for (w=1; w<maxW+1; w++) {
@ -474,18 +478,20 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
} }
{ U32 const minBits = tableLog+1 - maxW;
U32 consumed;
for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) {
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
U32* const rankValPtr = rankVal[consumed];
U32 w;
for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
HUF_fillDTableX4(dt, memLog,
HUF_fillDTableX4(dt, maxTableLog,
sortedSymbol, sizeOfSort,
rankStart0, rankVal, maxW,
tableLog+1);
dtd.currentTableLog = (BYTE)maxTableLog;
memcpy(DTable, &dtd, sizeof(dtd));
return iSize;
}
@ -536,7 +542,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
}
/* closer to the end */
/* closer to end : up to 2 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2))
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
@ -553,23 +559,24 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
size_t HUF_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
const HUF_DTable* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
BIT_DStream_t bitD;
/* Init */
BIT_DStream_t bitD;
{ size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode;
}
/* decode */
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtLog);
{ BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
DTableDesc dtd;
memcpy(&dtd, DTable, sizeof(dtd));
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.currentTableLog);
}
/* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
@ -595,32 +602,30 @@ size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cS
size_t HUF_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
const HUF_DTable* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
const void* const dtPtr = DTable+1;
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
size_t const length1 = MEM_readLE16(istart);
size_t const length2 = MEM_readLE16(istart+2);
size_t const length3 = MEM_readLE16(istart+4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
size_t const segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
@ -629,17 +634,21 @@ size_t HUF_decompress4X4_usingDTable(
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
DTableDesc dtd;
U32 dtLog;
memcpy(&dtd, DTable, sizeof(dtd));
dtLog = dtd.currentTableLog;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode;
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
@ -677,8 +686,8 @@ size_t HUF_decompress4X4_usingDTable(
HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endCheck) return ERROR(corruption_detected); }
/* decoded size */
return dstSize;
@ -701,386 +710,6 @@ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cS
}
/* ********************************/
/* quad-symbol decoding */
/* ********************************/
typedef struct { BYTE nbBits; BYTE nbBytes; } HUF_DDescX6;
typedef union { BYTE byte[4]; U32 sequence; } HUF_DSeqX6;
/* recursive, up to level 3; may benefit from <template>-like strategy to nest each level inline */
static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSequence, int sizeLog,
const rankVal_t rankValOrigin, const U32 consumed, const int minWeight, const U32 maxWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, const U32* rankStart,
const U32 nbBitsBaseline, HUF_DSeqX6 baseSeq, HUF_DDescX6 DDesc)
{
const int scaleLog = nbBitsBaseline - sizeLog; /* note : targetLog >= (nbBitsBaseline-1), hence scaleLog <= 1 */
const int minBits = nbBitsBaseline - maxWeight;
const U32 level = DDesc.nbBytes;
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
U32 symbolStartPos, s;
/* local rankVal, will be modified */
memcpy(rankVal, rankValOrigin[consumed], sizeof(rankVal));
/* fill skipped values */
if (minWeight>1) {
U32 i;
const U32 skipSize = rankVal[minWeight];
for (i = 0; i < skipSize; i++) {
DSequence[i] = baseSeq;
DDescription[i] = DDesc;
} }
/* fill DTable */
DDesc.nbBytes++;
symbolStartPos = rankStart[minWeight];
for (s=symbolStartPos; s<sortedListSize; s++) {
const BYTE symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; /* >= 1 (sorted) */
const int nbBits = nbBitsBaseline - weight; /* >= 1 (by construction) */
const int totalBits = consumed+nbBits;
const U32 start = rankVal[weight];
const U32 length = 1 << (sizeLog-nbBits);
baseSeq.byte[level] = symbol;
DDesc.nbBits = (BYTE)totalBits;
if ((level<3) && (sizeLog-totalBits >= minBits)) { /* enough room for another symbol */
int nextMinWeight = totalBits + scaleLog;
if (nextMinWeight < 1) nextMinWeight = 1;
HUF_fillDTableX6LevelN(DDescription+start, DSequence+start, sizeLog-nbBits,
rankValOrigin, totalBits, nextMinWeight, maxWeight,
sortedSymbols, sortedListSize, rankStart,
nbBitsBaseline, baseSeq, DDesc); /* recursive (max : level 3) */
} else {
U32 i;
const U32 end = start + length;
for (i = start; i < end; i++) {
DDescription[i] = DDesc;
DSequence[i] = baseSeq;
} }
rankVal[weight] += length;
}
}
/* note : same preparation as X4 */
size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 };
U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 };
U32* const rankStart = rankStart0+1;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal;
const U32 memLog = DTable[0];
size_t iSize;
if (memLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable is too small */
/* find maxWeight */
for (maxW = tableLog; maxW && rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
/* Get start index of each weight */
{ U32 w, nextRankStart = 0;
for (w=1; w<maxW+1; w++) {
U32 current = nextRankStart;
nextRankStart += rankStats[w];
rankStart[w] = current;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
sizeOfSort = nextRankStart;
}
/* sort symbols by weight */
{ U32 s;
for (s=0; s<nbSymbols; s++) {
U32 w = weightList[s];
U32 r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{ const U32 minBits = tableLog+1 - maxW;
U32 nextRankVal = 0;
U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0];
for (w=1; w<maxW+1; w++) {
U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current;
}
for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
U32* rankValPtr = rankVal[consumed];
for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } }
/* fill tables */
{ void* ddPtr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr;
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr;
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
DDesc.nbBits = 0;
DDesc.nbBytes = 0;
HUF_fillDTableX6LevelN(DDescription, DSequence, memLog,
(const U32 (*)[HUF_TABLELOG_ABSOLUTEMAX + 1])rankVal, 0, 1, maxW,
sortedSymbol, sizeOfSort, rankStart0,
tableLog+1, DSeq, DDesc);
}
return iSize;
}
static U32 HUF_decodeSymbolX6(void* op, BIT_DStream_t* DStream, const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, ds+val, sizeof(HUF_DSeqX6));
BIT_skipBits(DStream, dd[val].nbBits);
return dd[val].nbBytes;
}
static U32 HUF_decodeLastSymbolsX6(void* op, U32 const maxL, BIT_DStream_t* DStream,
const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
U32 const length = dd[val].nbBytes;
if (length <= maxL) {
memcpy(op, ds+val, length);
BIT_skipBits(DStream, dd[val].nbBits);
return length;
}
memcpy(op, ds+val, maxL);
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
BIT_skipBits(DStream, dd[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
}
return maxL;
}
#define HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX6(ptr, DStreamPtr, dd, ds, dtLog)
#define HUF_DECODE_SYMBOLX6_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
#define HUF_DECODE_SYMBOLX6_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
BYTE* const pStart = p;
/* up to 16 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-16)) {
HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_1(p, bitDPtr);
HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_0(p, bitDPtr);
}
/* closer to the end, up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4))
HUF_DECODE_SYMBOLX6_0(p, bitDPtr);
while ((BIT_reloadDStream(bitDPtr) <= BIT_DStream_endOfBuffer) && (p < pEnd))
p += HUF_decodeLastSymbolsX6(p, (U32)(pEnd-p), bitDPtr, dd, ds, dtLog);
return p-pStart;
}
size_t HUF_decompress1X6_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
BIT_DStream_t bitD;
/* Init */
{ size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize);
if (HUF_isError(errorCode)) return errorCode; }
/* finish bitStreams one by one */
{ U32 const dtLog = DTable[0];
HUF_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog); }
/* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_TABLELOG_MAX);
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress1X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
#define HUF_DECODE_ROUNDX6 \
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); \
HUF_DECODE_SYMBOLX6_1(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_1(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_1(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_1(op4, &bitD4); \
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); \
HUF_DECODE_SYMBOLX6_0(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_0(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_0(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
size_t HUF_decompress4X6_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
if (dstSize < 64) return ERROR(dstSize_tooSmall); /* only work for dstSize >= 64 */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 4-64 symbols per loop (1-16 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
if (endSignal==BIT_DStream_unfinished) {
HUF_DECODE_ROUNDX6;
if (sizeof(bitD1.bitContainer)==4) { /* need to decode at least 4 bytes per stream */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
HUF_DECODE_ROUNDX6;
}
{ U32 const saved2 = MEM_read32(opStart2); /* saved from overwrite */
U32 const saved3 = MEM_read32(opStart3);
U32 const saved4 = MEM_read32(opStart4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; ) {
HUF_DECODE_ROUNDX6;
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
MEM_write32(opStart2, saved2);
MEM_write32(opStart3, saved3);
MEM_write32(opStart4, saved4);
} }
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
HUF_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog);
HUF_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog);
HUF_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_TABLELOG_MAX);
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/* ********************************/
/* Generic decompression selector */
/* ********************************/
@ -1107,12 +736,29 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
{
/* decoder timing evaluation */
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
U32 const D256 = (U32)(dstSize >> 8);
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
return DTime1 < DTime0;
}
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, HUF_decompress4X6 };
U32 Dtime[3]; /* decompression time estimation */
static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 };
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
@ -1120,22 +766,10 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
/* decoder timing evaluation */
{ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
U32 const D256 = (U32)(dstSize >> 8);
U32 n; for (n=0; n<3; n++)
Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256);
}
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
{ U32 algoNb = 0;
if (Dtime[1] < Dtime[0]) algoNb = 1;
// if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUF_decompress4X6 is not good */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
}
//return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
//return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
//return HUF_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
}

99
lib/decompress/zstd_decompress.c
View File

@ -53,7 +53,7 @@
/*-*******************************************************
* Dependencies
*********************************************************/
#include <string.h> /* memcpy, memmove */
#include <string.h> /* memcpy, memmove, memset */
#include <stdio.h> /* debug only : printf */
#include "mem.h" /* low level memory routines */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
@ -112,7 +112,7 @@ struct ZSTD_DCtx_s
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
unsigned hufTableX4[HUF_DTABLE_SIZE(HufLog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog+1)];
const void* previousDstEnd;
const void* base;
const void* vBase;
@ -143,7 +143,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->base = NULL;
dctx->vBase = NULL;
dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog;
dctx->hufTable[0] = (HUF_DTable)((HufLog+1)*0x101);
dctx->flagRepeatTable = 0;
dctx->dictID = 0;
return 0;
@ -508,7 +508,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
{ size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
{ size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable);
if (HUF_isError(errorCode)) return ERROR(corruption_detected);
}
dctx->litPtr = dctx->litBuffer;
@ -938,6 +938,14 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
}
size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length)
{
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
memset(dst, byte, length);
return length;
}
/*! ZSTD_decompressFrame() :
* `dctx` must be properly initialized */
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
@ -982,7 +990,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
break;
case bt_rle :
return ERROR(GENERIC); /* not yet supported */
decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize);
break;
case bt_end :
/* end of frame */
@ -1004,6 +1012,11 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
}
/*! ZSTD_decompress_usingPreparedDCtx() :
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
@ -1180,7 +1193,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t const d
{
size_t dictSize = dictSizeStart;
{ size_t const hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
{ size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + hSize;
dictSize -= hSize;
@ -1259,3 +1272,77 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
return 0;
}
struct ZSTD_DDict_s {
void* dictContent;
size_t dictContentSize;
ZSTD_DCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within zstd.h */
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
{
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict));
void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
if (!dictContent || !ddict || !dctx) {
customMem.customFree(customMem.opaque, dictContent);
customMem.customFree(customMem.opaque, ddict);
customMem.customFree(customMem.opaque, dctx);
return NULL;
}
memcpy(dictContent, dict, dictSize);
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
customMem.customFree(customMem.opaque, dictContent);
customMem.customFree(customMem.opaque, ddict);
customMem.customFree(customMem.opaque, dctx);
return NULL;
} }
ddict->dictContent = dictContent;
ddict->dictContentSize = dictSize;
ddict->refContext = dctx;
return ddict;
}
}
/*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, allocator);
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree;
void* const opaque = ddict->refContext->customMem.opaque;
ZSTD_freeDCtx(ddict->refContext);
cFree(opaque, ddict->dictContent);
cFree(opaque, ddict);
return 0;
}
/*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* In contrast with older ZSTD_decompress_usingDict(), use dictionary without significant overhead. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_DDict* ddict)
{
return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
dst, dstCapacity,
src, srcSize);
}

23
programs/bench.c
View File

@ -216,23 +216,21 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&clockStart);
{ U32 nbLoops = 0;
ZSTD_CDict* cdict = ZSTD_createCDict(dictBuffer, dictBufferSize, cLevel);
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict() allocation failure");
do {
U32 blockNb;
{ ZSTD_parameters params;
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
params.fParams.contentSizeFlag = 1;
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
if (ZSTD_isError(initResult)) break;
} }
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
size_t const rSize = ZSTD_compress_usingCDict(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
cdict);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
blockTable[blockNb].cSize = rSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ZSTD_freeCDict(cdict);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
} }
@ -254,13 +252,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_getTime(&clockStart);
{ U32 nbLoops = 0;
ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
do {
U32 blockNb;
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
ddict);
if (ZSTD_isError(regenSize)) {
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
blockNb, ZSTD_getErrorName(regenSize));
@ -271,6 +271,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
ZSTD_freeDDict(ddict);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
} }

45
programs/datagen.c
View File

@ -27,9 +27,9 @@
* Includes
**************************************/
#include <stdlib.h> /* malloc */
#include <stdio.h> /* FILE, fwrite */
#include <stdio.h> /* FILE, fwrite, fprintf */
#include <string.h> /* memcpy */
#include "mem.h"
#include "mem.h" /* U32 */
/*-************************************
@ -108,43 +108,52 @@ static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
}
#define RDG_RAND15BITS ( RDG_rand(seed) & 0x7FFF )
#define RDG_RANDLENGTH ( (RDG_rand(seed) & 7) ? (RDG_rand(seed) & 0xF) : (RDG_rand(seed) & 0x1FF) + 0xF)
static U32 RDG_rand15Bits (unsigned* seedPtr)
{
return RDG_rand(seedPtr) & 0x7FFF;
}
static U32 RDG_randLength(unsigned* seedPtr)
{
if (RDG_rand(seedPtr) & 7)
return (RDG_rand(seedPtr) & 0xF);
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
}
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
{
BYTE* buffPtr = (BYTE*)buffer;
const U32 matchProba32 = (U32)(32768 * matchProba);
BYTE* const buffPtr = (BYTE*)buffer;
U32 const matchProba32 = (U32)(32768 * matchProba);
size_t pos = prefixSize;
U32* seed = seedPtr;
U32 prevOffset = 1;
/* special case : sparse content */
while (matchProba >= 1.0) {
size_t size0 = RDG_rand(seed) & 3;
size_t size0 = RDG_rand(seedPtr) & 3;
size0 = (size_t)1 << (16 + size0 * 2);
size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/
if (buffSize < pos + size0) {
memset(buffPtr+pos, 0, buffSize-pos);
return;
}
memset(buffPtr+pos, 0, size0);
pos += size0;
buffPtr[pos-1] = RDG_genChar(seed, ldt);
buffPtr[pos-1] = RDG_genChar(seedPtr, ldt);
continue;
}
/* init */
if (pos==0) buffPtr[0] = RDG_genChar(seed, ldt), pos=1;
if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1;
/* Generate compressible data */
while (pos < buffSize) {
/* Select : Literal (char) or Match (within 32K) */
if (RDG_RAND15BITS < matchProba32) {
if (RDG_rand15Bits(seedPtr) < matchProba32) {
/* Copy (within 32K) */
U32 const length = RDG_RANDLENGTH + 4;
U32 const length = RDG_randLength(seedPtr) + 4;
U32 const d = (U32) MIN(pos + length , buffSize);
U32 const repeatOffset = (RDG_rand(seed) & 15) == 2;
U32 const randOffset = RDG_RAND15BITS + 1;
U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2;
U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
size_t match = pos - offset;
//TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length);
@ -152,9 +161,9 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
prevOffset = offset;
} else {
/* Literal (noise) */
U32 const length = RDG_RANDLENGTH;
U32 const length = RDG_randLength(seedPtr);
U32 const d = (U32) MIN(pos + length, buffSize);
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, ldt);
while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt);
} }
}
@ -174,7 +183,7 @@ void RDG_genStdout(unsigned long long size, double matchProba, double litProba,
{
size_t const stdBlockSize = 128 KB;
size_t const stdDictSize = 32 KB;
BYTE* buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
U64 total = 0;
BYTE ldt[LTSIZE];

1
programs/fileio.c
View File

@ -622,7 +622,6 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Main decompression Loop */
while (1) {
/* Decode */
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));