sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-05 00:03:19 -05:00
Code Issues Projects Releases Wiki Activity

updated fse

Browse Source
This commit is contained in:
Yann Collet 2016-01-06 01:58:37 +01:00
parent 8ab94b653c
commit 3b994cb465
6 changed files with 198 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
lib/fse.c
View File

@ -34,10 +34,10 @@
#ifndef FSE_COMMONDEFS_ONLY #ifndef FSE_COMMONDEFS_ONLY
/**************************************************************** /* **************************************************************
* Tuning parameters * Tuning parameters
****************************************************************/ ****************************************************************/
/* MEMORY_USAGE : /*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio * Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect * Reduced memory usage can improve speed, due to cache effect
@ -45,26 +45,23 @@
#define FSE_MAX_MEMORY_USAGE 14 #define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13 #define FSE_DEFAULT_MEMORY_USAGE 13
/* FSE_MAX_SYMBOL_VALUE : /*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized. * Maximum symbol value authorized.
* Required for proper stack allocation */ * Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255 #define FSE_MAX_SYMBOL_VALUE 255
/**************************************************************** /* **************************************************************
* template functions type & suffix * template functions type & suffix
****************************************************************/ ****************************************************************/
#define FSE_FUNCTION_TYPE BYTE #define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION #define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
/****************************************************************
* Byte symbol type
****************************************************************/
#endif /* !FSE_COMMONDEFS_ONLY */ #endif /* !FSE_COMMONDEFS_ONLY */
/* **************************************************************
/****************************************************************
* Compiler specifics * Compiler specifics
****************************************************************/ ****************************************************************/
#ifdef _MSC_VER /* Visual Studio */ #ifdef _MSC_VER /* Visual Studio */
@ -82,7 +79,7 @@
#endif #endif
/**************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */ #include <stdlib.h> /* malloc, free, qsort */
@ -92,7 +89,7 @@
#include "fse_static.h" #include "fse_static.h"
/**************************************************************** /* ***************************************************************
* Constants * Constants
*****************************************************************/ *****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) #define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
@ -107,20 +104,20 @@
#endif #endif
/**************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/**************************************************************** /* **************************************************************
* Complex types * Complex types
****************************************************************/ ****************************************************************/
typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/**************************************************************** /* **************************************************************
* Templates * Templates
****************************************************************/ ****************************************************************/
/* /*
@ -144,8 +141,7 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/* Function templates */ /* Function templates */
size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) size_t FSE_count_generic(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize, unsigned safe)
{ {
const FSE_FUNCTION_TYPE* ip = source; const FSE_FUNCTION_TYPE* ip = source;
const FSE_FUNCTION_TYPE* const iend = ip+sourceSize; const FSE_FUNCTION_TYPE* const iend = ip+sourceSize;
@ -226,7 +222,7 @@ size_t FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION)
size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION) size_t FSE_FUNCTION_NAME(FSE_countFast, FSE_FUNCTION_EXTENSION)
(unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize) (unsigned* count, unsigned* maxSymbolValuePtr, const FSE_FUNCTION_TYPE* source, size_t sourceSize)
{ {
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0); return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0);
} }
size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION) size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
@ -235,25 +231,26 @@ size_t FSE_FUNCTION_NAME(FSE_count, FSE_FUNCTION_EXTENSION)
if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255)) if ((sizeof(FSE_FUNCTION_TYPE)==1) && (*maxSymbolValuePtr >= 255))
{ {
*maxSymbolValuePtr = 255; *maxSymbolValuePtr = 255;
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 0); return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 0);
} }
return FSE_FUNCTION_NAME(FSE_count_generic, FSE_FUNCTION_EXTENSION) (count, maxSymbolValuePtr, source, sourceSize, 1); return FSE_count_generic(count, maxSymbolValuePtr, source, sourceSize, 1);
} }
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION) size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
const unsigned tableSize = 1 << tableLog; const unsigned tableSize = 1 << tableLog;
const unsigned tableMask = tableSize - 1; const unsigned tableMask = tableSize - 1;
U16* tableU16 = ( (U16*) ct) + 2; void* const ptr = ct;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) (((U32*)ct) + 1 + (tableLog ? tableSize>>1 : 1) ); U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
const unsigned step = FSE_tableStep(tableSize); const unsigned step = FSE_tableStep(tableSize);
unsigned cumul[FSE_MAX_SYMBOL_VALUE+2]; unsigned cumul[FSE_MAX_SYMBOL_VALUE+2];
U32 position = 0; U32 position = 0;
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* init isn't necessary, even if static analyzer complain about it */ FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
U32 highThreshold = tableSize-1; U32 highThreshold = tableSize-1;
unsigned symbol; unsigned symbol;
unsigned i; unsigned i;
@ -269,7 +266,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
cumul[0] = 0; cumul[0] = 0;
for (i=1; i<=maxSymbolValue+1; i++) for (i=1; i<=maxSymbolValue+1; i++)
{ {
if (normalizedCounter[i-1]==-1) /* Low prob symbol */ if (normalizedCounter[i-1]==-1) /* Low proba symbol */
{ {
cumul[i] = cumul[i-1] + 1; cumul[i] = cumul[i-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1); tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(i-1);
@ -287,7 +284,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
{ {
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask; position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Lowprob area */ while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
} }
} }
@ -296,7 +293,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
/* Build table */ /* Build table */
for (i=0; i<tableSize; i++) for (i=0; i<tableSize; i++)
{ {
FSE_FUNCTION_TYPE s = tableSymbol[i]; /* static analyzer doesn't understand tableSymbol is properly initialized */ FSE_FUNCTION_TYPE s = tableSymbol[i]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */ tableU16[cumul[s]++] = (U16) (tableSize+i); /* TableU16 : sorted by symbol order; gives next state value */
} }
@ -332,24 +329,22 @@ size_t FSE_FUNCTION_NAME(FSE_buildCTable, FSE_FUNCTION_EXTENSION)
} }
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION) FSE_DTable* FSE_createDTable (unsigned tableLog)
FSE_DTable* FSE_FUNCTION_NAME(FSE_createDTable, FSE_FUNCTION_EXTENSION) (unsigned tableLog)
{ {
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
} }
void FSE_FUNCTION_NAME(FSE_freeDTable, FSE_FUNCTION_EXTENSION) (FSE_DTable* dt) void FSE_freeDTable (FSE_DTable* dt)
{ {
free(dt); free(dt);
} }
size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; FSE_DTableHeader DTableH;
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (dt+1); /* because dt is unsigned, 32-bits aligned on 32-bits */ void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
const U32 tableSize = 1 << tableLog; const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1; const U32 tableMask = tableSize-1;
const U32 step = FSE_tableStep(tableSize); const U32 step = FSE_tableStep(tableSize);
@ -365,7 +360,7 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */ /* Init, lay down lowprob symbols */
DTableH[0].tableLog = (U16)tableLog; DTableH.tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++) for (s=0; s<=maxSymbolValue; s++)
{ {
if (normalizedCounter[s]==-1) if (normalizedCounter[s]==-1)
@ -406,7 +401,8 @@ size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION)
} }
} }
DTableH->fastMode = (U16)noLarge; DTableH.fastMode = (U16)noLarge;
memcpy(dt, &DTableH, sizeof(DTableH));
return 0; return 0;
} }
@ -890,8 +886,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
const unsigned tableSize = 1 << nbBits; const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1; const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask; const unsigned maxSymbolValue = tableMask;
U16* tableU16 = ( (U16*) ct) + 2; void* const ptr = ct;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((((U32*)ct)+1) + (tableSize>>1)); U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
unsigned s; unsigned s;
/* Sanity checks */ /* Sanity checks */
@ -918,8 +916,10 @@ size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
/* fake FSE_CTable, for rle (100% always same symbol) input */ /* fake FSE_CTable, for rle (100% always same symbol) input */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
{ {
U16* tableU16 = ( (U16*) ct) + 2; void* ptr = ct;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) ((U32*)ct + 2); U16* tableU16 = ( (U16*) ptr) + 2;
void* FSCTptr = (U32*)ptr + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
/* header */ /* header */
tableU16[-2] = (U16) 0; tableU16[-2] = (U16) 0;
@ -1076,8 +1076,10 @@ size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
*********************************************************/ *********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{ {
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; void* ptr = dt;
FSE_decode_t* const cell = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
DTableH->tableLog = 0; DTableH->tableLog = 0;
DTableH->fastMode = 0; DTableH->fastMode = 0;
@ -1092,8 +1094,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{ {
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)dt; void* ptr = dt;
FSE_decode_t* const dinfo = (FSE_decode_t*)(dt + 1); /* because dt is unsigned */ FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
const unsigned tableSize = 1 << nbBits; const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1; const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask; const unsigned maxSymbolValue = tableMask;
@ -1189,7 +1193,8 @@ size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize, const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt) const FSE_DTable* dt)
{ {
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)dt; const void* ptr = dt;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
const U32 fastMode = DTableH->fastMode; const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */ /* select fast mode (static) */

46
lib/fse.h
View File

@ -40,20 +40,20 @@ extern "C" {
#endif #endif
/****************************************** /* *****************************************
* Includes * Includes
******************************************/ ******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */ #include <stddef.h> /* size_t, ptrdiff_t */
/****************************************** /* *****************************************
* FSE simple functions * FSE simple functions
******************************************/ ******************************************/
size_t FSE_compress(void* dst, size_t maxDstSize, size_t FSE_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize); const void* src, size_t srcSize);
size_t FSE_decompress(void* dst, size_t maxDstSize, size_t FSE_decompress(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize); const void* cSrc, size_t cSrcSize);
/* /*!
FSE_compress(): FSE_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize) 'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= FSE_compressBound(srcSize)
@ -74,7 +74,7 @@ FSE_decompress():
*/ */
/****************************************** /* *****************************************
* Tool functions * Tool functions
******************************************/ ******************************************/
size_t FSE_compressBound(size_t size); /* maximum compressed size */ size_t FSE_compressBound(size_t size); /* maximum compressed size */
@ -84,10 +84,10 @@ unsigned FSE_isError(size_t code); /* tells if a return value is an er
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/****************************************** /* *****************************************
* FSE advanced functions * FSE advanced functions
******************************************/ ******************************************/
/* /*!
FSE_compress2(): FSE_compress2():
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value Both parameters can be defined as '0' to mean : use default value
@ -99,10 +99,10 @@ FSE_compress2():
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/****************************************** /* *****************************************
* FSE detailed API * FSE detailed API
******************************************/ ******************************************/
/* /*!
FSE_compress() does the following: FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[] 1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) 2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
@ -122,7 +122,7 @@ or to save and provide normalized distribution using external method.
/* *** COMPRESSION *** */ /* *** COMPRESSION *** */
/* /*!
FSE_count(): FSE_count():
Provides the precise count of each symbol within a table 'count' Provides the precise count of each symbol within a table 'count'
'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1). 'count' is a table of unsigned int, of minimum size (maxSymbolValuePtr[0]+1).
@ -132,14 +132,14 @@ FSE_count():
if FSE_isError(return), it's an error code. */ if FSE_isError(return), it's an error code. */
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize); size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
/* /*!
FSE_optimalTableLog(): FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met. dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
return : recommended tableLog (necessarily <= initial 'tableLog') */ return : recommended tableLog (necessarily <= initial 'tableLog') */
unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue); unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue);
/* /*!
FSE_normalizeCount(): FSE_normalizeCount():
normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@ -147,13 +147,13 @@ FSE_normalizeCount():
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/* /*!
FSE_NCountWriteBound(): FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog' Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'
Typically useful for allocation purpose. */ Typically useful for allocation purpose. */
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/* /*!
FSE_writeNCount(): FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'. Compactly save 'normalizedCounter' into 'buffer'.
return : size of the compressed table return : size of the compressed table
@ -161,21 +161,21 @@ FSE_writeNCount():
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/* /*!
Constructor and Destructor of type FSE_CTable Constructor and Destructor of type FSE_CTable
Note that its size depends on 'tableLog' and 'maxSymbolValue' */ Note that its size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct); void FSE_freeCTable (FSE_CTable* ct);
/* /*!
FSE_buildCTable(): FSE_buildCTable():
Builds 'ct', which must be already allocated, using FSE_createCTable() Builds 'ct', which must be already allocated, using FSE_createCTable()
return : 0 return : 0
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/* /*!
FSE_compress_usingCTable(): FSE_compress_usingCTable():
Compress 'src' using 'ct' into 'dst' which must be already allocated Compress 'src' using 'ct' into 'dst' which must be already allocated
return : size of compressed data (<= maxDstSize) return : size of compressed data (<= maxDstSize)
@ -183,7 +183,7 @@ FSE_compress_usingCTable():
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct); size_t FSE_compress_usingCTable (void* dst, size_t maxDstSize, const void* src, size_t srcSize, const FSE_CTable* ct);
/* /*!
Tutorial : Tutorial :
---------- ----------
The first step is to count all symbols. FSE_count() does this job very fast. The first step is to count all symbols. FSE_count() does this job very fast.
@ -229,7 +229,7 @@ If there is an error, the function will return an ErrorCode (which can be tested
/* *** DECOMPRESSION *** */ /* *** DECOMPRESSION *** */
/* /*!
FSE_readNCount(): FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'. Read compactly saved 'normalizedCounter' from 'rBuffer'.
return : size read from 'rBuffer' return : size read from 'rBuffer'
@ -237,21 +237,21 @@ FSE_readNCount():
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/* /*!
Constructor and Destructor of type FSE_DTable Constructor and Destructor of type FSE_DTable
Note that its size depends on 'tableLog' */ Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog); FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt); void FSE_freeDTable(FSE_DTable* dt);
/* /*!
FSE_buildDTable(): FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable() Builds 'dt', which must be already allocated, using FSE_createDTable()
return : 0, return : 0,
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/* /*!
FSE_decompress_usingDTable(): FSE_decompress_usingDTable():
Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt' Decompress compressed source 'cSrc' of size 'cSrcSize' using 'dt'
into 'dst' which must be already allocated. into 'dst' which must be already allocated.
@ -259,7 +259,7 @@ FSE_decompress_usingDTable():
or an errorCode, which can be tested using FSE_isError() */ or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); size_t FSE_decompress_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/* /*!
Tutorial : Tutorial :
---------- ----------
(Note : these functions only decompress FSE-compressed blocks. (Note : these functions only decompress FSE-compressed blocks.

53
lib/fse_static.h
View File

@ -40,31 +40,31 @@ extern "C" {
#endif #endif
/****************************************** /* *****************************************
* FSE API compatible with DLL * Dependencies
******************************************/ *******************************************/
#include "fse.h" #include "fse.h"
#include "bitstream.h" #include "bitstream.h"
/****************************************** /* *****************************************
* Static allocation * Static allocation
******************************************/ *******************************************/
/* FSE buffer bounds */ /* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512 #define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7)) #define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */ /* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/****************************************** /* *****************************************
* FSE advanced API * FSE advanced API
******************************************/ *******************************************/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize); size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const unsigned char* src, size_t srcSize);
/* same as FSE_count(), but blindly trust that all values within src are <= maxSymbolValuePtr[0] */ /* same as FSE_count(), but blindly trust that all values within src are <= *maxSymbolValuePtr */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */ /* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
@ -79,10 +79,10 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/* build a fake FSE_DTable, designed to always generate the same symbolValue */ /* build a fake FSE_DTable, designed to always generate the same symbolValue */
/****************************************** /* *****************************************
* FSE symbol compression API * FSE symbol compression API
******************************************/ *******************************************/
/* /*!
This API consists of small unitary functions, which highly benefit from being inlined. This API consists of small unitary functions, which highly benefit from being inlined.
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary. You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically. Visual seems to do it automatically.
@ -103,7 +103,7 @@ static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsig
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/* /*!
These functions are inner components of FSE_compress_usingCTable(). These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources. They allow the creation of custom streams, mixing multiple tables and bit sources.
@ -147,9 +147,9 @@ If there is an error, it returns an errorCode (which can be tested using FSE_isE
*/ */
/****************************************** /* *****************************************
* FSE symbol decompression API * FSE symbol decompression API
******************************************/ *******************************************/
typedef struct typedef struct
{ {
size_t state; size_t state;
@ -163,7 +163,7 @@ static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bi
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/* /*!
Let's now decompose FSE_decompress_usingDTable() into its unitary components. Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream, You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**. and also any other bitFields you put in, **in reverse order**.
@ -213,16 +213,16 @@ Check also the states. There might be some symbols left there, if some high prob
*/ */
/****************************************** /* *****************************************
* FSE unsafe API * FSE unsafe API
******************************************/ *******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/****************************************** /* *****************************************
* Implementation of inline functions * Implementation of inlined functions
******************************************/ *******************************************/
typedef struct typedef struct
{ {
int deltaFindState; int deltaFindState;
@ -231,10 +231,12 @@ typedef struct
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{ {
const U32 tableLog = ( (const U16*) ct) [0]; const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = *u16ptr;
statePtr->value = (ptrdiff_t)1<<tableLog; statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = ((const U16*) ct) + 2; statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = (const void*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog; statePtr->stateLog = tableLog;
} }
@ -269,7 +271,8 @@ typedef struct
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{ {
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt; const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD); BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1; DStatePtr->table = dt + 1;

81
lib/huff0.c
View File

@ -32,7 +32,7 @@
- Public forum : https://groups.google.com/forum/#!forum/lz4c - Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */ ****************************************************************** */
/**************************************************************** /* **************************************************************
* Compiler specifics * Compiler specifics
****************************************************************/ ****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
@ -57,7 +57,7 @@
#endif #endif
/**************************************************************** /* **************************************************************
* Includes * Includes
****************************************************************/ ****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */ #include <stdlib.h> /* malloc, free, qsort */
@ -68,7 +68,7 @@
#include "fse.h" /* header compression */ #include "fse.h" /* header compression */
/**************************************************************** /* **************************************************************
* Constants * Constants
****************************************************************/ ****************************************************************/
#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
@ -80,27 +80,21 @@
#endif #endif
/**************************************************************** /* **************************************************************
* Error Management * Error Management
****************************************************************/ ****************************************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/****************************************** /* *******************************************************
* Helper functions
******************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*********************************************************
* Huff0 : Huffman block compression * Huff0 : Huffman block compression
*********************************************************/ *********************************************************/
typedef struct HUF_CElt_s { struct HUF_CElt_s {
U16 val; U16 val;
BYTE nbBits; BYTE nbBits;
} HUF_CElt ; }; /* typedef'd to HUF_CElt within huff0_static.h */
typedef struct nodeElt_s { typedef struct nodeElt_s {
U32 count; U32 count;
@ -320,7 +314,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
/* sort, decreasing order */ /* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue); HUF_sort(huffNode, count, maxSymbolValue);
// init for parents /* init for parents */
nonNullRank = maxSymbolValue; nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--; while(huffNode[nonNullRank].count == 0) nonNullRank--;
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
@ -330,7 +324,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
huffNode0[0].count = (U32)(1U<<31); huffNode0[0].count = (U32)(1U<<31);
// create parents /* create parents */
while (nodeNb <= nodeRoot) while (nodeNb <= nodeRoot)
{ {
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
@ -340,7 +334,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
nodeNb++; nodeNb++;
} }
// distribute weights (unlimited tree height) /* distribute weights (unlimited tree height) */
huffNode[nodeRoot].nbBits = 0; huffNode[nodeRoot].nbBits = 0;
for (n=nodeRoot-1; n>=STARTNODE; n--) for (n=nodeRoot-1; n>=STARTNODE; n--)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
@ -368,9 +362,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
} }
} }
for (n=0; n<=maxSymbolValue; n++) for (n=0; n<=maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; // push nbBits per symbol, symbol order tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n=0; n<=maxSymbolValue; n++) for (n=0; n<=maxSymbolValue; n++)
tree[n].val = valPerRank[tree[n].nbBits]++; // assign value within rank, symbol order tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
} }
return maxNbBits; return maxNbBits;
@ -636,12 +630,12 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0; U32 tableLog = 0;
const BYTE* ip = (const BYTE*) src; size_t iSize;
size_t iSize = ip[0];
U32 nbSymbols = 0; U32 nbSymbols = 0;
U32 n; U32 n;
U32 nextRankStart; U32 nextRankStart;
HUF_DEltX2* const dt = (HUF_DEltX2*)(DTable + 1); void* const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
@ -730,7 +724,8 @@ size_t HUF_decompress1X2_usingDTable(
BYTE* const oend = op + dstSize; BYTE* const oend = op + dstSize;
size_t errorCode; size_t errorCode;
const U32 dtLog = DTable[0]; const U32 dtLog = DTable[0];
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1; const void* dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1;
BIT_DStream_t bitD; BIT_DStream_t bitD;
errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode; if (HUF_isError(errorCode)) return errorCode;
@ -770,8 +765,8 @@ size_t HUF_decompress4X2_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc; const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst; BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)DTable) +1; const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1;
const U32 dtLog = DTable[0]; const U32 dtLog = DTable[0];
size_t errorCode; size_t errorCode;
@ -978,9 +973,9 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
rankVal_t rankVal; rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols; U32 tableLog, maxW, sizeOfSort, nbSymbols;
const U32 memLog = DTable[0]; const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src; size_t iSize;
size_t iSize = ip[0]; void* dtPtr = DTable;
HUF_DEltX4* const dt = ((HUF_DEltX4*)DTable) + 1; HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
@ -1127,7 +1122,8 @@ size_t HUF_decompress1X4_usingDTable(
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0]; const U32 dtLog = DTable[0];
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1; const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
size_t errorCode; size_t errorCode;
/* Init */ /* Init */
@ -1170,8 +1166,8 @@ size_t HUF_decompress4X4_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc; const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst; BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)DTable) +1; const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
const U32 dtLog = DTable[0]; const U32 dtLog = DTable[0];
size_t errorCode; size_t errorCode;
@ -1352,8 +1348,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
U32 tableLog, maxW, sizeOfSort, nbSymbols; U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal; rankVal_t rankVal;
const U32 memLog = DTable[0]; const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src; size_t iSize;
size_t iSize = ip[0];
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
@ -1418,8 +1413,10 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* fill tables */ /* fill tables */
{ {
HUF_DDescX6* DDescription = (HUF_DDescX6*)(DTable+1); void* ddPtr = DTable+1;
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(memLog-1))); HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr;
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr;
HUF_DSeqX6 DSeq; HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc; HUF_DDescX6 DDesc;
DSeq.sequence = 0; DSeq.sequence = 0;
@ -1478,8 +1475,10 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog) static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{ {
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); const void* const ddPtr = DTable+1;
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
BYTE* const pStart = p; BYTE* const pStart = p;
/* up to 16 symbols at a time */ /* up to 16 symbols at a time */
@ -1557,8 +1556,10 @@ size_t HUF_decompress4X6_usingDTable(
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0]; const U32 dtLog = DTable[0];
const HUF_DDescX6* dd = (const HUF_DDescX6*)(DTable+1); const void* const ddPtr = DTable+1;
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(DTable + 1 + ((size_t)1<<(dtLog-1))); const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
size_t errorCode; size_t errorCode;
/* Init */ /* Init */

16
lib/huff0.h
View File

@ -40,25 +40,25 @@ extern "C" {
#endif #endif
/****************************************** /* ****************************************
* Dependency * Dependency
******************************************/ ******************************************/
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/****************************************** /* ****************************************
* Huff0 simple functions * Huff0 simple functions
******************************************/ ******************************************/
size_t HUF_compress(void* dst, size_t maxDstSize, size_t HUF_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize); const void* src, size_t srcSize);
size_t HUF_decompress(void* dst, size_t dstSize, size_t HUF_decompress(void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize); const void* cSrc, size_t cSrcSize);
/* /*!
HUF_compress(): HUF_compress():
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster if maxDstSize >= HUF_compressBound(srcSize). 'dst' buffer must be already allocated. Compression runs faster if maxDstSize >= HUF_compressBound(srcSize).
Note : srcSize must be <= 128 KB Note : srcSize must be <= 128 KB
return : size of compressed data (<= maxDstSize) @return : size of compressed data (<= maxDstSize)
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression) if return == 1, srcData is a single repeated byte symbol (RLE compression)
if HUF_isError(return), compression failed (more details using HUF_getErrorName()) if HUF_isError(return), compression failed (more details using HUF_getErrorName())
@ -68,12 +68,12 @@ HUF_decompress():
into already allocated destination buffer 'dst', of size 'dstSize'. into already allocated destination buffer 'dst', of size 'dstSize'.
'dstSize' must be the exact size of original (uncompressed) data. 'dstSize' must be the exact size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate. Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate.
return : size of regenerated data (== dstSize) @return : size of regenerated data (== dstSize)
or an error code, which can be tested using HUF_isError() or an error code, which can be tested using HUF_isError()
*/ */
/****************************************** /* ****************************************
* Tool functions * Tool functions
******************************************/ ******************************************/
size_t HUF_compressBound(size_t size); /* maximum compressed size */ size_t HUF_compressBound(size_t size); /* maximum compressed size */
@ -83,7 +83,7 @@ unsigned HUF_isError(size_t code); /* tells if a return value is an er
const char* HUF_getErrorName(size_t code); /* provides error code string (useful for debugging) */ const char* HUF_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/****************************************** /* ****************************************
* Advanced functions * Advanced functions
******************************************/ ******************************************/
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);

51
lib/huff0_static.h
View File

@ -40,13 +40,13 @@ extern "C" {
#endif #endif
/****************************************** /* ****************************************
* Dependency * Dependency
******************************************/ ******************************************/
#include "huff0.h" #include "huff0.h"
/****************************************** /* ****************************************
* Static allocation macros * Static allocation macros
******************************************/ ******************************************/
/* Huff0 buffer bounds */ /* Huff0 buffer bounds */
@ -64,14 +64,57 @@ extern "C" {
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog } unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
/****************************************** /* ****************************************
* Advanced functions * Advanced decompression functions
******************************************/ ******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */ size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
/* ****************************************
* Huff0 detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. build Huffman table from count using HUF_buildCTable()
3. save Huffman table to memory buffer using HUF_writeCTable()
4. encode the data stream using HUF_compress_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*!
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif