sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-10 00:03:36 -04:00
Code Issues Projects Releases Wiki Activity

frame format can support window sizes up to 128 MB

Browse Source
This commit is contained in:
Yann Collet 2016-03-08 18:24:21 +01:00
parent ff14b351f5
commit 95cd0c2e26
5 changed files with 27 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/bitstream.h
View File

@ -185,7 +185,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t ma
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
{ {
static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF }; /* up to 25 bits */ static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos; bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos;
bitC->bitPos += nbBits; bitC->bitPos += nbBits;
} }

22
lib/zstd_compress.c
View File

@ -74,7 +74,6 @@ size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) +
/*-************************************* /*-*************************************
* Sequence storage * Sequence storage
***************************************/ ***************************************/
static void ZSTD_resetSeqStore(seqStore_t* ssPtr) static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
{ {
ssPtr->offset = ssPtr->offsetStart; ssPtr->offset = ssPtr->offsetStart;
@ -709,10 +708,10 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/ BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 17 */ /* 17 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 17 */ /* 17 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */ if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */ FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 17 */ /* 27 */
FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */ FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 26 */ /* 36 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */ if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 42 */ /* 24 bits max in 32-bits mode */ BIT_addBits(&blockStream, offset, nbBits); /* 31 */ /* 62 */ /* 24 bits max in 32-bits mode */
BIT_flushBits(&blockStream); /* 7 */ /* 7 */ BIT_flushBits(&blockStream); /* 7 */ /* 7 */
} }
@ -733,10 +732,10 @@ _check_compressibility:
} }
/*! ZSTD_storeSeq /*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
@offsetCode : distance to match, or 0 == repCode `offsetCode` : distance to match, or 0 == repCode.
@matchCode : matchLength - MINMATCH `matchCode` : matchLength - MINMATCH
*/ */
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode) MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
{ {
@ -771,8 +770,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
} else { } else {
MEM_writeLE32(seqStorePtr->dumps, (U32)((litLength<<1)+1)); MEM_writeLE32(seqStorePtr->dumps, (U32)((litLength<<1)+1));
seqStorePtr->dumps += 3; seqStorePtr->dumps += 3;
} } } }
} }
else *(seqStorePtr->litLength++) = (BYTE)litLength; else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */ /* match offset */
@ -791,8 +789,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
} else { } else {
MEM_writeLE32(seqStorePtr->dumps, (U32)((matchCode<<1)+1)); MEM_writeLE32(seqStorePtr->dumps, (U32)((matchCode<<1)+1));
seqStorePtr->dumps += 3; seqStorePtr->dumps += 3;
} } } }
} }
else *(seqStorePtr->matchLength++) = (BYTE)matchCode; else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
} }
@ -1972,7 +1969,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
op += cSize; op += cSize;
} }
#if ZSTD_OPT_DEBUG == 3 #if ZSTD_OPT_DEBUG == 3
ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4); ssPtr->realMatchSum += ssPtr->realSeqSum * ((zc->params.searchLength == 3) ? 3 : 4);
printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc); printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d priceFunc=%d\n", (float)ssPtr->realMatchSum/ssPtr->realSeqSum, (float)ssPtr->realLitSum/ssPtr->realSeqSum, 100.0*ssPtr->realMatchSum/(ssPtr->realMatchSum+ssPtr->realLitSum), 100.0*ssPtr->realLitSum/(ssPtr->realMatchSum+ssPtr->realLitSum), ssPtr->realRepSum, ssPtr->realSeqSum, ssPtr->priceFunc);

6
lib/zstd_decompress.c
View File

@ -648,9 +648,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls)
/* Offset */ /* Offset */
{ {
static const U32 offsetPrefix[MaxOff+1] = { static const U32 offsetPrefix[MaxOff+1] = {
1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256, 1 /*fake*/, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 0x100,
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000,
524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 }; 0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, /*fake*/ 1, 1, 1, 1 };
U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ U32 offsetCode = FSE_peakSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
U32 nbBits = offsetCode - 1; U32 nbBits = offsetCode - 1;
if (offsetCode==0) nbBits = 0; /* cmove */ if (offsetCode==0) nbBits = 0; /* cmove */

1
lib/zstd_internal.h
View File

@ -91,6 +91,7 @@ static const size_t ZSTD_frameHeaderSize_min = 5;
#define MINMATCH 4 #define MINMATCH 4
#define REPCODE_STARTVALUE 1 #define REPCODE_STARTVALUE 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12
#define Litbits 8 #define Litbits 8
#define MLbits 7 #define MLbits 7

27
lib/zstd_static.h
View File

@ -57,20 +57,19 @@ extern "C" {
/*-************************************* /*-*************************************
* Types * Types
***************************************/ ***************************************/
#define ZSTD_WINDOWLOG_MAX 26 #define ZSTD_WINDOWLOG_MAX 27
#define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11 #define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_HASHLOG_MAX 28
#define ZSTD_HASHLOG_MAX 28 #define ZSTD_HASHLOG_MIN 12
#define ZSTD_HASHLOG_MIN 12 #define ZSTD_HASHLOG3_MAX 16
#define ZSTD_HASHLOG3_MAX 24 #define ZSTD_HASHLOG3_MIN 16
#define ZSTD_HASHLOG3_MIN 2 #define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MAX 7 #define ZSTD_SEARCHLENGTH_MIN 3
#define ZSTD_SEARCHLENGTH_MIN 3 #define ZSTD_TARGETLENGTH_MIN 4
#define ZSTD_TARGETLENGTH_MIN 4
#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_TARGETLENGTH_MAX 999
/* from faster to stronger */ /* from faster to stronger */