sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

minor code formatting

Browse Source 
added a trace during sequence encoding
This commit is contained in:
Yann Collet 2017-10-13 02:36:16 -07:00
parent 7f6a783862
commit a9e5705077
2 changed files with 93 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
lib/compress/zstd_compress.c
View File

@ -1238,7 +1238,7 @@ static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy,
ostart[4] = (BYTE)(cLitSize >> 10); ostart[4] = (BYTE)(cLitSize >> 10);
break; break;
} }
default: /* not possible : lhSize is {3,4,5} */ default: /* not possible : lhSize is {3,4,5} */
assert(0); assert(0);
} }
return lhSize+cLitSize; return lhSize+cLitSize;
@ -1273,7 +1273,8 @@ typedef enum {
ZSTD_defaultAllowed = 1 ZSTD_defaultAllowed = 1
} ZSTD_defaultPolicy_e; } ZSTD_defaultPolicy_e;
MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType( MEM_STATIC
symbolEncodingType_e ZSTD_selectEncodingType(
FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq, FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq,
U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed) U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed)
{ {
@ -1288,10 +1289,12 @@ MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType(
*repeatMode = FSE_repeat_check; *repeatMode = FSE_repeat_check;
return set_rle; return set_rle;
} }
if (isDefaultAllowed && (*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { if ( isDefaultAllowed
&& (*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) {
return set_repeat; return set_repeat;
} }
if (isDefaultAllowed && ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1))))) { if ( isDefaultAllowed
&& ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1)))) ) {
*repeatMode = FSE_repeat_valid; *repeatMode = FSE_repeat_valid;
return set_basic; return set_basic;
} }
@ -1299,7 +1302,8 @@ MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType(
return set_compressed; return set_compressed;
} }
MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, MEM_STATIC
size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type, FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type,
U32* count, U32 max, U32* count, U32 max,
BYTE const* codeTable, size_t nbSeq, BYTE const* codeTable, size_t nbSeq,
@ -1317,7 +1321,7 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
case set_repeat: case set_repeat:
return 0; return 0;
case set_basic: case set_basic:
CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */
return 0; return 0;
case set_compressed: { case set_compressed: {
S16 norm[MaxSeq + 1]; S16 norm[MaxSeq + 1];
@ -1339,11 +1343,13 @@ MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity,
} }
} }
MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity, MEM_STATIC
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, size_t ZSTD_encodeSequences(
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, void* dst, size_t dstCapacity,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets) FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
seqDef const* sequences, size_t nbSeq, int longOffsets)
{ {
BIT_CStream_t blockStream; BIT_CStream_t blockStream;
FSE_CState_t stateMatchLength; FSE_CState_t stateMatchLength;
@ -1380,8 +1386,12 @@ MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity,
BYTE const ofCode = ofCodeTable[n]; BYTE const ofCode = ofCodeTable[n];
BYTE const mlCode = mlCodeTable[n]; BYTE const mlCode = mlCodeTable[n];
U32 const llBits = LL_bits[llCode]; U32 const llBits = LL_bits[llCode];
U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ U32 const ofBits = ofCode;
U32 const mlBits = ML_bits[mlCode]; U32 const mlBits = ML_bits[mlCode];
DEBUGLOG(6, "encoding: litlen:%u - matchlen:%u - offCode:%u",
sequences[n].litLength,
sequences[n].matchLength + MINMATCH,
sequences[n].offset); /* 32b*/ /* 64b*/
/* (7)*/ /* (7)*/ /* (7)*/ /* (7)*/
FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
@ -1462,7 +1472,7 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
/* convert length/distances into codes */ /* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr); ZSTD_seqToCodes(seqStorePtr);
/* CTable for Literal Lengths */ /* build CTable for Literal Lengths */
{ U32 max = MaxLL; { U32 max = MaxLL;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace); size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace);
LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed); LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed);
@ -1472,11 +1482,11 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
if (ZSTD_isError(countSize)) return countSize; if (ZSTD_isError(countSize)) return countSize;
op += countSize; op += countSize;
} } } }
/* CTable for Offsets */ /* build CTable for Offsets */
{ U32 max = MaxOff; { U32 max = MaxOff;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace); size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace);
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_defaultPolicy_e const defaultPolicy = max <= DefaultMaxOff ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy); Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy);
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
@ -1484,7 +1494,7 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
if (ZSTD_isError(countSize)) return countSize; if (ZSTD_isError(countSize)) return countSize;
op += countSize; op += countSize;
} } } }
/* CTable for MatchLengths */ /* build CTable for MatchLengths */
{ U32 max = MaxML; { U32 max = MaxML;
size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace); size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace);
MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed); MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed);
@ -1497,13 +1507,15 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
{ size_t const streamSize = ZSTD_encodeSequences(op, oend - op, { size_t const bitstreamSize = ZSTD_encodeSequences(
CTable_MatchLength, mlCodeTable, op, oend - op,
CTable_OffsetBits, ofCodeTable, CTable_MatchLength, mlCodeTable,
CTable_LitLength, llCodeTable, CTable_OffsetBits, ofCodeTable,
sequences, nbSeq, longOffsets); CTable_LitLength, llCodeTable,
if (ZSTD_isError(streamSize)) return streamSize; sequences, nbSeq,
op += streamSize; longOffsets);
if (ZSTD_isError(bitstreamSize)) return bitstreamSize;
op += bitstreamSize;
} }
return op - ostart; return op - ostart;
@ -1517,27 +1529,27 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
{ {
size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams, size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams,
dst, dstCapacity); dst, dstCapacity);
size_t const minGain = ZSTD_minGain(srcSize);
size_t const maxCSize = srcSize - minGain;
/* If the srcSize <= dstCapacity, then there is enough space to write a /* If the srcSize <= dstCapacity, then there is enough space to write a
* raw uncompressed block. Since we ran out of space, the block must not * raw uncompressed block. Since we ran out of space, the block must not
* be compressible, so fall back to a raw uncompressed block. * be compressible, so fall back to a raw uncompressed block.
*/ */
int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity; int const uncompressibleError = (cSize == ERROR(dstSize_tooSmall)) && (srcSize <= dstCapacity);
if (ZSTD_isError(cSize) && !uncompressibleError) if (ZSTD_isError(cSize) && !uncompressibleError)
return cSize; return cSize;
/* Check compressibility */ /* Check compressibility */
if (cSize >= maxCSize || uncompressibleError) { { size_t const minGain = ZSTD_minGain(srcSize); /* note : fixed formula, maybe should depend on compression level, or strategy */
entropy->hufCTable_repeatMode = HUF_repeat_none; size_t const maxCSize = srcSize - minGain;
entropy->offcode_repeatMode = FSE_repeat_none; if (cSize >= maxCSize || uncompressibleError) {
entropy->matchlength_repeatMode = FSE_repeat_none; entropy->hufCTable_repeatMode = HUF_repeat_none;
entropy->litlength_repeatMode = FSE_repeat_none; entropy->offcode_repeatMode = FSE_repeat_none;
return 0; entropy->matchlength_repeatMode = FSE_repeat_none;
} entropy->litlength_repeatMode = FSE_repeat_none;
return 0; /* block not compressed */
} }
assert(!ZSTD_isError(cSize)); assert(!ZSTD_isError(cSize));
/* confirm repcodes */ /* block is compressed => confirm repcodes in history */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; } { int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; }
return cSize; return cSize;
} }
@ -1559,9 +1571,9 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict
ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict } ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict }
}; };
ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
assert((U32)strat >= (U32)ZSTD_fast); assert((U32)strat >= (U32)ZSTD_fast);
assert((U32)strat <= (U32)ZSTD_btultra); assert((U32)strat <= (U32)ZSTD_btultra);
return blockCompressor[extDict!=0][(U32)strat]; return blockCompressor[extDict!=0][(U32)strat];
} }
@ -1574,28 +1586,28 @@ static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr,
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
const BYTE* const base = zc->base; if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1)
const BYTE* const istart = (const BYTE*)src; return 0; /* don't even attempt compression below a certain srcSize */
const U32 current = (U32)(istart-base);
size_t lastLLSize;
const BYTE* anchor;
U32 const extDict = zc->lowLimit < zc->dictLimit;
const ZSTD_blockCompressor blockCompressor =
zc->appliedParams.ldmParams.enableLdm
? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm)
: ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict);
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */
ZSTD_resetSeqStore(&(zc->seqStore)); ZSTD_resetSeqStore(&(zc->seqStore));
if (current > zc->nextToUpdate + 384)
zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* limited update after finding a very long match */
lastLLSize = blockCompressor(zc, src, srcSize);
/* Last literals */
anchor = (const BYTE*)src + srcSize - lastLLSize;
ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize);
/* limited update after a very long match */
{ const BYTE* const base = zc->base;
const BYTE* const istart = (const BYTE*)src;
const U32 current = (U32)(istart-base);
if (current > zc->nextToUpdate + 384)
zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384));
}
/* find and store sequences */
{ U32 const extDict = zc->lowLimit < zc->dictLimit;
const ZSTD_blockCompressor blockCompressor =
zc->appliedParams.ldmParams.enableLdm
? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm)
: ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict);
size_t const lastLLSize = blockCompressor(zc, src, srcSize);
const BYTE* const anchor = (const BYTE*)src + srcSize - lastLLSize;
ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize);
}
/* encode */
return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize); return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize);
} }
@ -1618,13 +1630,13 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart; BYTE* op = ostart;
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
assert(cctx->appliedParams.cParams.windowLog <= 31);
if (cctx->appliedParams.fParams.checksumFlag && srcSize) if (cctx->appliedParams.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize); XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) { while (remaining) {
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
size_t cSize;
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE)
return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
@ -1666,34 +1678,37 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
else cctx->nextToUpdate -= correction; else cctx->nextToUpdate -= correction;
DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit); DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit);
} }
/* enforce maxDist */
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
/* enforce maxDist */
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
} }
cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); { size_t cSize = ZSTD_compressBlock_internal(cctx,
if (ZSTD_isError(cSize)) return cSize; op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0) { /* block is not compressible */ if (cSize == 0) { /* block is not compressible */
U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3);
if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall);
MEM_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ MEM_writeLE32(op, cBlockHeader24); /* 4th byte will be overwritten */
memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); memcpy(op + ZSTD_blockHeaderSize, ip, blockSize);
cSize = ZSTD_blockHeaderSize+blockSize; cSize = ZSTD_blockHeaderSize + blockSize;
} else { } else {
U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader24); MEM_writeLE24(op, cBlockHeader24);
cSize += ZSTD_blockHeaderSize; cSize += ZSTD_blockHeaderSize;
} }
remaining -= blockSize; ip += blockSize;
dstCapacity -= cSize; assert(remaining >= blockSize);
ip += blockSize; remaining -= blockSize;
op += cSize; op += cSize;
} assert(dstCapacity >= cSize);
dstCapacity -= cSize;
} }
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
return op-ostart; return op-ostart;

2
lib/compress/zstd_compress.h
View File

@ -127,7 +127,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6) #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
static const BYTE* g_start = NULL; static const BYTE* g_start = NULL;
U32 const pos = (U32)((const BYTE*)literals - g_start); U32 const pos = (U32)((const BYTE*)literals - g_start);
if (g_start==NULL) g_start = (const BYTE*)literals; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
if ((pos > 0) && (pos < 1000000000)) if ((pos > 0) && (pos < 1000000000))
DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u", DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);