sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

zdict stores standard rep-offset.

Browse Source 
It can use custom ones, but the proper formula and impact on statistics is not done yet.
This commit is contained in:
Yann Collet 2016-06-16 01:14:41 +02:00
parent 736d419289
commit ad39b7a718
1 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/dictBuilder/zdict.c
View File

@ -727,6 +727,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
} }
huffLog = (U32)errorCode; huffLog = (U32)errorCode;
/* looking for most common first offsets */
{ U32 offset;
for (offset=1; offset<MAXREPOFFSET; offset++)
ZDICT_insertSortCount(bestRepOffset, offset, repOffset[offset]);
}
/* note : the result of this phase should be used to better appreciate the impact on statistics */
total=0; for (u=0; u<=OFFCODE_MAX; u++) total+=offcodeCount[u]; total=0; for (u=0; u<=OFFCODE_MAX; u++) total+=offcodeCount[u];
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, OFFCODE_MAX); errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, OFFCODE_MAX);
if (FSE_isError(errorCode)) { if (FSE_isError(errorCode)) {
@ -754,11 +761,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
} }
llLog = (U32)errorCode; llLog = (U32)errorCode;
{ U32 offset;
for (offset=1; offset<MAXREPOFFSET; offset++)
ZDICT_insertSortCount(bestRepOffset, offset, repOffset[offset]);
}
/* write result to buffer */ /* write result to buffer */
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
@ -810,12 +812,17 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
DISPLAYLEVEL(1, "not enough space to write RepOffsets"); DISPLAYLEVEL(1, "not enough space to write RepOffsets");
goto _cleanup; goto _cleanup;
} }
# if 0
MEM_writeLE32(dstPtr+0, bestRepOffset[0].offset); MEM_writeLE32(dstPtr+0, bestRepOffset[0].offset);
MEM_writeLE32(dstPtr+4, bestRepOffset[1].offset); MEM_writeLE32(dstPtr+4, bestRepOffset[1].offset);
MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset); MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset);
//MEM_writeLE32(dstPtr+0, 1); #else
//MEM_writeLE32(dstPtr+4, 4); /* at this stage, we don't use the result of "most common first offset",
//MEM_writeLE32(dstPtr+8, 8); as the impact of statistics is not properly evaluated */
MEM_writeLE32(dstPtr+0, repStartValue[0]);
MEM_writeLE32(dstPtr+4, repStartValue[1]);
MEM_writeLE32(dstPtr+8, repStartValue[2]);
#endif
dstPtr += 12; dstPtr += 12;
eSize += 12; eSize += 12;