sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-08 00:04:02 -04:00
Code Issues Projects Releases Wiki Activity

reduce splitBlock arguments

Browse Source
This commit is contained in:
Yann Collet 2024-10-23 19:59:03 -07:00
parent 94d7b07425
commit ca6e55cbf5
3 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/compress/zstd_compress.c
View File

@ -4507,11 +4507,11 @@ static size_t ZSTD_optimalBlockSize(ZSTD_CCtx* cctx, const void* src, size_t src
/* dynamic splitting has a cpu cost for analysis,
* due to that cost it's only used for higher levels */
if (strat >= ZSTD_btopt)
return ZSTD_splitBlock(src, srcSize, blockSizeMax, split_lvl3, cctx->tmpWorkspace, cctx->tmpWkspSize);
return ZSTD_splitBlock(src, blockSizeMax, split_lvl3, cctx->tmpWorkspace, cctx->tmpWkspSize);
if (strat >= ZSTD_lazy2)
return ZSTD_splitBlock(src, srcSize, blockSizeMax, split_lvl2, cctx->tmpWorkspace, cctx->tmpWkspSize);
return ZSTD_splitBlock(src, blockSizeMax, split_lvl2, cctx->tmpWorkspace, cctx->tmpWkspSize);
if (strat >= ZSTD_greedy)
return ZSTD_splitBlock(src, srcSize, blockSizeMax, split_lvl1, cctx->tmpWorkspace, cctx->tmpWkspSize);
return ZSTD_splitBlock(src, blockSizeMax, split_lvl1, cctx->tmpWorkspace, cctx->tmpWkspSize);
/* blind split strategy
* heuristic value, tested as being "generally better".
* no cpu cost, but can over-split homegeneous data.

25
lib/compress/zstd_preSplit.c
View File

@ -141,16 +141,15 @@ static void removeEvents(Fingerprint* acc, const Fingerprint* slice)
#define CHUNKSIZE (8 << 10)
/* Note: technically, we use CHUNKSIZE, so that's 8 KB */
static size_t ZSTD_splitBlock_byChunks(const void* src, size_t srcSize,
size_t blockSizeMax, RecordEvents_f record_f,
static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize,
RecordEvents_f record_f,
void* workspace, size_t wkspSize)
{
FPStats* const fpstats = (FPStats*)workspace;
const char* p = (const char*)src;
const char* p = (const char*)blockStart;
int penalty = THRESHOLD_PENALTY;
size_t pos = 0;
if (srcSize <= blockSizeMax) return srcSize;
assert(blockSizeMax == (128 << 10));
assert(blockSize == (128 << 10));
assert(workspace != NULL);
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
@ -158,7 +157,7 @@ static size_t ZSTD_splitBlock_byChunks(const void* src, size_t srcSize,
initStats(fpstats);
record_f(&fpstats->pastEvents, p, CHUNKSIZE);
for (pos = CHUNKSIZE; pos <= blockSizeMax - CHUNKSIZE; pos += CHUNKSIZE) {
for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) {
record_f(&fpstats->newEvents, p + pos, CHUNKSIZE);
if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty)) {
return pos;
@ -167,21 +166,21 @@ static size_t ZSTD_splitBlock_byChunks(const void* src, size_t srcSize,
if (penalty > 0) penalty--;
}
}
assert(pos == blockSizeMax);
return blockSizeMax;
assert(pos == blockSize);
return blockSize;
(void)flushEvents; (void)removeEvents;
}
size_t ZSTD_splitBlock(const void* src, size_t srcSize,
size_t blockSizeMax, ZSTD_SplitBlock_strategy_e splitStrat,
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
ZSTD_SplitBlock_strategy_e splitStrat,
void* workspace, size_t wkspSize)
{
if (splitStrat == split_lvl3)
return ZSTD_splitBlock_byChunks(src, srcSize, blockSizeMax, FP_RECORD_RATE(1), workspace, wkspSize);
return ZSTD_splitBlock_byChunks(blockStart, blockSize, FP_RECORD_RATE(1), workspace, wkspSize);
if (splitStrat == split_lvl2)
return ZSTD_splitBlock_byChunks(src, srcSize, blockSizeMax, FP_RECORD_RATE(5), workspace, wkspSize);
return ZSTD_splitBlock_byChunks(blockStart, blockSize, FP_RECORD_RATE(5), workspace, wkspSize);
assert(splitStrat == split_lvl1);
return ZSTD_splitBlock_byChunks(src, srcSize, blockSizeMax, FP_RECORD_RATE(11), workspace, wkspSize);
return ZSTD_splitBlock_byChunks(blockStart, blockSize, FP_RECORD_RATE(11), workspace, wkspSize);
}

4
lib/compress/zstd_preSplit.h
View File

@ -29,8 +29,8 @@ typedef enum { split_lvl1, split_lvl2, split_lvl3 } ZSTD_SplitBlock_strategy_e;
* therefore @blockSizeMax must be == 128 KB.
* This could be extended to smaller sizes in the future.
*/
size_t ZSTD_splitBlock(const void* src, size_t srcSize,
size_t blockSizeMax, ZSTD_SplitBlock_strategy_e splitStrat,
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
ZSTD_SplitBlock_strategy_e splitStrat,
void* workspace, size_t wkspSize);
#if defined (__cplusplus)