sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-07 00:02:39 -05:00
Code Issues Projects Releases Wiki Activity

made search strategy switchable

Browse Source 
between cmov and branch
and use a simple heuristic based on wlog to select between them.

note: performance is not good on clang (yet)
This commit is contained in:
Yann Collet 2024-10-08 11:43:07 -07:00
parent 2cc600bab2
commit 186b132495
2 changed files with 47 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/compress/zstd_compress_internal.h
View File

@ -558,21 +558,21 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
} }
/* ZSTD_selectAddr: /* ZSTD_selectAddr:
* @return a >= b ? trueAddr : falseAddr, * @return index >= lowLimit ? candidate : backup,
* tries to force branchless codegen. */ * tries to force branchless codegen. */
MEM_STATIC const BYTE* MEM_STATIC const BYTE*
ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* trueAddr, const BYTE* falseAddr) ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* backup)
{ {
#if defined(__GNUC__) && defined(__x86_64__) #if defined(__GNUC__) && defined(__x86_64__)
__asm__ ( __asm__ (
"cmp %1, %2\n" "cmp %1, %2\n"
"cmova %3, %0\n" "cmova %3, %0\n"
: "+r"(trueAddr) : "+r"(candidate)
: "r"(index), "r"(lowLimit), "r"(falseAddr) : "r"(index), "r"(lowLimit), "r"(backup)
); );
return trueAddr; return candidate;
#else #else
return a >= b ? trueAddr : falseAddr; return index >= lowLimit ? candidate : backup;
#endif #endif
} }

72
lib/compress/zstd_fast.c
View File

@ -45,7 +45,7 @@ void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls);
if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */
ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p);
} } } } } } } }
} }
static static
@ -97,17 +97,34 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
} }
typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 currentIdx, U32 lowLimit, const BYTE* fakeAddress);
static int static int
ZSTD_findMatch_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 currentIdx, U32 lowLimit, const BYTE* fakeAddress) ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 currentIdx, U32 lowLimit, const BYTE* fakeAddress)
{ {
/* idx >= prefixStartIndex is a (somewhat) unpredictable branch. /* currentIdx >= lowLimit is a (somewhat) unpredictable branch.
* However expression below complies into conditional move. Since * However expression below compiles into conditional move.
* match is unlikely and we only *branch* on idxl0 > prefixLowestIndex */
* if there is a match, all branches become predictable. */
const BYTE* mvalAddr = ZSTD_selectAddr(currentIdx, lowLimit, matchAddress, fakeAddress); const BYTE* mvalAddr = ZSTD_selectAddr(currentIdx, lowLimit, matchAddress, fakeAddress);
return ((MEM_read32(currentPtr) == MEM_read32(mvalAddr)) & (currentIdx >= lowLimit)); return ((MEM_read32(currentPtr) == MEM_read32(mvalAddr)) & (currentIdx >= lowLimit));
} }
static int
ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 currentIdx, U32 lowLimit, const BYTE* fakeAddress)
{
/* using a branch instead of a cmov,
* because it's faster in scenarios where currentIdx >= lowLimit is generally true,
* aka almost all candidates are within range */
U32 mval; (void)fakeAddress;
if (currentIdx >= lowLimit) {
mval = MEM_read32(matchAddress);
} else {
mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */
}
return (MEM_read32(currentPtr) == mval);
}
/** /**
* If you squint hard enough (and ignore repcodes), the search operation at any * If you squint hard enough (and ignore repcodes), the search operation at any
@ -160,7 +177,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_noDict_generic( size_t ZSTD_compressBlock_fast_noDict_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, void const* src, size_t srcSize,
U32 const mls, U32 const unpredictable) U32 const mls, int useCmov)
{ {
const ZSTD_compressionParameters* const cParams = &ms->cParams; const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable; U32* const hashTable = ms->hashTable;
@ -204,7 +221,7 @@ size_t ZSTD_compressBlock_fast_noDict_generic(
size_t step; size_t step;
const BYTE* nextStep; const BYTE* nextStep;
const size_t kStepIncr = (1 << (kSearchStrength - 1)); const size_t kStepIncr = (1 << (kSearchStrength - 1));
(void)unpredictable; const ZSTD_match4Found findMatch = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch;
DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
ip0 += (ip0 == prefixStart); ip0 += (ip0 == prefixStart);
@ -253,15 +270,15 @@ _start: /* Requires: ip0 */
offcode = REPCODE1_TO_OFFBASE; offcode = REPCODE1_TO_OFFBASE;
mLength += 4; mLength += 4;
/* First write next hash table entry; we've already calculated it. /* Write next hash table entry: it's already calculated.
* This write is known to be safe because the ip1 is before the * This write is known to be safe because ip1 is before the
* repcode (ip2). */ * repcode (ip2). */
hashTable[hash1] = (U32)(ip1 - base); hashTable[hash1] = (U32)(ip1 - base);
goto _match; goto _match;
} }
if (ZSTD_findMatch_cmov(ip0, base + idx, idx, prefixStartIndex, dummy)) { if (findMatch(ip0, base + idx, idx, prefixStartIndex, dummy)) {
/* found a match! */ /* found a match! */
/* Write next hash table entry (it's already calculated). /* Write next hash table entry (it's already calculated).
@ -288,19 +305,13 @@ _start: /* Requires: ip0 */
current0 = (U32)(ip0 - base); current0 = (U32)(ip0 - base);
hashTable[hash0] = current0; hashTable[hash0] = current0;
if (ZSTD_findMatch_cmov(ip0, base + idx, idx, prefixStartIndex, dummy)) { if (findMatch(ip0, base + idx, idx, prefixStartIndex, dummy)) {
/* found a match! */ /* found a match! */
/* first write next hash table entry; it's already calculated */ /* Write next hash table entry; it's already calculated */
if (step <= 4) { if (step <= 4) {
/* We need to avoid writing an index into the hash table >= the /* Avoid writing an index if it's >= position where search will resume.
* position at which we will pick up our searching after we've * The minimum possible match has length 4, so search can resume at ip0 + 4.
* taken this match.
*
* The minimum possible match has length 4, so the earliest ip0
* can be after we take this match will be the current ip0 + 4.
* ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely
* write this position.
*/ */
hashTable[hash1] = (U32)(ip1 - base); hashTable[hash1] = (U32)(ip1 - base);
} }
@ -331,7 +342,7 @@ _start: /* Requires: ip0 */
} while (ip3 < ilimit); } while (ip3 < ilimit);
_cleanup: _cleanup:
/* Note that there are probably still a couple positions we could search. /* Note that there are probably still a couple positions one could search.
* However, it seems to be a meaningful performance hit to try to search * However, it seems to be a meaningful performance hit to try to search
* them. So let's not. */ * them. So let's not. */
@ -405,12 +416,12 @@ _match: /* Requires: ip0, match0, offcode */
goto _start; goto _start;
} }
#define ZSTD_GEN_FAST_FN(dictMode, mls, cmov) \ #define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \
static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##cmov( \ static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \ void const* src, size_t srcSize) \
{ \ { \
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, cmov); \ return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \
} }
ZSTD_GEN_FAST_FN(noDict, 4, 1) ZSTD_GEN_FAST_FN(noDict, 4, 1)
@ -427,12 +438,12 @@ size_t ZSTD_compressBlock_fast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize) void const* src, size_t srcSize)
{ {
U32 const mls = ms->cParams.minMatch; U32 const mml = ms->cParams.minMatch;
/* use cmov instead of branch when the branch is likely unpredictable */ /* use cmov when "candidate in range" branch is likely unpredictable */
int const useCmov = 1; int const useCmov = ms->cParams.windowLog < 19;
assert(ms->dictMatchState == NULL); assert(ms->dictMatchState == NULL);
if (useCmov) { if (useCmov) {
switch(mls) switch(mml)
{ {
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4 :
@ -446,7 +457,7 @@ size_t ZSTD_compressBlock_fast(
} }
} else { } else {
/* use a branch instead */ /* use a branch instead */
switch(mls) switch(mml)
{ {
default: /* includes case 3 */ default: /* includes case 3 */
case 4 : case 4 :
@ -458,7 +469,6 @@ size_t ZSTD_compressBlock_fast(
case 7 : case 7 :
return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
} }
} }
} }