sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-12 00:05:03 -05:00
Code Issues Projects Releases Wiki Activity

[ldm] Update to hash every position

Browse Source
This commit is contained in:
Stella Lau 2017-07-07 14:52:40 -07:00
parent 7945f9ee47
commit 4076be09ec
1 changed files with 34 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
contrib/long_distance_matching/ldm.c
View File

@ -11,7 +11,8 @@
#define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2) #define LDM_HASHTABLESIZE_U32 ((LDM_HASHTABLESIZE) >> 2)
#define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG)) #define LDM_HASH_SIZE_U32 (1 << (LDM_HASHLOG))
#define WINDOW_SIZE (1 << 15) #define WINDOW_SIZE (1 << 20)
#define MAX_WINDOW_SIZE 31
#define HASH_SIZE 4 #define HASH_SIZE 4
#define MINMATCH 4 #define MINMATCH 4
@ -73,6 +74,11 @@ static U32 LDM_read32(const void *ptr) {
return *(const U32 *)ptr; return *(const U32 *)ptr;
} }
static U64 LDM_read64(const void *ptr) {
return *(const U64 *)ptr;
}
static void LDM_copy8(void *dst, const void *src) { static void LDM_copy8(void *dst, const void *src) {
memcpy(dst, src, 8); memcpy(dst, src, 8);
} }
@ -87,7 +93,6 @@ static void LDM_wild_copy(void *dstPtr, const void *srcPtr, void *dstEnd) {
d += 8; d += 8;
s += 8; s += 8;
} while (d < e); } while (d < e);
} }
struct hash_entry { struct hash_entry {
@ -99,12 +104,23 @@ static U32 LDM_hash(U32 sequence) {
return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG)); return ((sequence * 2654435761U) >> ((32)-LDM_HASHLOG));
} }
static U32 LDM_hash5(U64 sequence) {
static const U64 prime5bytes = 889523592379ULL;
static const U64 prime8bytes = 11400714785074694791ULL;
const U32 hashLog = LDM_HASHLOG;
if (LDM_isLittleEndian())
return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
else
return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
}
static U32 LDM_hash_position(const void * const p) { static U32 LDM_hash_position(const void * const p) {
return LDM_hash(LDM_read32(p)); return LDM_hash(LDM_read32(p));
} }
static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase, static void LDM_put_position_on_hash(const BYTE *p, U32 h, void *tableBase,
const BYTE *srcBase) { const BYTE *srcBase) {
// printf("Hashing: %zu\n", p - srcBase);
U32 *hashTable = (U32 *) tableBase; U32 *hashTable = (U32 *) tableBase;
hashTable[h] = (U32)(p - srcBase); hashTable[h] = (U32)(p - srcBase);
} }
@ -170,6 +186,7 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
/* Hash first byte: put into hash table */ /* Hash first byte: put into hash table */
LDM_put_position(ip, hashTable, istart); LDM_put_position(ip, hashTable, istart);
const BYTE *lastHash = ip;
ip++; ip++;
forwardH = LDM_hash_position(ip); forwardH = LDM_hash_position(ip);
@ -196,8 +213,9 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
forwardH = LDM_hash_position(forwardIp); forwardH = LDM_hash_position(forwardIp);
LDM_put_position_on_hash(ip, h, hashTable, istart); LDM_put_position_on_hash(ip, h, hashTable, istart);
lastHash = ip;
} while (ip - match > WINDOW_SIZE || } while (ip - match > WINDOW_SIZE ||
LDM_read32(match) != LDM_read32(ip)); LDM_read64(match) != LDM_read64(ip));
} }
// TODO catchup // TODO catchup
@ -215,10 +233,6 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
printf("Cur position: %zu\n", anchor - istart); printf("Cur position: %zu\n", anchor - istart);
printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match); printf("LitLength %zu. (Match offset). %zu\n", litLength, ip - match);
#endif #endif
/*
fwrite(match, 4, 1, stdout);
printf("\n");
*/
if (litLength >= RUN_MASK) { if (litLength >= RUN_MASK) {
int len = (int)litLength - RUN_MASK; int len = (int)litLength - RUN_MASK;
@ -242,8 +256,8 @@ size_t LDM_compress(void const *source, void *dest, size_t source_size,
_next_match: _next_match:
/* Encode offset */ /* Encode offset */
{ {
LDM_writeLE16(op, (U16)(ip - match)); LDM_write32(op, ip - match);
op += 2; op += 4;
} }
/* Encode Match Length */ /* Encode Match Length */
@ -256,7 +270,13 @@ _next_match:
fwrite(ip, MINMATCH + matchCode, 1, stdout); fwrite(ip, MINMATCH + matchCode, 1, stdout);
printf("\n"); printf("\n");
#endif #endif
ip += MINMATCH + matchCode;
unsigned ctr = 1;
ip++;
for (; ctr < MINMATCH + matchCode; ip++, ctr++) {
LDM_put_position(ip, hashTable, istart);
}
// ip += MINMATCH + matchCode;
if (matchCode >= ML_MASK) { if (matchCode >= ML_MASK) {
*token += ML_MASK; *token += ML_MASK;
matchCode -= ML_MASK; matchCode -= ML_MASK;
@ -280,6 +300,7 @@ _next_match:
LDM_put_position(ip, hashTable, istart); LDM_put_position(ip, hashTable, istart);
forwardH = LDM_hash_position(++ip); forwardH = LDM_hash_position(++ip);
lastHash = ip;
} }
_last_literals: _last_literals:
/* Encode last literals */ /* Encode last literals */
@ -340,12 +361,12 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
op = cpy; op = cpy;
/* get offset */ /* get offset */
offset = LDM_readLE16(ip); offset = LDM_read32(ip);
#ifdef LDM_DEBUG #ifdef LDM_DEBUG
printf("Offset: %zu\n", offset); printf("Offset: %zu\n", offset);
#endif #endif
ip += 2; ip += 4;
match = op - offset; match = op - offset;
// LDM_write32(op, (U32)offset); // LDM_write32(op, (U32)offset);
@ -365,12 +386,11 @@ size_t LDM_decompress(void const *source, void *dest, size_t compressed_size,
/* copy match */ /* copy match */
cpy = op + length; cpy = op + length;
// printf("TMP_PREV: %zu\n", op - (BYTE *)dest);
// Inefficient for now // Inefficient for now
while (match < cpy - offset && op < oend) { while (match < cpy - offset && op < oend) {
*op++ = *match++; *op++ = *match++;
} }
// printf("TMP: %zu\n", op - (BYTE *)dest);
} }
// memcpy(dest, source, compressed_size); // memcpy(dest, source, compressed_size);
return op - (BYTE *)dest; return op - (BYTE *)dest;