sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-05 00:07:15 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #65 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet 2015-11-05 02:42:59 +01:00
commit 9876e1e1f5
13 changed files with 670 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -32,7 +32,7 @@
# ################################################################
# Version number
export VERSION := 0.3.2
export VERSION := 0.3.3
PRGDIR = programs
ZSTDDIR = lib

3
NEWS
View File

@ -1,3 +1,6 @@
v0.3.3
Small compression ratio improvement
v0.3.2
Fixed Visual Studio

BIN
images/CSpeed.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

8
lib/zstd.c
View File

@ -120,8 +120,6 @@
#define GB *(1U<<30)
#define BLOCKSIZE (128 KB) /* define, for static allocation */
#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
#define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
#define IS_RAW BIT0
#define IS_RLE BIT1
@ -570,7 +568,7 @@ static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, c
ZSTD_resetSeqStore(seqStorePtr);
/* Main Search Loop */
while (ip <= ilimit)
while (ip < ilimit) /* < instead of <=, because unconditionnal ZSTD_addPtr(ip+1) */
{
const BYTE* match = ZSTD_updateMatch(HashTable, ip, base);
@ -591,7 +589,8 @@ static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, c
ZSTD_addPtr(HashTable, ip+1, base);
ip += matchLength + MINMATCH;
anchor = ip;
if (ip <= ilimit) ZSTD_addPtr(HashTable, ip-2, base);
if (ip < ilimit) /* same test as loop, for speed */
ZSTD_addPtr(HashTable, ip-2, base);
}
}
@ -920,6 +919,7 @@ size_t ZSTD_decodeLiteralsBlock(void* ctx,
{
if (litSize > srcSize-3) return ERROR(corruption_detected);
memcpy(dctx->litBuffer, istart, litSize);
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = BLOCKSIZE+8;
dctx->litSize = litSize;
return litSize+3;

2
lib/zstd.h
View File

@ -48,7 +48,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void);

10
lib/zstd_internal.h
View File

@ -154,7 +154,7 @@ MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pI
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
@ -171,7 +171,9 @@ static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do COPY8(op, ip) while (op < oend);
do
COPY8(op, ip)
while (op < oend);
}
@ -212,6 +214,10 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr);
#define MaxLL ((1<<LLbits) - 1)
#define MaxOff 31
#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
#define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
/** ZSTD_storeSeq
Store a sequence (literal length, literals, offset code and match length) into seqStore_t
@offsetCode : distance to match, or 0 == repCode

702
lib/zstdhc.c
View File

@ -107,20 +107,42 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
return 0;
}
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
{
const U32 chainplus = (params->strategy == ZSTD_HC_btlazy2);
/* validate params */
if (params->windowLog > ZSTD_HC_WINDOWLOG_MAX) params->windowLog = ZSTD_HC_WINDOWLOG_MAX;
if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
/* correct params, to use less memory */
if (srcSize > 0)
{
U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
}
if (params->chainLog > params->windowLog + chainplus) params->chainLog = params->windowLog+chainplus; /* <= ZSTD_HC_CHAINLOG_MAX */
if (params->chainLog < ZSTD_HC_CHAINLOG_MIN) params->chainLog = ZSTD_HC_CHAINLOG_MIN;
if (params->hashLog > ZSTD_HC_HASHLOG_MAX) params->hashLog = ZSTD_HC_HASHLOG_MAX;
if (params->hashLog < ZSTD_HC_HASHLOG_MIN) params->hashLog = ZSTD_HC_HASHLOG_MIN;
if (params->searchLog > ZSTD_HC_SEARCHLOG_MAX) params->searchLog = ZSTD_HC_SEARCHLOG_MAX;
if (params->searchLog < ZSTD_HC_SEARCHLOG_MIN) params->searchLog = ZSTD_HC_SEARCHLOG_MIN;
if (params->searchLength> ZSTD_HC_SEARCHLENGTH_MAX) params->searchLength = ZSTD_HC_SEARCHLENGTH_MAX;
if (params->searchLength< ZSTD_HC_SEARCHLENGTH_MIN) params->searchLength = ZSTD_HC_SEARCHLENGTH_MIN;
if ((U32)params->strategy>(U32)ZSTD_HC_btlazy2) params->strategy = ZSTD_HC_btlazy2;
if ((int)params->strategy<(int)ZSTD_HC_greedy) params->strategy = ZSTD_HC_greedy;
}
static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
ZSTD_HC_parameters params)
{
/* validate params */
if (params.windowLog > ZSTD_HC_WINDOWLOG_MAX) params.windowLog = ZSTD_HC_WINDOWLOG_MAX;
if (params.windowLog < ZSTD_HC_WINDOWLOG_MIN) params.windowLog = ZSTD_HC_WINDOWLOG_MIN;
if (params.chainLog > params.windowLog) params.chainLog = params.windowLog; /* <= ZSTD_HC_CHAINLOG_MAX */
if (params.chainLog < ZSTD_HC_CHAINLOG_MIN) params.chainLog = ZSTD_HC_CHAINLOG_MIN;
if (params.hashLog > ZSTD_HC_HASHLOG_MAX) params.hashLog = ZSTD_HC_HASHLOG_MAX;
if (params.hashLog < ZSTD_HC_HASHLOG_MIN) params.hashLog = ZSTD_HC_HASHLOG_MIN;
if (params.searchLog > ZSTD_HC_SEARCHLOG_MAX) params.searchLog = ZSTD_HC_SEARCHLOG_MAX;
if (params.searchLog < ZSTD_HC_SEARCHLOG_MIN) params.searchLog = ZSTD_HC_SEARCHLOG_MIN;
if (params.searchLength> ZSTD_HC_SEARCHLENGTH_MAX) params.searchLength = ZSTD_HC_SEARCHLENGTH_MAX;
if (params.searchLength< ZSTD_HC_SEARCHLENGTH_MIN) params.searchLength = ZSTD_HC_SEARCHLENGTH_MIN;
ZSTD_HC_validateParams(&params, 0);
/* reserve table memory */
{
@ -173,123 +195,197 @@ static const U64 prime6bytes = 227718039650203ULL;
static size_t ZSTD_HC_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
static size_t ZSTD_HC_hash6Ptr(const void* p, U32 h) { return ZSTD_HC_hash6(MEM_read64(p), h); }
static size_t ZSTD_HC_hashPtr(const void* p, U32 h, U32 mls)
static size_t ZSTD_HC_hashPtr(const void* p, U32 hBits, U32 mls)
{
switch(mls)
{
default:
case 4: return ZSTD_HC_hash4Ptr(p,h);
case 5: return ZSTD_HC_hash5Ptr(p,h);
case 6: return ZSTD_HC_hash6Ptr(p,h);
case 4: return ZSTD_HC_hash4Ptr(p, hBits);
case 5: return ZSTD_HC_hash5Ptr(p, hBits);
case 6: return ZSTD_HC_hash6Ptr(p, hBits);
}
}
#define NEXT_IN_CHAIN(d) chainTable[(d) & chainMask] /* flexible, CHAINSIZE dependent */
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask]
/* *************************************
* HC Compression
* Binary Tree search
***************************************/
/* Update chains up to ip (excluded) */
static U32 ZSTD_HC_insertAndFindFirstIndex (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls)
/** ZSTD_HC_insertBt1 : add one ptr to tree
@ip : assumed <= iend-8 */
static void ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
{
U32* const hashTable = zc->hashTable;
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
U32* const chainTable = zc->chainTable;
const U32 chainMask = (1 << zc->params.chainLog) - 1;
const size_t h = ZSTD_HC_hashPtr(ip, hashLog, mls);
U32* const bt = zc->chainTable;
const U32 btLog = zc->params.chainLog - 1;
const U32 btMask= (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
U32 dummy32; /* to be nullified at the end */
const U32 windowSize = 1 << zc->params.windowLog;
const U32 windowLow = windowSize >= current ? 0 : current - windowSize;
while(idx < target)
hashTable[h] = (U32)(ip-base); /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow))
{
size_t h = ZSTD_HC_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx) = hashTable[h];
hashTable[h] = idx;
idx++;
U32* nextPtr = bt + 2*(matchIndex & btMask);
const BYTE* match = base + matchIndex;
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* just drop , to guarantee consistency (miss a bit of compression; if someone knows better, please tell) */
if (match[matchLength] < ip[matchLength])
{
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
}
else
{
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
}
}
zc->nextToUpdate = target;
return hashTable[ZSTD_HC_hashPtr(ip, hashLog, mls)];
*smallerPtr = *largerPtr = 0;
}
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_HC_insertAndFindBestMatch (
ZSTD_HC_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
const U32 maxNbAttempts, const U32 matchLengthSearch)
size_t ZSTD_HC_insertBtAndFindBestMatch (
ZSTD_HC_CCtx* zc,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
U32 nbCompares, const U32 mls)
{
U32* const chainTable = zc->chainTable;
const U32 chainSize = (1 << zc->params.chainLog);
const U32 chainMask = chainSize-1;
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
const size_t h = ZSTD_HC_hashPtr(ip, hashLog, mls);
U32* const bt = zc->chainTable;
const U32 btLog = zc->params.chainLog - 1;
const U32 btMask= (1 << btLog) - 1;
U32 matchIndex = hashTable[h];
size_t commonLengthSmaller=0, commonLengthLarger=0;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const U32 maxDistance = (1 << zc->params.windowLog);
const U32 lowLimit = (zc->lowLimit + maxDistance > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (maxDistance - 1);
U32 matchIndex;
const BYTE* match;
int nbAttempts=maxNbAttempts;
size_t ml=0;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
const U32 windowSize = 1 << zc->params.windowLog;
const U32 windowLow = windowSize >= current ? 0 : current - windowSize;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
size_t bestLength = 0;
U32 dummy32; /* to be nullified at the end */
/* HC4 match finder */
matchIndex = ZSTD_HC_insertAndFindFirstIndex (zc, ip, matchLengthSearch);
hashTable[h] = (U32)(ip-base); /* Update Hash Table */
while ((matchIndex>=lowLimit) && (nbAttempts))
while (nbCompares-- && (matchIndex > windowLow))
{
nbAttempts--;
if (matchIndex >= dictLimit)
U32* nextPtr = bt + 2*(matchIndex & btMask);
const BYTE* match = base + matchIndex;
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
if (matchLength > bestLength)
{
match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (MEM_read32(match) == MEM_read32(ip)))
{
const size_t mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; if (ip+ml >= iLimit) break; }
}
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
bestLength = matchLength, *offsetPtr = current - matchIndex;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, next to null, to guarantee consistency (is there a way to do better ?) */
}
if (match[matchLength] < ip[matchLength])
{
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
}
else
{
match = dictBase + matchIndex;
if (MEM_read32(match) == MEM_read32(ip))
{
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iLimit))
mlt += ZSTD_count(ip+mlt, base+dictLimit, iLimit);
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
}
if (base + matchIndex <= ip - chainSize) break;
matchIndex = NEXT_IN_CHAIN(matchIndex);
}
return ml;
*smallerPtr = *largerPtr = 0;
zc->nextToUpdate = current+1; /* current has been inserted */
if (bestLength < MINMATCH) return 0;
return bestLength;
}
FORCE_INLINE size_t ZSTD_HC_insertAndFindBestMatch_selectMLS (
static void ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
{
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
//size_t dummy;
for( ; idx < target ; idx++)
ZSTD_HC_insertBt1(zc, base+idx, mls, iend, nbCompares);
//ZSTD_HC_insertBtAndFindBestMatch(zc, base+idx, iend, &dummy, nbCompares, mls);
zc->nextToUpdate = target;
}
/** Tree updater, providing best match */
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_HC_BtFindBestMatch (
ZSTD_HC_CCtx* zc,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 mls)
{
ZSTD_HC_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
return ZSTD_HC_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls);
}
FORCE_INLINE size_t ZSTD_HC_BtFindBestMatch_selectMLS (
ZSTD_HC_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{
default :
case 4 : return ZSTD_HC_insertAndFindBestMatch(zc, ip, iLimit, matchpos, maxNbAttempts, 4);
case 5 : return ZSTD_HC_insertAndFindBestMatch(zc, ip, iLimit, matchpos, maxNbAttempts, 5);
case 6 : return ZSTD_HC_insertAndFindBestMatch(zc, ip, iLimit, matchpos, maxNbAttempts, 6);
case 4 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 6 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
}
}
size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
size_t ZSTD_HC_compressBlock_btLazy2(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
@ -297,7 +393,6 @@ size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSiz
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* match = istart;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog;
@ -311,7 +406,7 @@ size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSiz
while (ip <= ilimit)
{
size_t matchLength;
size_t offset;
size_t offset=999999;
const BYTE* start;
/* try to find a first match */
@ -330,11 +425,10 @@ size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSiz
}
offset_2 = offset_1;
matchLength = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &match, maxSearches, mls);
matchLength = ZSTD_HC_BtFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls);
if (!matchLength) { ip++; continue; }
/* let's try to find a better solution */
offset = ip - match;
start = ip;
while (ip<ilimit)
@ -343,19 +437,368 @@ size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSiz
if (MEM_read32(ip) == MEM_read32(ip - offset_1))
{
size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(ml2 * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset));
int gain2 = (int)(ml2 * 3);
int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
if (gain2 > gain1)
matchLength = ml2, offset = 0, start = ip;
}
{
size_t offset2=999999;
size_t ml2 = ZSTD_HC_BtFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
if (gain2 > gain1)
{
matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */
}
}
/* let's find an even better one */
if (ip<ilimit)
{
ip ++;
if (MEM_read32(ip) == MEM_read32(ip - offset_1))
{
size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(ml2 * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
if (gain2 > gain1)
matchLength = ml2, offset = 0, start = ip;
}
{
size_t offset2=999999;
size_t ml2 = ZSTD_HC_BtFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
if (gain2 > gain1)
{
matchLength = ml2, offset = offset2, start = ip;
continue;
}
}
}
break; /* nothing found : store previous solution */
}
/* store sequence */
{
size_t litLength = start - anchor;
if (offset) offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
ip = start + matchLength;
anchor = ip;
}
}
/* Last Literals */
{
size_t lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
}
/* Final compression stage */
return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
seqStorePtr, srcSize);
}
/* ***********************
* Hash Chain
*************************/
/* Update chains up to ip (excluded) */
static U32 ZSTD_HC_insertAndFindFirstIndex (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls)
{
U32* const hashTable = zc->hashTable;
const U32 hashLog = zc->params.hashLog;
U32* const chainTable = zc->chainTable;
const U32 chainMask = (1 << zc->params.chainLog) - 1;
const BYTE* const base = zc->base;
const U32 target = (U32)(ip - base);
U32 idx = zc->nextToUpdate;
while(idx < target)
{
size_t h = ZSTD_HC_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
idx++;
}
zc->nextToUpdate = target;
return hashTable[ZSTD_HC_hashPtr(ip, hashLog, mls)];
}
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
size_t ZSTD_HC_insertAndFindBestMatch (
ZSTD_HC_CCtx* zc, /* Index table will be updated */
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
U32* const chainTable = zc->chainTable;
const U32 chainSize = (1 << zc->params.chainLog);
const U32 chainMask = chainSize-1;
const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit;
const U32 maxDistance = (1 << zc->params.windowLog);
const U32 lowLimit = (zc->lowLimit + maxDistance > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (maxDistance - 1);
U32 matchIndex;
const BYTE* match;
int nbAttempts=maxNbAttempts;
size_t ml=0;
/* HC4 match finder */
matchIndex = ZSTD_HC_insertAndFindFirstIndex (zc, ip, matchLengthSearch);
while ((matchIndex>lowLimit) && (nbAttempts))
{
nbAttempts--;
if (matchIndex >= dictLimit)
{
match = base + matchIndex;
if ( (match[ml] == ip[ml])
&& (MEM_read32(match) == MEM_read32(ip)) ) /* ensures minimum match of 4 */
{
const size_t mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml)
//if (((int)(4*mlt) - (int)ZSTD_highbit((U32)(ip-match)+1)) > ((int)(4*ml) - (int)ZSTD_highbit((U32)((*offsetPtr)+1))))
{
ml = mlt; *offsetPtr = ip-match;
if (ip+ml >= iLimit) break;
}
}
}
else
{
match = dictBase + matchIndex;
if (MEM_read32(match) == MEM_read32(ip))
{
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
mlt = ZSTD_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iLimit))
mlt += ZSTD_count(ip+mlt, base+dictLimit, iLimit);
if (mlt > ml) { ml = mlt; *offsetPtr = (ip-base) - matchIndex; }
}
}
if (base + matchIndex <= ip - chainSize) break;
matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
}
return ml;
}
FORCE_INLINE size_t ZSTD_HC_insertAndFindBestMatch_selectMLS (
ZSTD_HC_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 maxNbAttempts, const U32 matchLengthSearch)
{
switch(matchLengthSearch)
{
default :
case 4 : return ZSTD_HC_insertAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
case 5 : return ZSTD_HC_insertAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
case 6 : return ZSTD_HC_insertAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
}
}
size_t ZSTD_HC_compressBlock_lazydeep(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog;
const U32 mls = ctx->params.searchLength;
/* init */
ZSTD_resetSeqStore(seqStorePtr);
if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
/* Match Loop */
while (ip <= ilimit)
{
size_t matchLength;
size_t offset=999999;
const BYTE* start;
/* try to find a first match */
if (MEM_read32(ip) == MEM_read32(ip - offset_2))
{
/* repcode : we take it*/
size_t offtmp = offset_2;
size_t litLength = ip - anchor;
matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend);
offset_2 = offset_1;
offset_1 = offtmp;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, 0, matchLength);
ip += matchLength+MINMATCH;
anchor = ip;
continue;
}
offset_2 = offset_1;
matchLength = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls);
if (!matchLength) { ip++; continue; }
/* let's try to find a better solution */
start = ip;
while (ip<ilimit)
{
ip ++;
if (MEM_read32(ip) == MEM_read32(ip - offset_1))
{
size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(ml2 * 3);
int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
if (gain2 > gain1)
matchLength = ml2, offset = 0, start = ip;
}
{
size_t offset2=999999;
size_t ml2 = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
if (gain2 > gain1)
{
matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */
}
}
/* let's find an even better one */
if (ip<ilimit)
{
ip ++;
if (MEM_read32(ip) == MEM_read32(ip - offset_1))
{
size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(ml2 * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
if (gain2 > gain1)
matchLength = ml2, offset = 0, start = ip;
}
{
size_t offset2=999999;
size_t ml2 = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
if (gain2 > gain1)
{
matchLength = ml2, offset = offset2, start = ip;
continue;
}
}
}
break; /* nothing found : store previous solution */
}
/* store sequence */
{
size_t litLength = start - anchor;
if (offset) offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH);
ip = start + matchLength;
anchor = ip;
}
}
/* Last Literals */
{
size_t lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
}
/* Final compression stage */
return ZSTD_compressSequences((BYTE*)dst, maxDstSize,
seqStorePtr, srcSize);
}
size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog;
const U32 mls = ctx->params.searchLength;
/* init */
ZSTD_resetSeqStore(seqStorePtr);
if (((ip-ctx->base) - ctx->dictLimit) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
/* Match Loop */
while (ip <= ilimit)
{
size_t matchLength;
size_t offset=0;
const BYTE* start;
/* try to find a first match */
if (MEM_read32(ip) == MEM_read32(ip - offset_2))
{
/* repcode : we take it*/
size_t offtmp = offset_2;
size_t litLength = ip - anchor;
matchLength = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend);
offset_2 = offset_1;
offset_1 = offtmp;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, 0, matchLength);
ip += matchLength+MINMATCH;
anchor = ip;
continue;
}
offset_2 = offset_1;
matchLength = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls);
if (!matchLength) { ip++; continue; }
/* let's try to find a better solution */
start = ip;
while (ip<ilimit)
{
ip ++;
if (MEM_read32(ip) == MEM_read32(ip - offset_1))
{
size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_1, iend) + MINMATCH;
int gain2 = (int)(ml2 * 3);
int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
if (gain2 > gain1)
{
matchLength = ml2, offset = 0, start = ip;
break;
}
}
{
size_t ml2 = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &match, maxSearches, mls);
size_t offset2 = ip - match;
int gain2 = (int)(ml2*5 - ZSTD_highbit((U32)offset2)); /* raw approx */
int gain1 = (int)(matchLength*5 - ZSTD_highbit((U32)offset));
size_t offset2=999999;
size_t ml2 = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &offset2, maxSearches, mls);
int gain2 = (int)(ml2*3 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 3);
if (gain2 > gain1)
{
matchLength = ml2, offset = offset2, start = ip;
@ -398,7 +841,6 @@ size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* match = istart;
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
const U32 maxSearches = 1 << ctx->params.searchLog;
@ -441,12 +883,13 @@ size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
/* search */
{
size_t matchLength = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &match, maxSearches, mls);
size_t offset=999999;
size_t matchLength = ZSTD_HC_insertAndFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls);
if (!matchLength) { ip++; continue; }
/* store sequence */
{
size_t litLength = ip-anchor;
offset_1 = ip-match;
offset_1 = offset;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset_1, matchLength-MINMATCH);
ip += matchLength;
anchor = ip;
@ -468,11 +911,30 @@ size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
}
typedef size_t (*ZSTD_HC_blockCompressor) (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
static ZSTD_HC_blockCompressor ZSTD_HC_selectBlockCompressor(ZSTD_HC_strategy strat)
{
switch(strat)
{
default :
case ZSTD_HC_greedy:
return ZSTD_HC_compressBlock_greedy;
case ZSTD_HC_lazy:
return ZSTD_HC_compressBlock_lazy;
case ZSTD_HC_lazydeep:
return ZSTD_HC_compressBlock_lazydeep;
case ZSTD_HC_btlazy2:
return ZSTD_HC_compressBlock_btLazy2;
}
}
size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
if (ctx->params.strategy == ZSTD_HC_greedy)
return ZSTD_HC_compressBlock_greedy(ctx, dst, maxDstSize, src, srcSize);
return ZSTD_HC_compressBlock_lazy(ctx, dst, maxDstSize, src, srcSize);
ZSTD_HC_blockCompressor blockCompressor = ZSTD_HC_selectBlockCompressor(ctx->params.strategy);
return blockCompressor(ctx, dst, maxDstSize, src, srcSize);
}
@ -480,23 +942,22 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize)
{
static const size_t blockSize = 128 KB;
size_t blockSize = BLOCKSIZE;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
BYTE* const oend = op + maxDstSize;
size_t (*blockCompressor) (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
const ZSTD_HC_blockCompressor blockCompressor = ZSTD_HC_selectBlockCompressor(ctxPtr->params.strategy);
if (ctxPtr->params.strategy == ZSTD_HC_greedy)
blockCompressor = ZSTD_HC_compressBlock_greedy;
else
blockCompressor = ZSTD_HC_compressBlock_lazy;
while (remaining > blockSize)
while (remaining)
{
size_t cSize = blockCompressor(ctxPtr, op+3, oend-op, ip, blockSize);
size_t cSize;
if (maxDstSize < 3 + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
cSize = blockCompressor(ctxPtr, op+3, maxDstSize-3, ip, blockSize);
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0)
{
@ -512,30 +973,9 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
}
remaining -= blockSize;
maxDstSize -= cSize;
ip += blockSize;
op += cSize;
if (ZSTD_isError(cSize)) return cSize;
}
/* last block */
{
size_t cSize = blockCompressor(ctxPtr, op+3, oend-op, ip, remaining);
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, remaining); /* block is not compressible */
}
else
{
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
op[2] = (BYTE)cSize;
op[0] += (BYTE)(bt_compressed << 6); /* is a compressed block */
cSize += 3;
}
op += cSize;
if (ZSTD_isError(cSize)) return cSize;
}
return op-ostart;
@ -620,7 +1060,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
/* body (compression) */
ctx->base = (const BYTE*)src;
op += ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize);
oSize = ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;

67
lib/zstdhc_static.h
View File

@ -45,7 +45,7 @@ extern "C" {
/* *************************************
* Types
***************************************/
typedef enum { ZSTD_HC_greedy, ZSTD_HC_lazy } ZSTD_HC_strategy;
typedef enum { ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazydeep, ZSTD_HC_btlazy2 } ZSTD_HC_strategy;
typedef struct
{
U32 windowLog; /* largest match distance : impact decompression buffer size */
@ -53,13 +53,13 @@ typedef struct
U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 searchLog; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */
ZSTD_HC_strategy strategy; /* greedy, lazy (stronger, slower) */
ZSTD_HC_strategy strategy; /* greedy, lazy, lazydeep */
} ZSTD_HC_parameters;
/* parameters boundaries */
#define ZSTD_HC_WINDOWLOG_MAX 26
#define ZSTD_HC_WINDOWLOG_MIN 18
#define ZSTD_HC_CHAINLOG_MAX ZSTD_HC_WINDOWLOG_MAX
#define ZSTD_HC_CHAINLOG_MAX (ZSTD_HC_WINDOWLOG_MAX+1)
#define ZSTD_HC_CHAINLOG_MIN 4
#define ZSTD_HC_HASHLOG_MAX 28
#define ZSTD_HC_HASHLOG_MIN 4
@ -73,12 +73,17 @@ typedef struct
* Advanced function
***************************************/
/** ZSTD_HC_compress_advanced
* Same as ZSTD_HC_compressCCtx(), but can fine-tune each compression parameter */
* Same as ZSTD_HC_compressCCtx(), with fine-tune control of each compression parameter */
size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
void* dst, size_t maxDstSize,
const void* src, size_t srcSize,
ZSTD_HC_parameters params);
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize);
/* *************************************
* Streaming functions
@ -91,38 +96,36 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
/* *************************************
* Pre-defined compression levels
***************************************/
#define ZSTD_HC_MAX_CLEVEL 26
#define ZSTD_HC_MAX_CLEVEL 22
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {
/* W, C, H, S, L, strat */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 0 - never used */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 1 - in fact redirected towards zstd fast */
{ 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */
{ 19, 13, 17, 3, 5, ZSTD_HC_greedy }, /* level 3 */
{ 20, 18, 19, 2, 5, ZSTD_HC_greedy }, /* level 4 */
{ 20, 19, 19, 3, 5, ZSTD_HC_greedy }, /* level 5 */
{ 20, 18, 20, 3, 5, ZSTD_HC_lazy }, /* level 6 */
{ 20, 18, 20, 4, 5, ZSTD_HC_lazy }, /* level 7 */
{ 21, 19, 20, 4, 5, ZSTD_HC_lazy }, /* level 8 */
{ 21, 19, 20, 5, 5, ZSTD_HC_lazy }, /* level 9 */
{ 21, 20, 20, 5, 5, ZSTD_HC_lazy }, /* level 10 */
{ 21, 21, 20, 5, 5, ZSTD_HC_lazy }, /* level 11 */
{ 22, 20, 22, 6, 5, ZSTD_HC_lazy }, /* level 12 */
{ 22, 21, 22, 6, 5, ZSTD_HC_lazy }, /* level 13 */
{ 23, 21, 22, 6, 5, ZSTD_HC_lazy }, /* level 14 */
{ 23, 21, 23, 7, 5, ZSTD_HC_lazy }, /* level 15 */
{ 23, 22, 22, 6, 5, ZSTD_HC_lazy }, /* level 16 */
{ 23, 22, 22, 7, 5, ZSTD_HC_lazy }, /* level 17 */
{ 23, 23, 22, 7, 5, ZSTD_HC_lazy }, /* level 18 */
{ 23, 22, 23, 8, 5, ZSTD_HC_lazy }, /* level 19 */
{ 23, 23, 23, 8, 5, ZSTD_HC_lazy }, /* level 20 */
{ 23, 23, 23, 8, 5, ZSTD_HC_lazy }, /* level 21 */
{ 24, 24, 24, 8, 5, ZSTD_HC_lazy }, /* level 22 */
{ 24, 23, 23, 9, 5, ZSTD_HC_lazy }, /* level 23 */
{ 24, 24, 24, 9, 5, ZSTD_HC_lazy }, /* level 24 */
{ 24, 24, 24, 9, 5, ZSTD_HC_lazy }, /* level 25 */
{ 24, 24, 24, 10, 5, ZSTD_HC_lazy }, /* level 26 */ /* ZSTD_HC_MAX_CLEVEL */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 0 - never used */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 1 - in fact redirected towards zstd fast */
{ 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */
{ 19, 14, 18, 2, 5, ZSTD_HC_greedy }, /* level 3 */
{ 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 4 */
{ 20, 18, 19, 2, 5, ZSTD_HC_lazy }, /* level 5 */
{ 21, 18, 20, 3, 5, ZSTD_HC_lazy }, /* level 6 */
{ 21, 20, 20, 3, 5, ZSTD_HC_lazy }, /* level 7 */
{ 21, 19, 20, 4, 5, ZSTD_HC_lazy }, /* level 8 */
{ 21, 19, 20, 5, 5, ZSTD_HC_lazy }, /* level 9 */
{ 21, 20, 20, 5, 5, ZSTD_HC_lazy }, /* level 10 */
{ 21, 20, 20, 5, 5, ZSTD_HC_lazydeep }, /* level 11 */
{ 22, 20, 22, 5, 5, ZSTD_HC_lazydeep }, /* level 12 */
{ 22, 20, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 13 */
{ 22, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 14 */
{ 22, 21, 22, 6, 5, ZSTD_HC_lazydeep }, /* level 15 */
{ 22, 21, 22, 5, 5, ZSTD_HC_btlazy2 }, /* level 16 */
{ 22, 22, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 17 */
{ 23, 23, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 18 */
{ 25, 25, 22, 5, 5, ZSTD_HC_btlazy2 }, /* level 19 */
{ 25, 25, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 20 */
{ 25, 26, 23, 9, 5, ZSTD_HC_btlazy2 }, /* level 21 */
{ 25, 26, 23, 9, 5, ZSTD_HC_btlazy2 }, /* level 22 */
};
#if defined (__cplusplus)
}
#endif

2
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= 0.3.2
VERSION?= 0.3.3
DESTDIR?=
PREFIX ?= /usr/local

6
programs/bench.c
View File

@ -83,7 +83,7 @@
#define MB *(1 <<20)
#define GB *(1U<<30)
#define MAX_MEM (2 GB - 64 MB)
static const size_t maxMemory = sizeof(size_t)==4 ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
#define DEFAULT_CHUNKSIZE (4 MB)
static U32 g_compressibilityDefault = 50;
@ -241,7 +241,7 @@ static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
{
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t));
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
@ -401,7 +401,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
requiredMem = (((requiredMem >> 26) + 1) << 26);
requiredMem += 2 * step;
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
if (requiredMem > maxMemory) requiredMem = maxMemory;
while (!testmem)
{

21
programs/fuzzer.c
View File

@ -328,14 +328,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
/* HC compression test */
cLevel = (FUZ_rand(&lseed) & 3) + 2;
cSize = ZSTD_HC_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
/* compression test */
/* covered by HC cLevel 1
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize);
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed");
CHECK(ZSTD_isError(cSize), "ZSTD_compress failed"); */
/* HC compression test */
cLevel = (FUZ_rand(&lseed) & 3) +1;
cSize = ZSTD_HC_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_HC_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3)
@ -346,16 +347,16 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
static const U32 endMark = 0x4DC2B1A9;
U32 endCheck;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compress should have failed ! (buffer too small)");
errorCode = ZSTD_HC_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_HC_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compress : dst buffer overflow");
CHECK(endCheck != endMark, "ZSTD_HC_compressCCtx : dst buffer overflow");
}
/* successfull decompression tests*/
dSupSize = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s)", ZSTD_getErrorName(dSize));
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);

47
programs/paramgrill.c
View File

@ -348,7 +348,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
if (totalTime > g_maxParamTime) break;
/* Compression */
DISPLAY("%1u-%s : %9u ->\r", loopNb, name, (U32)srcSize);
DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0;
@ -371,8 +371,9 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)srcSize / (double)cSize;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s\r", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
resultPtr->cSize = cSize;
resultPtr->cSpeed = (U32)((double)srcSize / fastestC);
@ -393,9 +394,10 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAY("\r");
DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.);
DISPLAY("%7.1f MB/s\r", (double)srcSize / fastestD / 1000.);
DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000.);
resultPtr->dSpeed = (U32)((double)srcSize / fastestD);
/* CRC Checking */
@ -420,21 +422,23 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
}
/* End cleaning */
DISPLAY("\r");
free(compressedBuffer);
free(resultBuffer);
return 0;
}
const char* g_stratName[2] = { "ZSTD_HC_greedy", "ZSTD_HC_lazy " };
const char* g_stratName[] = { "ZSTD_HC_greedy ", "ZSTD_HC_lazy ", "ZSTD_HC_lazydeep", "ZSTD_HC_btlazy2 " };
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
{
DISPLAY("\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u, %s }, ",
fprintf(f," {%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
g_stratName[params.strategy]);
fprintf(f,
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
cLevel, (double)srcSize / result.cSize, (double)result.cSpeed / 1000., (double)result.dSpeed / 1000.);
}
@ -576,7 +580,7 @@ static BYTE g_alreadyTested[ZSTD_HC_WINDOWLOG_MAX+1-ZSTD_HC_WINDOWLOG_MIN]
[ZSTD_HC_HASHLOG_MAX+1-ZSTD_HC_HASHLOG_MIN]
[ZSTD_HC_SEARCHLOG_MAX+1-ZSTD_HC_SEARCHLOG_MIN]
[ZSTD_HC_SEARCHLENGTH_MAX+1-ZSTD_HC_SEARCHLENGTH_MIN]
[2 /* strategy */ ] = {}; /* init to zero */
[4 /* strategy */ ] = {}; /* init to zero */
#define NB_TESTS_PLAYED(p) \
g_alreadyTested[p.windowLog-ZSTD_HC_WINDOWLOG_MIN] \
@ -592,7 +596,6 @@ static void playAround(FILE* f, winnerInfo_t* winners,
const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx)
{
const U32 srcLog = BMK_highbit((U32)( (g_blockSize ? g_blockSize : srcSize) -1))+1;
int nbVariations = 0;
const int startTime = BMK_GetMilliStart();
@ -628,26 +631,18 @@ static void playAround(FILE* f, winnerInfo_t* winners,
case 9:
p.searchLength--; break;
case 10:
p.strategy = ZSTD_HC_lazy; break;
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)+1); break;
case 11:
p.strategy = ZSTD_HC_greedy; break;
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)-1); break;
}
}
/* validate new conf */
if (p.windowLog > srcLog) continue;
if (p.windowLog > ZSTD_HC_WINDOWLOG_MAX) continue;
if (p.windowLog < MAX(ZSTD_HC_WINDOWLOG_MIN, p.chainLog)) continue;
if (p.chainLog > p.windowLog) continue;
if (p.chainLog < ZSTD_HC_CHAINLOG_MIN) continue;
if (p.hashLog > ZSTD_HC_HASHLOG_MAX) continue;
if (p.hashLog < ZSTD_HC_HASHLOG_MIN) continue;
if (p.searchLog > p.chainLog) continue;
if (p.searchLog < ZSTD_HC_SEARCHLOG_MIN) continue;
if (p.searchLength > ZSTD_HC_SEARCHLENGTH_MAX) continue;
if (p.searchLength < ZSTD_HC_SEARCHLENGTH_MIN) continue;
if (p.strategy < ZSTD_HC_greedy) continue;
if (p.strategy > ZSTD_HC_lazy) continue;
{
ZSTD_HC_parameters saved = p;
ZSTD_HC_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
}
/* exclude faster if already played params */
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))
@ -680,7 +675,7 @@ static void BMK_selectRandomStart(
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN;
p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) & 1);
p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % 4);
playAround(f, winners, p, srcBuffer, srcSize, ctx);
}
else
@ -701,6 +696,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (g_singleRun)
{
BMK_result_t testResult;
ZSTD_HC_validateParams(&g_params, g_blockSize ? g_blockSize : srcSize);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n");
return;
@ -983,7 +979,8 @@ int main(int argc, char** argv)
argument++;
while ((*argument>= '0') && (*argument<='9'))
{
if (*argument++) g_params.strategy = ZSTD_HC_lazy;
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy *10);
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy + *argument++ - '0');
}
continue;
case 'L':

19
programs/zstdcli.c
View File

@ -70,7 +70,7 @@
**************************************/
#define COMPRESSOR_NAME "zstd command line interface"
#ifndef ZSTD_VERSION
# define ZSTD_VERSION "v0.3.0"
# define ZSTD_VERSION "v0.3.3"
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__
@ -178,7 +178,7 @@ int main(int argc, char** argv)
const char* inFileName = NULL;
const char* outFileName = NULL;
char* dynNameSpace = NULL;
char extension[] = ZSTD_EXTENSION;
const char extension[] = ZSTD_EXTENSION;
displayOut = stderr;
/* Pick out basename component. Don't rely on stdlib because of conflicting behavior. */
@ -351,16 +351,17 @@ int main(int argc, char** argv)
}
/* decompression to file (automatic name will work only if input filename has correct format extension) */
{
size_t outl;
size_t inl = strlen(inFileName);
dynNameSpace = (char*)calloc(1,inl+1);
size_t filenameSize = strlen(inFileName);
if (strcmp(inFileName + (filenameSize-4), extension))
{
DISPLAYLEVEL(1, "unknown suffix - cannot determine destination filename\n");
return badusage(programName);
}
dynNameSpace = (char*)calloc(1,filenameSize+1);
if (dynNameSpace==NULL) { DISPLAY("not enough memory\n"); exit(1); }
outFileName = dynNameSpace;
strcpy(dynNameSpace, inFileName);
outl = inl;
if (inl>4)
while ((outl >= inl-4) && (inFileName[outl] == extension[outl-inl+4])) dynNameSpace[outl--]=0;
if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); return badusage(programName); }
dynNameSpace[filenameSize-4]=0;
DISPLAYLEVEL(2, "Decoding file %s \n", outFileName);
}
}