mirror of
https://github.com/facebook/zstd.git
synced 2025-10-04 00:02:33 -04:00
refactor optimal parser
store stretches as intermediate solution instead of sequences. makes it possible to link a solution to a predecessor.
This commit is contained in:
parent
de10f56be2
commit
4683667785
@ -1661,8 +1661,8 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
|
||||
+ ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32))
|
||||
+ ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32))
|
||||
+ ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32))
|
||||
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
|
||||
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
|
||||
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+2) * sizeof(ZSTD_match_t))
|
||||
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+2) * sizeof(ZSTD_optimal_t));
|
||||
size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
|
||||
? ZSTD_cwksp_aligned_alloc_size(hSize)
|
||||
: 0;
|
||||
@ -2045,8 +2045,8 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
|
||||
ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned));
|
||||
ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned));
|
||||
ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned));
|
||||
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t));
|
||||
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
|
||||
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+2) * sizeof(ZSTD_match_t));
|
||||
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+2) * sizeof(ZSTD_optimal_t));
|
||||
}
|
||||
|
||||
ms->cParams = *cParams;
|
||||
|
@ -159,11 +159,11 @@ typedef struct {
|
||||
UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
|
||||
|
||||
typedef struct {
|
||||
int price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
int price; /* price from beginning of segment to this position */
|
||||
U32 off; /* offset of previous match */
|
||||
U32 mlen; /* length of previous match */
|
||||
U32 litlen; /* nb of literals after previous match */
|
||||
U32 rep[ZSTD_REP_NUM]; /* offset history after previous match */
|
||||
} ZSTD_optimal_t;
|
||||
|
||||
typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
|
||||
@ -174,8 +174,8 @@ typedef struct {
|
||||
unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
|
||||
unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
|
||||
unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
|
||||
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
|
||||
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
|
||||
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+2 */
|
||||
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+2 */
|
||||
|
||||
U32 litSum; /* nb of literals */
|
||||
U32 litLengthSum; /* nb of litLength codes */
|
||||
|
@ -1047,11 +1047,6 @@ ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
|
||||
* Optimal parser
|
||||
*********************************/
|
||||
|
||||
static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
|
||||
{
|
||||
return sol.litlen + sol.mlen;
|
||||
}
|
||||
|
||||
#if 0 /* debug */
|
||||
|
||||
static void
|
||||
@ -1101,10 +1096,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
|
||||
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
|
||||
ZSTD_match_t* const matches = optStatePtr->matchTable;
|
||||
ZSTD_optimal_t lastSequence;
|
||||
ZSTD_optimal_t lastStretch;
|
||||
ZSTD_optLdm_t optLdm;
|
||||
|
||||
ZSTD_memset(&lastSequence, 0, sizeof(ZSTD_optimal_t));
|
||||
ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t));
|
||||
|
||||
optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
|
||||
optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
|
||||
@ -1127,18 +1122,23 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch);
|
||||
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
|
||||
(U32)(ip-istart), (U32)(iend-ip));
|
||||
if (!nbMatches) { ip++; continue; }
|
||||
if (!nbMatches) {
|
||||
DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart));
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* initialize opt[0] */
|
||||
{ U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
|
||||
opt[0].mlen = 0; /* means is_a_literal */
|
||||
opt[0].mlen = 0; /* there are only literals so far */
|
||||
opt[0].litlen = litlen;
|
||||
/* We don't need to include the actual price of the literals because
|
||||
* it is static for the duration of the forward pass, and is included
|
||||
* in every price. We include the literal length to avoid negative
|
||||
* prices when we subtract the previous literal length.
|
||||
/* No need to include the actual price of the literals before the segment
|
||||
* because it is static for the duration of the forward pass, and is included
|
||||
* in every subsequent price. We include the literal length as the cost variation
|
||||
* of litlen depends on the value of litlen.
|
||||
*/
|
||||
opt[0].price = LL_PRICE(litlen);
|
||||
ZSTD_STATIC_ASSERT(sizeof(opt[0].rep[0]) == sizeof(rep[0]));
|
||||
memcpy(&opt[0].rep, rep, sizeof(opt[0].rep));
|
||||
|
||||
/* large match -> immediate encoding */
|
||||
{ U32 const maxML = matches[nbMatches-1].len;
|
||||
@ -1147,37 +1147,36 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart));
|
||||
|
||||
if (maxML > sufficient_len) {
|
||||
lastSequence.litlen = litlen;
|
||||
lastSequence.mlen = maxML;
|
||||
lastSequence.off = maxOffBase;
|
||||
DEBUGLOG(6, "large match (%u>%u), immediate encoding",
|
||||
lastStretch.litlen = 0;
|
||||
lastStretch.mlen = maxML;
|
||||
lastStretch.off = maxOffBase;
|
||||
DEBUGLOG(6, "large match (%u>%u) => immediate encoding",
|
||||
maxML, sufficient_len);
|
||||
cur = 0;
|
||||
last_pos = ZSTD_totalLen(lastSequence);
|
||||
last_pos = maxML;
|
||||
goto _shortestPath;
|
||||
} }
|
||||
|
||||
/* set prices for first matches starting position == 0 */
|
||||
assert(opt[0].price >= 0);
|
||||
{ U32 const literalsPrice = (U32)opt[0].price + (U32)LL_PRICE(0);
|
||||
U32 pos;
|
||||
{ U32 pos;
|
||||
U32 matchNb;
|
||||
for (pos = 1; pos < minMatch; pos++) {
|
||||
opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */
|
||||
opt[pos].mlen = 0;
|
||||
opt[pos].price = ZSTD_MAX_PRICE;
|
||||
/* will be updated later on at match check */
|
||||
}
|
||||
for (matchNb = 0; matchNb < nbMatches; matchNb++) {
|
||||
U32 const offBase = matches[matchNb].off;
|
||||
U32 const end = matches[matchNb].len;
|
||||
for ( ; pos <= end ; pos++ ) {
|
||||
U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
|
||||
U32 const sequencePrice = literalsPrice + matchPrice;
|
||||
int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
|
||||
int const sequencePrice = opt[0].price + matchPrice;
|
||||
DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
|
||||
pos, ZSTD_fCost((int)sequencePrice));
|
||||
pos, ZSTD_fCost(sequencePrice));
|
||||
opt[pos].mlen = pos;
|
||||
opt[pos].off = offBase;
|
||||
opt[pos].litlen = litlen;
|
||||
opt[pos].price = (int)sequencePrice;
|
||||
opt[pos].litlen = 0; /* end of match */
|
||||
opt[pos].price = sequencePrice + LL_PRICE(0);
|
||||
}
|
||||
opt[pos].price = ZSTD_MAX_PRICE;
|
||||
}
|
||||
@ -1192,7 +1191,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur);
|
||||
|
||||
/* Fix current position with one literal if cheaper */
|
||||
{ U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1;
|
||||
{ U32 const litlen = opt[cur-1].litlen + 1;
|
||||
int const price = opt[cur-1].price
|
||||
+ LIT_PRICE(ip+cur-1)
|
||||
+ LL_INCPRICE(litlen);
|
||||
@ -1201,7 +1200,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
|
||||
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
|
||||
opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
|
||||
if ( (optLevel == 2) /* additional check only for high modes */
|
||||
if ( 0 && (optLevel == 2) /* additional check only for high modes */
|
||||
&& (opt[cur].mlen > 0) /* interrupt a match */
|
||||
&& (LL_INCPRICE(1) < 0) ) /* ll1 is cheaper than ll0 */
|
||||
{
|
||||
@ -1214,15 +1213,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) !!!", ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals));
|
||||
/* do not take this literal */
|
||||
} else {
|
||||
opt[cur].mlen = 0;
|
||||
opt[cur].off = 0;
|
||||
opt[cur].mlen = opt[cur-1].mlen;
|
||||
opt[cur].off = opt[cur-1].off;
|
||||
opt[cur].litlen = litlen;
|
||||
opt[cur].price = price;
|
||||
}
|
||||
} else {
|
||||
/* normal case: take the literal, it's expected to be cheaper */
|
||||
opt[cur].mlen = 0;
|
||||
opt[cur].off = 0;
|
||||
/* normal case: take the literal, it's expected to be cheaper at position @cur */
|
||||
opt[cur].mlen = opt[cur-1].mlen;
|
||||
opt[cur].off = opt[cur-1].off;
|
||||
opt[cur].litlen = litlen;
|
||||
opt[cur].price = price;
|
||||
}
|
||||
@ -1240,9 +1239,10 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
*/
|
||||
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
|
||||
assert(cur >= opt[cur].mlen);
|
||||
if (opt[cur].mlen != 0) {
|
||||
if (opt[cur].litlen == 0) {
|
||||
/* just finished a match => alter offset history */
|
||||
U32 const prev = cur - opt[cur].mlen;
|
||||
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0);
|
||||
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
|
||||
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
|
||||
} else {
|
||||
ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
|
||||
@ -1255,15 +1255,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
|
||||
if ( (optLevel==0) /*static_test*/
|
||||
&& (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) {
|
||||
DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1);
|
||||
DEBUGLOG(7, "skip current position : next rPos(%u) price is cheaper", cur+1);
|
||||
continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
|
||||
}
|
||||
|
||||
assert(opt[cur].price >= 0);
|
||||
{ U32 const ll0 = (opt[cur].mlen != 0);
|
||||
U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
|
||||
U32 const previousPrice = (U32)opt[cur].price;
|
||||
U32 const basePrice = previousPrice + (U32)LL_PRICE(0);
|
||||
{ U32 const ll0 = (opt[cur].litlen == 0);
|
||||
int const previousPrice = opt[cur].price;
|
||||
int const basePrice = previousPrice + LL_PRICE(0);
|
||||
U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch);
|
||||
U32 matchNb;
|
||||
|
||||
@ -1275,18 +1274,16 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
continue;
|
||||
}
|
||||
|
||||
{ U32 const maxML = matches[nbMatches-1].len;
|
||||
DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u",
|
||||
inr-istart, cur, nbMatches, maxML);
|
||||
{ U32 const longestML = matches[nbMatches-1].len;
|
||||
DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of longest ML=%u",
|
||||
inr-istart, cur, nbMatches, longestML);
|
||||
|
||||
if ( (maxML > sufficient_len)
|
||||
|| (cur + maxML >= ZSTD_OPT_NUM) ) {
|
||||
lastSequence.mlen = maxML;
|
||||
lastSequence.off = matches[nbMatches-1].off;
|
||||
lastSequence.litlen = litlen;
|
||||
cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */
|
||||
last_pos = cur + ZSTD_totalLen(lastSequence);
|
||||
if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */
|
||||
if ( (longestML > sufficient_len)
|
||||
|| (cur + longestML >= ZSTD_OPT_NUM) ) {
|
||||
lastStretch.mlen = longestML;
|
||||
lastStretch.off = matches[nbMatches-1].off;
|
||||
lastStretch.litlen = 0;
|
||||
last_pos = cur + longestML;
|
||||
goto _shortestPath;
|
||||
} }
|
||||
|
||||
@ -1298,11 +1295,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
U32 mlen;
|
||||
|
||||
DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u",
|
||||
matchNb, matches[matchNb].off, lastML, litlen);
|
||||
matchNb, matches[matchNb].off, lastML, opt[cur].litlen);
|
||||
|
||||
for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */
|
||||
U32 const pos = cur + mlen;
|
||||
int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
|
||||
int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
|
||||
|
||||
if ((pos > last_pos) || (price < opt[pos].price)) {
|
||||
DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)",
|
||||
@ -1310,7 +1307,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */
|
||||
opt[pos].mlen = mlen;
|
||||
opt[pos].off = offset;
|
||||
opt[pos].litlen = litlen;
|
||||
opt[pos].litlen = 0;
|
||||
opt[pos].price = price;
|
||||
} else {
|
||||
DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)",
|
||||
@ -1321,41 +1318,77 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
||||
opt[last_pos+1].price = ZSTD_MAX_PRICE;
|
||||
} /* for (cur = 1; cur <= last_pos; cur++) */
|
||||
|
||||
lastSequence = opt[last_pos];
|
||||
cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */
|
||||
assert(cur < ZSTD_OPT_NUM); /* control overflow*/
|
||||
lastStretch = opt[last_pos];
|
||||
assert(cur >= lastStretch.mlen);
|
||||
cur = last_pos - lastStretch.mlen;
|
||||
|
||||
_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
assert(opt[0].mlen == 0);
|
||||
assert(last_pos >= lastStretch.mlen);
|
||||
assert(cur == last_pos - lastStretch.mlen);
|
||||
assert(lastStretch.rep[0] != 0);
|
||||
|
||||
/* Set the next chunk's repcodes based on the repcodes of the beginning
|
||||
* of the last match, and the last sequence. This avoids us having to
|
||||
* update them while traversing the sequences.
|
||||
*/
|
||||
if (lastSequence.mlen != 0) {
|
||||
repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0);
|
||||
ZSTD_memcpy(rep, &reps, sizeof(reps));
|
||||
if (lastStretch.mlen==0) {
|
||||
/* no solution : all matches have been converted into literals */
|
||||
assert(lastStretch.litlen == (ip - anchor) + last_pos);
|
||||
ip += last_pos;
|
||||
continue;
|
||||
}
|
||||
assert(lastStretch.off > 0);
|
||||
|
||||
/* Update offset history */
|
||||
if (lastStretch.litlen == 0) {
|
||||
/* finishing on a match : update offset history */
|
||||
repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0);
|
||||
ZSTD_memcpy(rep, &reps, sizeof(repcodes_t));
|
||||
} else {
|
||||
ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t));
|
||||
ZSTD_memcpy(rep, lastStretch.rep, sizeof(repcodes_t));
|
||||
assert(cur >= lastStretch.litlen);
|
||||
cur -= lastStretch.litlen;
|
||||
}
|
||||
|
||||
{ U32 const storeEnd = cur + 1;
|
||||
/* let's write the shortest path solution
|
||||
* solution is stored in @opt,
|
||||
* in reverse order,
|
||||
* starting from @storeEnd (==cur+1)
|
||||
* (effectively partially overwriting @opt).
|
||||
* Content is changed too:
|
||||
* - So far, @opt stored stretches, aka a match followed by literals
|
||||
* - Now, it will store sequences, aka literals followed by a match
|
||||
*/
|
||||
{ U32 const storeEnd = cur + 2;
|
||||
U32 storeStart = storeEnd;
|
||||
U32 seqPos = cur;
|
||||
U32 stretchPos = cur;
|
||||
ZSTD_optimal_t nextStretch;
|
||||
|
||||
DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
|
||||
last_pos, cur); (void)last_pos;
|
||||
assert(storeEnd < ZSTD_OPT_NUM);
|
||||
DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
||||
storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
|
||||
opt[storeEnd] = lastSequence;
|
||||
while (seqPos > 0) {
|
||||
U32 const backDist = ZSTD_totalLen(opt[seqPos]);
|
||||
DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
||||
storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off);
|
||||
if (lastStretch.litlen > 0) {
|
||||
/* last "sequence" is unfinished: just a bunch of literals */
|
||||
opt[storeEnd].litlen = lastStretch.litlen;
|
||||
opt[storeEnd].mlen = 0;
|
||||
storeStart = storeEnd-1;
|
||||
opt[storeStart] = lastStretch;
|
||||
} {
|
||||
opt[storeEnd] = lastStretch; /* note: litlen will be fixed */
|
||||
storeStart = storeEnd;
|
||||
}
|
||||
while (1) {
|
||||
nextStretch = opt[stretchPos];
|
||||
opt[storeStart].litlen = nextStretch.litlen;
|
||||
DEBUGLOG(6, "selected sequence (llen=%u,mlen=%u,ofc=%u)",
|
||||
opt[storeStart].litlen, opt[storeStart].mlen, opt[storeStart].off);
|
||||
if (nextStretch.mlen == 0) {
|
||||
/* reaching beginning of segment */
|
||||
break;
|
||||
}
|
||||
storeStart--;
|
||||
DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
||||
seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off);
|
||||
opt[storeStart] = opt[seqPos];
|
||||
seqPos = (seqPos > backDist) ? seqPos - backDist : 0;
|
||||
opt[storeStart] = nextStretch; /* note: litlen will be fixed */
|
||||
assert(nextStretch.litlen + nextStretch.mlen <= stretchPos);
|
||||
stretchPos -= nextStretch.litlen + nextStretch.mlen;
|
||||
}
|
||||
|
||||
/* save sequences */
|
||||
@ -1381,6 +1414,9 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
anchor += advance;
|
||||
ip = anchor;
|
||||
} }
|
||||
DEBUGLOG(7, "new offset history : %u, %u, %u", rep[0], rep[1], rep[2]);
|
||||
|
||||
/* update all costs */
|
||||
ZSTD_setBasePrices(optStatePtr, optLevel);
|
||||
}
|
||||
} /* while (ip < ilimit) */
|
||||
@ -1476,7 +1512,7 @@ size_t ZSTD_compressBlock_btultra2(
|
||||
* Consequently, this can only work if no data has been previously loaded in tables,
|
||||
* aka, no dictionary, no prefix, no ldm preprocessing.
|
||||
* The compression ratio gain is generally small (~0.5% on first block),
|
||||
** the cost is 2x cpu time on first block. */
|
||||
* the cost is 2x cpu time on first block. */
|
||||
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
|
||||
if ( (ms->opt.litLengthSum==0) /* first block */
|
||||
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
|
||||
|
@ -1585,7 +1585,8 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
|
||||
/* last literal segment */
|
||||
if (dctx->litBufferLocation == ZSTD_split) {
|
||||
/* split hasn't been reached yet, first get dst then copy litExtraBuffer */
|
||||
size_t const lastLLSize = litBufferEnd - litPtr;
|
||||
size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
|
||||
DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize);
|
||||
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
|
||||
if (op != NULL) {
|
||||
ZSTD_memmove(op, litPtr, lastLLSize);
|
||||
@ -1596,14 +1597,16 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
|
||||
dctx->litBufferLocation = ZSTD_not_in_dst;
|
||||
}
|
||||
/* copy last literals from internal buffer */
|
||||
{ size_t const lastLLSize = litBufferEnd - litPtr;
|
||||
{ size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
|
||||
DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize);
|
||||
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
|
||||
if (op != NULL) {
|
||||
ZSTD_memcpy(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE size_t
|
||||
@ -1673,14 +1676,16 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
|
||||
}
|
||||
|
||||
/* last literal segment */
|
||||
{ size_t const lastLLSize = litEnd - litPtr;
|
||||
{ size_t const lastLLSize = (size_t)(litEnd - litPtr);
|
||||
DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize);
|
||||
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
|
||||
if (op != NULL) {
|
||||
ZSTD_memcpy(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@ -1878,7 +1883,7 @@ ZSTD_decompressSequencesLong_body(
|
||||
}
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
return (size_t)(op - ostart);
|
||||
}
|
||||
|
||||
static size_t
|
||||
|
Loading…
x
Reference in New Issue
Block a user