sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-07 00:02:39 -05:00
Code Issues Projects Releases Wiki Activity

created ZSTD_storeSeqOnly()

Browse Source 
makes it possible to register a sequence without copying its literals.
This commit is contained in:
Yann Collet 2024-12-09 21:39:34 -08:00
parent 125f05282b
commit a00f45a037
2 changed files with 57 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/compress/zstd_compress.c
View File

@ -6666,9 +6666,11 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
}
size_t
ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx,
ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
const void* src, size_t blockSize,
ZSTD_paramSwitch_e externalRepSearch)
{
U32 idx = seqPos->idx;
U32 startPosInSequence = seqPos->posInSequence;
@ -6855,7 +6857,7 @@ static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
}
}
/* Compress, block-by-block, all of the sequences given.
/* Compress all provided sequences, block-by-block.
*
* Returns the cumulative size of all compressed blocks (including their headers),
* otherwise a ZSTD error.
@ -6896,7 +6898,6 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
assert(blockSize <= remaining);
ZSTD_resetSeqStore(&cctx->seqStore);
DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");

76
lib/compress/zstd_compress_internal.h
View File

@ -671,6 +671,57 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con
#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM)
#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */
/*! ZSTD_storeSeqOnly() :
* Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
* Literals themselves are not copied, but @litPtr is updated.
* @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
* @matchLength : must be >= MINMATCH
*/
HINT_INLINE UNUSED_ATTR void
ZSTD_storeSeqOnly(seqStore_t* seqStorePtr,
size_t litLength,
U32 offBase,
size_t matchLength)
{
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
static const BYTE* g_start = NULL;
if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */
{ U32 const pos = (U32)((const BYTE*)literals - g_start);
DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u",
pos, (U32)litLength, (U32)matchLength, (U32)offBase);
}
#endif
assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
/* update seqStorePtr->lit, so that we know how many literals were or will be copied */
assert(seqStorePtr->maxNbLit <= 128 KB);
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */
seqStorePtr->sequences[0].offBase = offBase;
/* match Length */
assert(matchLength >= MINMATCH);
{ size_t const mlBase = matchLength - MINMATCH;
if (mlBase>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
}
seqStorePtr->sequences++;
}
/*! ZSTD_storeSeq() :
* Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
* @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
@ -710,31 +761,8 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
} else {
ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w);
}
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */
seqStorePtr->sequences[0].offBase = offBase;
/* match Length */
assert(matchLength >= MINMATCH);
{ size_t const mlBase = matchLength - MINMATCH;
if (mlBase>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
}
seqStorePtr->sequences++;
ZSTD_storeSeqOnly(seqStorePtr, litLength, offBase, matchLength);
}
/* ZSTD_updateRep() :