sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

ZSTD_compress_generic() supports multiple successive frames

Browse Source 
also : clarified streaming API implementation
This commit is contained in:
Yann Collet 2017-05-19 10:17:59 -07:00
parent 6d4fef36de
commit 009d604e00
4 changed files with 141 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

227
lib/compress/zstd_compress.c
View File

@ -41,7 +41,12 @@
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
# include <stdio.h> # include <stdio.h>
static unsigned g_debugLevel = ZSTD_DEBUG; static unsigned g_debugLevel = ZSTD_DEBUG;
# define DEBUGLOG(l, ...) if (l<=g_debugLevel) { fprintf(stderr, __FILE__ ": "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, " \n"); } # define DEBUGLOG(l, ...) { \
if (l<=g_debugLevel) { \
fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \
} }
#else #else
# define DEBUGLOG(l, ...) {} /* disabled */ # define DEBUGLOG(l, ...) {} /* disabled */
#endif #endif
@ -185,6 +190,11 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
+ cctx->outBuffSize + cctx->inBuffSize; + cctx->outBuffSize + cctx->inBuffSize;
} }
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{
return ZSTD_sizeof_CCtx(zcs); /* same object */
}
/* private API call, for dictBuilder only */ /* private API call, for dictBuilder only */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
@ -423,12 +433,16 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
} }
static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) static U32 ZSTD_equivalentParams(ZSTD_compressionParameters cParams1,
ZSTD_compressionParameters cParams2)
{ {
return (param1.cParams.hashLog == param2.cParams.hashLog) U32 bslog1 = MIN(cParams1.windowLog, 17);
& (param1.cParams.chainLog == param2.cParams.chainLog) U32 bslog2 = MIN(cParams2.windowLog, 17);
& (param1.cParams.strategy == param2.cParams.strategy) /* opt parser space */ return (bslog1 == bslog2) /* same block size */
& ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3)); /* hashlog3 space */ & (cParams1.hashLog == cParams2.hashLog)
& (cParams1.chainLog == cParams2.chainLog)
& (cParams1.strategy == cParams2.strategy) /* opt parser space */
& ((cParams1.searchLength==3) == (cParams2.searchLength==3)); /* hashlog3 space */
} }
/*! ZSTD_continueCCtx() : /*! ZSTD_continueCCtx() :
@ -459,11 +473,12 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
ZSTD_parameters params, U64 frameContentSize, ZSTD_parameters params, U64 frameContentSize,
ZSTD_compResetPolicy_e const crp) ZSTD_compResetPolicy_e const crp)
{ {
DEBUGLOG(5, "ZSTD_resetCCtx_internal \n"); DEBUGLOG(5, "ZSTD_resetCCtx_internal : wlog=%u / old=%u",
params.cParams.windowLog, zc->params.cParams.windowLog);
if (crp == ZSTDcrp_continue) if (crp == ZSTDcrp_continue)
if (ZSTD_equivalentParams(params, zc->params)) { if (ZSTD_equivalentParams(params.cParams, zc->params.cParams)) {
DEBUGLOG(5, "ZSTD_equivalentParams()==1 \n"); DEBUGLOG(5, "ZSTD_equivalentParams()==1");
zc->fseCTables_ready = 0; zc->fseCTables_ready = 0;
zc->hufCTable_repeatMode = HUF_repeat_none; zc->hufCTable_repeatMode = HUF_repeat_none;
return ZSTD_continueCCtx(zc, params, frameContentSize); return ZSTD_continueCCtx(zc, params, frameContentSize);
@ -514,7 +529,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
/* init params */ /* init params */
zc->params = params; zc->params = params;
zc->blockSize = blockSize; zc->blockSize = blockSize;
DEBUGLOG(5, "blockSize = %uK \n", (U32)blockSize>>10); DEBUGLOG(5, "blockSize = %uK", (U32)blockSize>>10);
zc->frameContentSize = frameContentSize; zc->frameContentSize = frameContentSize;
zc->consumedSrcSize = 0; zc->consumedSrcSize = 0;
@ -544,7 +559,7 @@ static size_t ZSTD_resetCCtx_internal (ZSTD_CCtx* zc,
/* opt parser space */ /* opt parser space */
if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) { if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) {
DEBUGLOG(5, "reserving optimal parser space "); DEBUGLOG(5, "reserving optimal parser space");
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
zc->seqStore.litFreq = (U32*)ptr; zc->seqStore.litFreq = (U32*)ptr;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
@ -595,13 +610,13 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx,
ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize) ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(5, "ZSTD_copyCCtx_internal \n"); DEBUGLOG(5, "ZSTD_copyCCtx_internal");
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
{ ZSTD_parameters params = srcCCtx->params; { ZSTD_parameters params = srcCCtx->params;
params.fParams = fParams; params.fParams = fParams;
DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u \n", !fParams.noDictIDFlag); DEBUGLOG(5, "ZSTD_resetCCtx_internal : dictIDFlag : %u", !fParams.noDictIDFlag);
ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset);
} }
@ -1063,12 +1078,6 @@ _check_compressibility:
return op - ostart; return op - ostart;
} }
#if 0 /* for debug */
# define STORESEQ_DEBUG
#include <stdio.h> /* fprintf */
U32 g_startDebug = 0;
const BYTE* g_start = NULL;
#endif
/*! ZSTD_storeSeq() : /*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
@ -1077,16 +1086,16 @@ const BYTE* g_start = NULL;
*/ */
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
{ {
#ifdef STORESEQ_DEBUG #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6)
if (g_startDebug) { static const BYTE* g_start = NULL;
const U32 pos = (U32)((const BYTE*)literals - g_start); U32 const pos = (U32)((const BYTE*)literals - g_start);
if (g_start==NULL) g_start = (const BYTE*)literals; if (g_start==NULL) g_start = (const BYTE*)literals;
if ((pos > 1895000) && (pos < 1895300)) if ((pos > 0) && (pos < 1000000000))
DEBUGLOG(5, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
}
#endif #endif
/* copy Literals */ /* copy Literals */
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB);
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
seqStorePtr->lit += litLength; seqStorePtr->lit += litLength;
@ -2713,7 +2722,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
size_t pos; size_t pos;
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u \n", DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
!params.fParams.noDictIDFlag, dictID, dictIDSizeCode); !params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
MEM_writeLE32(dst, ZSTD_MAGICNUMBER); MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
@ -3024,7 +3033,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
BYTE* op = ostart; BYTE* op = ostart;
size_t fhSize = 0; size_t fhSize = 0;
DEBUGLOG(5, "ZSTD_writeEpilogue \n"); DEBUGLOG(5, "ZSTD_writeEpilogue");
if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */
/* special case : empty frame */ /* special case : empty frame */
@ -3062,7 +3071,8 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
size_t endResult; size_t endResult;
size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, size_t const cSize = ZSTD_compressContinue_internal(cctx,
dst, dstCapacity, src, srcSize,
1 /* frame mode */, 1 /* last chunk */); 1 /* frame mode */, 1 /* last chunk */);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize);
@ -3229,7 +3239,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
{ {
if (cdict==NULL) return ERROR(dictionary_wrong); /* does not support NULL cdict */ if (cdict==NULL) return ERROR(dictionary_wrong); /* does not support NULL cdict */
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u \n", !fParams.noDictIDFlag); DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced : dictIDFlag == %u", !fParams.noDictIDFlag);
if (cdict->dictContentSize) if (cdict->dictContentSize)
CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) ) CHECK_F( ZSTD_copyCCtx_internal(cctx, cdict->refContext, fParams, pledgedSrcSize) )
else { else {
@ -3246,7 +3256,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{ {
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u \n", !fParams.noDictIDFlag); DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0);
} }
@ -3318,7 +3328,7 @@ size_t ZSTD_CStreamOutSize(void)
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize) static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u \n", !zcs->params.fParams.noDictIDFlag); DEBUGLOG(5, "ZSTD_resetCStream_internal : dictIDFlag == %u", !zcs->params.fParams.noDictIDFlag);
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize)) if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict_advanced(zcs, zcs->cdict, params.fParams, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize)); else CHECK_F(ZSTD_compressBegin_internal(zcs, NULL, 0, params, pledgedSrcSize));
@ -3332,17 +3342,6 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, ZSTD_parameters para
return 0; /* ready to go */ return 0; /* ready to go */
} }
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
ZSTD_parameters params = zcs->params;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
}
return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
}
/* ZSTD_initCStream_internal() : /* ZSTD_initCStream_internal() :
* params are supposed validated at this stage * params are supposed validated at this stage
* and zcs->cdict is supposed to be correct */ * and zcs->cdict is supposed to be correct */
@ -3372,15 +3371,28 @@ static size_t ZSTD_initCStream_stage2(ZSTD_CStream* zcs,
zcs->outBuffSize = outBuffSize; zcs->outBuffSize = outBuffSize;
} }
DEBUGLOG(5, "ZSTD_initCStream_stage2 : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize); return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
} }
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
ZSTD_parameters params = zcs->params;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
}
return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
}
/* ZSTD_initCStream_usingCDict_advanced() : /* ZSTD_initCStream_usingCDict_advanced() :
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize, ZSTD_frameParameters fParams) size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
{ const ZSTD_CDict* cdict,
if (!cdict) return ERROR(dictionary_wrong); /* cannot handle NULL cdict (does not know what to do) */ unsigned long long pledgedSrcSize,
ZSTD_frameParameters fParams)
{ /* cannot handle NULL cdict (does not know what to do) */
if (!cdict) return ERROR(dictionary_wrong);
{ ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict); { ZSTD_parameters params = ZSTD_getParamsFromCDict(cdict);
params.fParams = fParams; params.fParams = fParams;
zcs->cdict = cdict; zcs->cdict = cdict;
@ -3409,7 +3421,6 @@ static size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
} }
DEBUGLOG(5, "ZSTD_initCStream_internal : dictIDFlag == %u \n", !params.fParams.noDictIDFlag);
return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize); return ZSTD_initCStream_stage2(zcs, params, pledgedSrcSize);
} }
@ -3418,8 +3429,6 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
ZSTD_parameters params, unsigned long long pledgedSrcSize) ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
DEBUGLOG(5, "ZSTD_initCStream_advanced : pledgedSrcSize == %u \n", (U32)pledgedSrcSize);
DEBUGLOG(5, "wlog %u \n", params.cParams.windowLog);
return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize); return ZSTD_initCStream_internal(zcs, dict, dictSize, params, pledgedSrcSize);
} }
@ -3442,11 +3451,6 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0); return ZSTD_initCStream_internal(zcs, NULL, 0, params, 0);
} }
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{
return ZSTD_sizeof_CCtx(zcs); /* same object */
}
/*====== Compression ======*/ /*====== Compression ======*/
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
@ -3459,7 +3463,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
void* dst, size_t* dstCapacityPtr, void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr, const void* src, size_t* srcSizePtr,
ZSTD_EndDirective const flush) ZSTD_EndDirective const flushMode)
{ {
U32 someMoreWork = 1; U32 someMoreWork = 1;
const char* const istart = (const char*)src; const char* const istart = (const char*)src;
@ -3469,9 +3473,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
char* const oend = ostart + *dstCapacityPtr; char* const oend = ostart + *dstCapacityPtr;
char* op = ostart; char* op = ostart;
DEBUGLOG(5, "ZSTD_compressStream_generic \n");
assert(zcs->inBuff != NULL); assert(zcs->inBuff != NULL);
assert(zcs->outBuff!= NULL); assert(zcs->outBuff!= NULL);
while (someMoreWork) { while (someMoreWork) {
switch(zcs->streamStage) switch(zcs->streamStage)
{ {
@ -3481,56 +3485,79 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
/* complete inBuffer */ /* complete inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip);
DEBUGLOG(5, "loading %u/%u \n", (U32)loaded, (U32)toLoad);
zcs->inBuffPos += loaded; zcs->inBuffPos += loaded;
ip += loaded; ip += loaded;
if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { if ( (flushMode == ZSTD_e_continue)
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ && (zcs->inBuffPos < zcs->inBuffTarget) ) {
} } /* not enough input to fill full block : stop here */
someMoreWork = 0; break;
}
if ( (flushMode == ZSTD_e_flush)
&& (zcs->inBuffPos == zcs->inToCompress) ) {
/* empty */
someMoreWork = 0; break;
}
}
/* compress current block (note : this stage cannot be stopped in the middle) */ /* compress current block (note : this stage cannot be stopped in the middle) */
DEBUGLOG(5, "stream compression stage (flush==%u)\n", flush); DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
{ void* cDst; { void* cDst;
size_t cSize; size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress; size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
size_t oSize = oend-op; size_t oSize = oend-op;
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
if (oSize >= ZSTD_compressBound(iSize)) if (oSize >= ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */ cDst = op; /* compress directly into output buffer (skip flush stage) */
else else
cDst = zcs->outBuff, oSize = zcs->outBuffSize; cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == ZSTD_e_end) ? cSize = lastBlock ?
ZSTD_compressEnd(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : ZSTD_compressEnd(zcs, cDst, oSize,
ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
DEBUGLOG(5, "cSize = %u \n", (U32)cSize); DEBUGLOG(5, "cSize = %u (lastBlock:%u)", (U32)cSize, lastBlock);
if (flush == ZSTD_e_end) zcs->frameEnded = 1; zcs->frameEnded = lastBlock;
/* prepare next block */ /* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize) if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffTarget == blockSize <= inBuffSize */ zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
assert(zcs->inBuffTarget <= zcs->inBuffSize); assert(zcs->inBuffTarget <= zcs->inBuffSize);
zcs->inToCompress = zcs->inBuffPos; zcs->inToCompress = zcs->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */ if (cDst == op) { /* no need to flush */
op += cSize;
if (zcs->frameEnded) {
DEBUGLOG(5, "Frame directly completed");
someMoreWork = 0;
zcs->streamStage = zcss_init;
}
break;
}
zcs->outBuffContentSize = cSize; zcs->outBuffContentSize = cSize;
zcs->outBuffFlushedSize = 0; zcs->outBuffFlushedSize = 0;
zcs->streamStage = zcss_flush; /* pass-through to flush stage */ zcs->streamStage = zcss_flush; /* pass-through to flush stage */
} }
/* fall-through */ /* fall-through */
case zcss_flush: case zcss_flush:
DEBUGLOG(5, "flush stage \n"); DEBUGLOG(5, "flush stage");
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
DEBUGLOG(5, "toFlush: %u ; flushed: %u \n", (U32)toFlush, (U32)flushed); DEBUGLOG(5, "toFlush: %u ; flushed: %u", (U32)toFlush, (U32)flushed);
op += flushed; op += flushed;
zcs->outBuffFlushedSize += flushed; zcs->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
if (zcs->frameEnded) {
DEBUGLOG(5, "Frame completed");
someMoreWork = 0;
zcs->streamStage = zcss_init;
break;
}
zcs->streamStage = zcss_load; zcs->streamStage = zcss_load;
break; break;
} }
case zcss_final: case zcss_final:
someMoreWork = 0; /* do nothing */ someMoreWork = 0; break; /* useless */
break;
default: default:
return ERROR(GENERIC); /* impossible */ return ERROR(GENERIC); /* impossible */
@ -3573,10 +3600,10 @@ size_t ZSTD_compress_generic_integral (
if (cctx->streamStage == zcss_init) { if (cctx->streamStage == zcss_init) {
/* transparent reset */ /* transparent reset */
ZSTD_parameters params = cctx->params; ZSTD_parameters params = cctx->params;
DEBUGLOG(5, "ZSTD_compress_generic_integral : transparent reset");
if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM) if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
params.cParams = ZSTD_getCParams(cctx->compressionLevel, params.cParams = ZSTD_getCParams(cctx->compressionLevel,
cctx->frameContentSize, 0 /* dictSize */); cctx->frameContentSize, 0 /* dictSize */);
DEBUGLOG(5, "starting ZSTD_resetCStream");
CHECK_F( ZSTD_initCStream_stage2(cctx, params, cctx->frameContentSize) ); CHECK_F( ZSTD_initCStream_stage2(cctx, params, cctx->frameContentSize) );
} }
@ -3626,46 +3653,26 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
BYTE* const ostart = (BYTE*)(output->dst) + output->pos; BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
DEBUGLOG(5, "ZSTD_endStream (dstCapacity : %u) \n", (U32)(oend-op));
if (zcs->streamStage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs,
DEBUGLOG(5, "calling ZSTD_endStream with outBuff=%u bytes",
(unsigned)sizeWritten);
size_t const result = ZSTD_compressStream_generic(zcs,
ostart, &sizeWritten, ostart, &sizeWritten,
&srcSize /* valid address */, &srcSize, &srcSize /* valid address */, &srcSize,
ZSTD_e_end); ZSTD_e_end);
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten;
if (remainingToFlush) {
output->pos += sizeWritten; output->pos += sizeWritten;
return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ if (ZSTD_isError(result)) return result;
+ ((zcs->params.fParams.checksumFlag > 0) * 4) /* optional 32-bits checksum */;
}
/* create epilogue */
zcs->streamStage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 :
/* write epilogue, including final empty block, into outBuff */
ZSTD_compressEnd(zcs, zcs->outBuff, zcs->outBuffSize, NULL, 0);
if (ZSTD_isError(zcs->outBuffContentSize)) return zcs->outBuffContentSize;
}
/* flush epilogue */ DEBUGLOG(5, "remaining to flush : %u",
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; (unsigned)(zcs->outBuffContentSize - zcs->outBuffFlushedSize));
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed; return zcs->outBuffContentSize - zcs->outBuffFlushedSize;
zcs->outBuffFlushedSize += flushed;
output->pos += op-ostart;
if (toFlush==flushed) zcs->streamStage = zcss_init; /* end reached */
return toFlush - flushed;
}
} }
/*-===== Pre-defined compression levels =====-*/ /*-===== Pre-defined compression levels =====-*/
#define ZSTD_MAX_CLEVEL 22 #define ZSTD_MAX_CLEVEL 22
@ -3781,11 +3788,11 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. * @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
* Size values are optional, provide 0 if not known or unused */ * Size values are optional, provide 0 if not known or unused */
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
{ {
ZSTD_compressionParameters cp; ZSTD_compressionParameters cp;
size_t const addedSize = srcSize ? 0 : 500; size_t const addedSize = srcSizeHint ? 0 : 500;
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */ if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
@ -3795,16 +3802,16 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
} }
cp = ZSTD_adjustCParams(cp, srcSize, dictSize); cp = ZSTD_adjustCParams(cp, srcSizeHint, dictSize);
return cp; return cp;
} }
/*! ZSTD_getParams() : /*! ZSTD_getParams() :
* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`).
* All fields of `ZSTD_frameParameters` are set to default (0) */ * All fields of `ZSTD_frameParameters` are set to default (0) */
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
ZSTD_parameters params; ZSTD_parameters params;
ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize);
memset(&params, 0, sizeof(params)); memset(&params, 0, sizeof(params));
params.cParams = cParams; params.cParams = cParams;
return params; return params;

6
lib/zstd.h
View File

@ -279,9 +279,11 @@ typedef struct ZSTD_outBuffer_s {
* ZSTD_endStream() instructs to finish a frame. * ZSTD_endStream() instructs to finish a frame.
* It will perform a flush and write frame epilogue. * It will perform a flush and write frame epilogue.
* The epilogue is required for decoders to consider a frame completed. * The epilogue is required for decoders to consider a frame completed.
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. * ZSTD_endStream() may not be able to flush full data if `output->size` is too small.
* In which case, call again ZSTD_endStream() to complete the flush. * In which case, call again ZSTD_endStream() to complete the flush.
* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed) * @return : 0 if frame fully completed and fully flushed,
or >0 if some data is still present within internal buffer
(value is minimum size estimation for remaining data to flush, but it could be more)
* or an error code, which can be tested using ZSTD_isError(). * or an error code, which can be tested using ZSTD_isError().
* *
* *******************************************************************/ * *******************************************************************/

12
tests/Makefile
View File

@ -26,16 +26,16 @@ PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest TESTARTEFACT := versionsTest namespaceTest
DEBUGFLAGS=-g -DZSTD_DEBUG=1 DEBUGFLAGS= -g -DZSTD_DEBUG=1
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
$(DEBUGFLAG)
CFLAGS ?= -O3 CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wformat-security \ -Wstrict-prototypes -Wundef -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wredundant-decls \
$(DEBUGFLAGS)
CFLAGS += $(MOREFLAGS) CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
@ -159,7 +159,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c
$(MAKE) -C $(ZSTDDIR) libzstd $(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
paramgrill : DEBUGFLAG = paramgrill : DEBUGFLAGS =
paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT) $(CC) $(FLAGS) $^ -lm -o $@$(EXT)

9
tests/zstreamtest.c
View File

@ -189,7 +189,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Basic compression test */ /* Basic compression test */
DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1); { size_t const r = ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1);
if (ZSTD_isError(r)) goto _output_error; }
outBuff.dst = (char*)(compressedBuffer)+cSize; outBuff.dst = (char*)(compressedBuffer)+cSize;
outBuff.size = compressedBufferSize; outBuff.size = compressedBufferSize;
outBuff.pos = 0; outBuff.pos = 0;
@ -797,13 +798,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
while (remainingToFlush) { while (remainingToFlush) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
U32 const enoughDstSize = (adjustedDstSize >= remainingToFlush);
outBuff.size = outBuff.pos + adjustedDstSize; outBuff.size = outBuff.pos + adjustedDstSize;
remainingToFlush = ZSTD_endStream(zc, &outBuff); remainingToFlush = ZSTD_endStream(zc, &outBuff);
CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
CHECK (enoughDstSize && remainingToFlush,
"ZSTD_endStream() not fully flushed (%u remaining), but enough space available (%u)",
(U32)remainingToFlush, (U32)adjustedDstSize);
} } } }
crcOrig = XXH64_digest(&xxhState); crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos; cSize = outBuff.pos;