sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-10 00:06:51 -05:00
Code Issues Projects Releases Wiki Activity

streaming compression : lazy parameter adaptation with stable input

Browse Source 
effectively makes ZSTD_c_stableInput compatible ZSTD_compressStream()
and zstd_e_continue operation mode.
This commit is contained in:
Yann Collet 2021-12-31 18:20:36 -08:00
parent 5684bae4f6
commit c0c5ffa973
2 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
lib/compress/zstd_compress.c
View File

@ -177,12 +177,9 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
if (cctx==NULL) return 0; /* support free on NULL */ if (cctx==NULL) return 0; /* support free on NULL */
RETURN_ERROR_IF(cctx->staticSize, memory_allocation, RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"not compatible with static CCtx"); "not compatible with static CCtx");
{ { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
ZSTD_freeCCtxContent(cctx); ZSTD_freeCCtxContent(cctx);
if (!cctxInWorkspace) { if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem);
ZSTD_customFree(cctx, cctx->customMem);
}
} }
return 0; return 0;
} }
@ -4487,6 +4484,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
/*! ZSTD_compressBegin_internal() : /*! ZSTD_compressBegin_internal() :
* Assumption : either @dict OR @cdict (or none) is non-NULL, never both
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
@ -4567,11 +4565,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
&cctxParams, pledgedSrcSize); &cctxParams, pledgedSrcSize);
} }
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) size_t
ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_CCtx_params cctxParams; ZSTD_CCtx_params cctxParams;
{ { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
} }
DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
@ -5648,6 +5646,12 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
/* transparent initialization stage */ /* transparent initialization stage */
if (cctx->streamStage == zcss_init) { if (cctx->streamStage == zcss_init) {
if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
&& (endOp == ZSTD_e_continue) /* more to come */
&& (input->pos < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */
cctx->expectedInBuffer = *input;
return (ZSTD_BLOCKSIZE_MAX - input->pos); /* don't do anything : allows lazy compression parameters adaptation */
}
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed");
ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */
} }
@ -6159,6 +6163,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci
/*====== Finalize ======*/ /*====== Finalize ======*/
/*! ZSTD_flushStream() : /*! ZSTD_flushStream() :
* Note : not compatible with ZSTD_c_stableInBuffer
* @return : amount of data remaining to flush */ * @return : amount of data remaining to flush */
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
@ -6169,7 +6174,9 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{ {
ZSTD_inBuffer input = { NULL, 0, 0 }; ZSTD_inBuffer const nullInput = { NULL, 0, 0 };
int const stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
ZSTD_inBuffer input = stableInput ? zcs->expectedInBuffer : nullInput;
size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed");
if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */

6
lib/zstd.h
View File

@ -1843,14 +1843,14 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
* avoid the memcpy() from the input buffer to the input window buffer. * avoid the memcpy() from the input buffer to the input window buffer.
* *
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. * NOTE: ZSTD_compressStream2() will error if ZSTD_e_flush is used.
* That means this flag cannot be used with ZSTD_compressStream(). * That means this flag cannot be used with ZSTD_flushStream().
* *
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
* this flag is ALWAYS memory safe, and will never access out-of-bounds * this flag is ALWAYS memory safe, and will never access out-of-bounds
* memory. However, compression WILL fail if you violate the preconditions. * memory. However, compression WILL fail if you violate the preconditions.
* *
* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
* not be modified during compression or you will get data corruption. This * not be modified during compression or you will get data corruption. This
* is because zstd needs to reference data in the ZSTD_inBuffer to find * is because zstd needs to reference data in the ZSTD_inBuffer to find
* matches. Normally zstd maintains its own window buffer for this purpose, * matches. Normally zstd maintains its own window buffer for this purpose,