mirror of
https://github.com/facebook/zstd.git
synced 2025-11-29 00:04:37 -05:00
added : streaming decompression API
This commit is contained in:
Yann Collet
parent
5a0c8e2439
commit
104e5b072d
@ -2758,7 +2758,9 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
|
||||
|
||||
|
||||
|
||||
/*-====== Streaming ======-*/
|
||||
/* ******************************************************************
|
||||
* Streaming
|
||||
********************************************************************/
|
||||
|
||||
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
|
||||
|
||||
@ -2815,7 +2817,10 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
|
||||
}
|
||||
|
||||
|
||||
/* Initialization */
|
||||
/*====== Initialization ======*/
|
||||
|
||||
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
|
||||
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
|
||||
|
||||
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
|
||||
const void* dict, size_t dictSize,
|
||||
@ -2863,7 +2868,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
|
||||
}
|
||||
|
||||
|
||||
/* Compression */
|
||||
/*====== Compression ======*/
|
||||
|
||||
typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e;
|
||||
|
||||
@ -2969,7 +2974,7 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor
|
||||
}
|
||||
|
||||
|
||||
/* Finalize */
|
||||
/*====== Finalize ======*/
|
||||
|
||||
/*! ZSTD_flushStream() :
|
||||
* @return : amount of data remaining to flush */
|
||||
|
||||
@ -1301,3 +1301,243 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
||||
dst, dstCapacity,
|
||||
src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
/*=====================================
|
||||
* Streaming decompression
|
||||
*====================================*/
|
||||
|
||||
typedef enum { zdss_init, zdss_loadHeader,
|
||||
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
|
||||
|
||||
/* *** Resource management *** */
|
||||
struct ZSTD_DStream_s {
|
||||
ZSTD_DCtx* zd;
|
||||
ZSTD_frameParams fParams;
|
||||
ZSTD_dStreamStage stage;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t blockSize;
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
size_t lhSize;
|
||||
ZSTD_customMem customMem;
|
||||
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
|
||||
|
||||
|
||||
ZSTD_DStream* ZSTD_createDStream(void)
|
||||
{
|
||||
return ZSTD_createDStream_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
ZSTD_DStream* zds;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream));
|
||||
if (zds==NULL) return NULL;
|
||||
memset(zds, 0, sizeof(ZSTD_DStream));
|
||||
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||
zds->zd = ZSTD_createDCtx_advanced(customMem);
|
||||
if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
|
||||
zds->stage = zdss_init;
|
||||
return zds;
|
||||
}
|
||||
|
||||
size_t ZSTD_freeDStream(ZSTD_DStream* zds)
|
||||
{
|
||||
if (zds==NULL) return 0; /* support free on null */
|
||||
ZSTD_freeDCtx(zds->zd);
|
||||
if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
|
||||
if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
|
||||
zds->customMem.customFree(zds->customMem.opaque, zds);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; }
|
||||
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
|
||||
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
|
||||
{
|
||||
zds->stage = zdss_loadHeader;
|
||||
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
|
||||
return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_initDStream(ZSTD_DStream* zds)
|
||||
{
|
||||
return ZSTD_initDStream_usingDict(zds, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input)
|
||||
{
|
||||
const char* const istart = (const char*)(input->ptr);
|
||||
const char* const iend = istart + input->size;
|
||||
const char* ip = istart;
|
||||
char* const ostart = (char*)(output->ptr);
|
||||
char* const oend = ostart + output->size;
|
||||
char* op = ostart;
|
||||
U32 someMoreWork = 1;
|
||||
|
||||
while (someMoreWork) {
|
||||
switch(zds->stage)
|
||||
{
|
||||
case zdss_init :
|
||||
return ERROR(init_missing);
|
||||
|
||||
case zdss_loadHeader :
|
||||
{ size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize);
|
||||
if (ZSTD_isError(hSize)) return hSize;
|
||||
if (hSize != 0) { /* need more input */
|
||||
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
|
||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
|
||||
zds->lhSize += iend-ip;
|
||||
input->ptr = iend;
|
||||
input->size = 0;
|
||||
return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
}
|
||||
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
|
||||
break;
|
||||
} }
|
||||
|
||||
/* Consume header */
|
||||
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */
|
||||
size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
|
||||
if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
|
||||
if (h1Size < zds->lhSize) { /* long header */
|
||||
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
|
||||
size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size);
|
||||
if (ZSTD_isError(h2Result)) return h2Result;
|
||||
} }
|
||||
|
||||
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
|
||||
size_t const neededOutSize = zds->fParams.windowSize + blockSize;
|
||||
zds->blockSize = blockSize;
|
||||
if (zds->inBuffSize < blockSize) {
|
||||
zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
|
||||
zds->inBuffSize = blockSize;
|
||||
zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize);
|
||||
if (zds->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
if (zds->outBuffSize < neededOutSize) {
|
||||
zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
|
||||
zds->outBuffSize = neededOutSize;
|
||||
zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize);
|
||||
if (zds->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
zds->stage = zdss_read;
|
||||
/* pass-through */
|
||||
|
||||
case zdss_read:
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
|
||||
if (neededInSize==0) { /* end of frame */
|
||||
zds->stage = zdss_init;
|
||||
someMoreWork = 0;
|
||||
break;
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||
const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
|
||||
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
|
||||
ip, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
ip += neededInSize;
|
||||
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
|
||||
zds->outEnd = zds->outStart + decodedSize;
|
||||
zds->stage = zdss_flush;
|
||||
break;
|
||||
}
|
||||
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
|
||||
zds->stage = zdss_load;
|
||||
/* pass-through */
|
||||
}
|
||||
|
||||
case zdss_load:
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd);
|
||||
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip);
|
||||
ip += loadedSize;
|
||||
zds->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->zd);
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zds->zd,
|
||||
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
|
||||
zds->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
zds->inPos = 0; /* input is consumed */
|
||||
if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */
|
||||
zds->outEnd = zds->outStart + decodedSize;
|
||||
zds->stage = zdss_flush;
|
||||
/* pass-through */
|
||||
} }
|
||||
|
||||
case zdss_flush:
|
||||
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
|
||||
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zds->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) { /* flush completed */
|
||||
zds->stage = zdss_read;
|
||||
if (zds->outStart + zds->blockSize > zds->outBuffSize)
|
||||
zds->outStart = zds->outEnd = 0;
|
||||
break;
|
||||
}
|
||||
/* cannot flush everything */
|
||||
someMoreWork = 0;
|
||||
break;
|
||||
}
|
||||
default: return ERROR(GENERIC); /* impossible */
|
||||
} }
|
||||
|
||||
/* result */
|
||||
input->ptr = ip;
|
||||
input->size = (size_t)(iend-ip);
|
||||
output->ptr = op;
|
||||
output->size = (size_t) (oend-op);
|
||||
output->nbBytesWritten += (size_t)(op-ostart);
|
||||
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
|
||||
if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
|
||||
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
|
||||
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
|
||||
nextSrcSizeHint -= zds->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
10
lib/zstd.h
10
lib/zstd.h
@ -344,6 +344,9 @@ typedef struct ZSTD_CStream_s ZSTD_CStream;
|
||||
ZSTD_CStream* ZSTD_createCStream(void);
|
||||
size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
|
||||
|
||||
size_t ZSTD_CStreamInSize(void); /*!< recommended size for input buffer */
|
||||
size_t ZSTD_CStreamOutSize(void); /*!< recommended size for output buffer */
|
||||
|
||||
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
|
||||
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input);
|
||||
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output);
|
||||
@ -362,9 +365,16 @@ typedef struct ZSTD_DStream_s ZSTD_DStream;
|
||||
ZSTD_DStream* ZSTD_createDStream(void);
|
||||
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
|
||||
|
||||
size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
|
||||
size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */
|
||||
|
||||
size_t ZSTD_initDStream(ZSTD_DStream* zds);
|
||||
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input);
|
||||
|
||||
/* advanced */
|
||||
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
|
||||
size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
|
||||
|
||||
|
||||
|
||||
/* ******************************************************************
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user