sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-18 00:03:50 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2708 from binhdvo/skippable

Browse Source 
Add API for fetching skippable frame content
This commit is contained in:
binhdvo 2021-06-14 19:00:31 -04:00 committed by GitHub
commit 0152435ab0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 104 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
lib/decompress/zstd_decompress.c
View File

@ -380,6 +380,19 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
return 0;
}
/*! ZSTD_isSkippableFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
*/
unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
{
if (size < ZSTD_FRAMEIDSIZE) return 0;
{ U32 const magic = MEM_readLE32(buffer);
if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
}
return 0;
}
/** ZSTD_frameHeaderSize_internal() :
* srcSize must be large enough to reach header size fields.
* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
@ -503,7 +516,6 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
}
/** ZSTD_getFrameContentSize() :
* compatible with legacy mode
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
@ -544,6 +556,37 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
}
}
/*! ZSTD_readSkippableFrame() :
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
*
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
* in the magicVariant.
*
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
*
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
const void* src, size_t srcSize)
{
U32 const magicNumber = MEM_readLE32(src);
size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
/* check input validity */
RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
/* deliver payload */
if (skippableContentSize > 0 && dst != NULL)
ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
if (magicVariant != NULL)
*magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
return skippableContentSize;
}
/** ZSTD_findDecompressedSize() :
* compatible with legacy mode
* `srcSize` must be the exact length of some number of ZSTD compressed and/or

20
lib/zstd.h
View File

@ -1441,6 +1441,26 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
ZSTDLIB_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
const void* src, size_t srcSize, unsigned magicVariant);
/*! ZSTD_readSkippableFrame() :
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
*
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
* in the magicVariant.
*
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
*
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
const void* src, size_t srcSize);
/*! ZSTD_isSkippableFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
*/
ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
/***************************************
* Memory management

40
tests/fuzzer.c
View File

@ -1830,6 +1830,46 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
DISPLAYLEVEL(3, "OK \n");
/* Simple API skippable frame test */
DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
{
U32 i;
unsigned readMagic;
unsigned long long receivedSize;
size_t skippableSize;
const U32 skipLen = 129 KB;
char* const skipBuff = (char*)malloc(skipLen);
assert(skipBuff != NULL);
for (i = 0; i < skipLen; i++)
skipBuff[i] = (BYTE) ((seed + i) % 256);
skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize,
skipBuff, skipLen, seed % 15);
CHECK_Z(skippableSize);
CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
CHECK_EQ(seed % 15, readMagic);
if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
free(skipBuff);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
{
unsigned readMagic;
unsigned long long receivedSize;
size_t skippableSize;
skippableSize = ZSTD_writeSkippableFrame((BYTE*)compressedBuffer, compressedBufferSize,
CNBuffer, 0, seed % 15);
CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
CHECK_EQ(seed % 15, readMagic);
}
DISPLAYLEVEL(3, "OK \n");
/* Dictionary and CCtx Duplication tests */
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();