sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2518 from facebook/seekTable

Browse Source 
New direct seekTable access methods
This commit is contained in:
Yann Collet 2021-03-04 15:29:23 -08:00 committed by GitHub
commit c4d54ab9bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 278 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
contrib/seekable_format/tests/Makefile
View File

@ -16,13 +16,13 @@ ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
CFLAGS ?= -O3
CFLAGS += -g
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \
-Wformat=2 -Wstrict-aliasing=1
SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
.PHONY: default clean test
default: seekable_tests
default: test
test: seekable_tests
./seekable_tests
@ -30,9 +30,9 @@ test: seekable_tests
$(ZSTDLIB):
$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
seekable_tests : seekable_tests.c $(SEEKABLE_OBJS)
seekable_tests : $(SEEKABLE_OBJS)
clean:
@rm -f core *.o tmp* result* *.zst \
@$(RM) core *.o tmp* result* *.zst \
seekable_tests
@echo Cleaning completed

159
contrib/seekable_format/tests/seekable_tests.c
View File

@ -1,6 +1,8 @@
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h> // malloc
#include <stdio.h>
#include <assert.h>
#include "zstd_seekable.h"
@ -8,7 +10,83 @@
int main(int argc, const char** argv)
{
unsigned testNb = 1;
(void)argc; (void)argv;
printf("Beginning zstd seekable format tests...\n");
printf("Test %u - simple round trip: ", testNb++);
{ size_t const inSize = 4000;
void* const inBuffer = malloc(inSize);
assert(inBuffer != NULL);
size_t const seekCapacity = 5000;
void* const seekBuffer = malloc(seekCapacity);
assert(seekBuffer != NULL);
size_t seekSize;
size_t const outCapacity = inSize;
void* const outBuffer = malloc(outCapacity);
assert(outBuffer != NULL);
ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream();
assert(zscs != NULL);
{ size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */);
assert(!ZSTD_isError(initStatus));
}
{ ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity };
ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize };
size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb);
assert(!ZSTD_isError(cStatus));
assert(inb.pos == inb.size);
size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb);
assert(!ZSTD_isError(endStatus));
seekSize = outb.pos;
}
ZSTD_seekable* const stream = ZSTD_seekable_create();
assert(stream != NULL);
{ size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize);
assert(!ZSTD_isError(initStatus)); }
{ size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0);
assert(decStatus == inSize); }
/* unit test ZSTD_seekTable functions */
ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream);
assert(zst != NULL);
unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst);
assert(nbFrames > 0);
unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0);
assert(frame0Offset == 0);
unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0);
assert(content0Offset == 0);
size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0);
assert(!ZSTD_isError(cSize));
assert(cSize <= seekCapacity);
size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0);
assert(origSize == inSize);
unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1);
assert(fo1idx == 0);
free(inBuffer);
free(seekBuffer);
free(outBuffer);
ZSTD_seekable_freeCStream(zscs);
ZSTD_seekTable_free(zst);
ZSTD_seekable_free(stream);
}
printf("Success!\n");
printf("Test %u - check that seekable decompress does not hang: ", testNb++);
{ /* Github issue #2335 */
const size_t compressed_size = 17;
@ -34,20 +112,21 @@ int main(int argc, const char** argv)
const size_t uncompressed_size = 32;
uint8_t uncompressed_data[32];
ZSTD_seekable* stream = ZSTD_seekable_create();
size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size);
if (ZSTD_isError(status)) {
ZSTD_seekable_free(stream);
goto _test_error;
}
ZSTD_seekable* const stream = ZSTD_seekable_create();
assert(stream != NULL);
{ size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size);
if (ZSTD_isError(status)) {
ZSTD_seekable_free(stream);
goto _test_error;
} }
const size_t offset = 2;
/* Should return an error, but not hang */
status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset);
if (!ZSTD_isError(status)) {
ZSTD_seekable_free(stream);
goto _test_error;
}
{ const size_t offset = 2;
size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset);
if (!ZSTD_isError(status)) {
ZSTD_seekable_free(stream);
goto _test_error;
} }
ZSTD_seekable_free(stream);
}
@ -57,34 +136,34 @@ int main(int argc, const char** argv)
{ /* Github issue #FIXME */
const size_t compressed_size = 27;
const uint8_t compressed_data[27] = {
'\x28',
'\xb5',
'\x2f',
'\xfd',
'\x00',
'\x32',
'\x91',
'\x00',
'\x00',
'\x00',
'\x5e',
'\x2a',
'\x4d',
'\x18',
'\x09',
'\x00',
'\x00',
'\x00',
'\x00',
'\x00',
'\x00',
'\x00',
'\x00',
'\xb1',
'\xea',
'\x92',
'\x8f',
};
(uint8_t)'\x28',
(uint8_t)'\xb5',
(uint8_t)'\x2f',
(uint8_t)'\xfd',
(uint8_t)'\x00',
(uint8_t)'\x32',
(uint8_t)'\x91',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x5e',
(uint8_t)'\x2a',
(uint8_t)'\x4d',
(uint8_t)'\x18',
(uint8_t)'\x09',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\x00',
(uint8_t)'\xb1',
(uint8_t)'\xea',
(uint8_t)'\x92',
(uint8_t)'\x8f',
};
const size_t uncompressed_size = 400;
uint8_t uncompressed_data[400];

45
contrib/seekable_format/zstd_seekable.h
View File

@ -29,6 +29,7 @@ extern "C" {
typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream;
typedef struct ZSTD_seekable_s ZSTD_seekable;
typedef struct ZSTD_seekTable_s ZSTD_seekTable;
/*-****************************************************************************
* Seekable compression - HowTo
@ -107,6 +108,7 @@ ZSTDLIB_API size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl);
ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum);
ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output);
/*-****************************************************************************
* Seekable decompression - HowTo
* A ZSTD_seekable object is required to tracking the seekTable.
@ -161,13 +163,42 @@ ZSTDLIB_API size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t
ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex);
#define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2)
/*===== Seek Table access functions =====*/
ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs);
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex);
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex);
ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long offset);
/*===== Seekable seek table access functions =====*/
ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs);
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long offset);
/*-****************************************************************************
* Direct exploitation of the seekTable
*
* Memory constrained use cases that manage multiple archives
* benefit from retaining multiple archive seek tables
* without retaining a ZSTD_seekable instance for each.
*
* Below API allow the above-mentioned use cases
* to initialize a ZSTD_seekable, extract its (smaller) ZSTD_seekTable,
* then throw the ZSTD_seekable away to save memory.
*
* Standard ZSTD operations can then be used
* to decompress frames based on seek table offsets.
******************************************************************************/
/*===== Independent seek table management =====*/
ZSTDLIB_API ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs);
ZSTDLIB_API size_t ZSTD_seekTable_free(ZSTD_seekTable* st);
/*===== Direct seek table access functions =====*/
ZSTDLIB_API unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st);
ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long offset);
/*===== Seekable advanced I/O API =====*/
typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n);

35
contrib/seekable_format/zstdseek_compress.c
View File

@ -19,6 +19,7 @@
#include "zstd.h"
#include "zstd_errors.h"
#include "mem.h"
#include "zstd_seekable.h"
#define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; }
@ -63,19 +64,18 @@ struct ZSTD_seekable_CStream_s {
int writingSeekTable;
};
size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl)
static size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl)
{
/* allocate some initial space */
size_t const FRAMELOG_STARTING_CAPACITY = 16;
fl->entries = (framelogEntry_t*)malloc(
sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY);
if (fl->entries == NULL) return ERROR(memory_allocation);
fl->capacity = FRAMELOG_STARTING_CAPACITY;
fl->capacity = (U32)FRAMELOG_STARTING_CAPACITY;
return 0;
}
size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
{
if (fl != NULL) free(fl->entries);
return 0;
@ -83,7 +83,7 @@ size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag)
{
ZSTD_frameLog* fl = malloc(sizeof(ZSTD_frameLog));
ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog));
if (fl == NULL) return NULL;
if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) {
@ -106,10 +106,9 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl)
return 0;
}
ZSTD_seekable_CStream* ZSTD_seekable_createCStream()
ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void)
{
ZSTD_seekable_CStream* zcs = malloc(sizeof(ZSTD_seekable_CStream));
ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream));
if (zcs == NULL) return NULL;
memset(zcs, 0, sizeof(*zcs));
@ -134,7 +133,6 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
ZSTD_freeCStream(zcs->cstream);
ZSTD_seekable_frameLog_freeVec(&zcs->framelog);
free(zcs);
return 0;
}
@ -152,9 +150,8 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
return ERROR(frameParameter_unsupported);
}
zcs->maxFrameSize = maxFrameSize
? maxFrameSize
: ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE;
zcs->maxFrameSize = maxFrameSize ?
maxFrameSize : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE;
zcs->framelog.checksumFlag = checksumFlag;
if (zcs->framelog.checksumFlag) {
@ -204,7 +201,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output
/* end the frame */
size_t ret = ZSTD_endStream(zcs->cstream, output);
zcs->frameCSize += output->pos - prevOutPos;
zcs->frameCSize += (U32)(output->pos - prevOutPos);
/* need to flush before doing the rest */
if (ret) return ret;
@ -224,8 +221,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output
zcs->frameDSize = 0;
ZSTD_CCtx_reset(zcs->cstream, ZSTD_reset_session_only);
if (zcs->framelog.checksumFlag)
XXH64_reset(&zcs->xxhState, 0);
if (zcs->framelog.checksumFlag) XXH64_reset(&zcs->xxhState, 0);
return 0;
}
@ -248,8 +244,8 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
XXH64_update(&zcs->xxhState, inBase, inTmp.pos);
}
zcs->frameCSize += output->pos - prevOutPos;
zcs->frameDSize += inTmp.pos;
zcs->frameCSize += (U32)(output->pos - prevOutPos);
zcs->frameDSize += (U32)inTmp.pos;
input->pos += inTmp.pos;
@ -290,7 +286,7 @@ static inline size_t ZSTD_stwrite32(ZSTD_frameLog* fl,
memcpy((BYTE*)output->dst + output->pos,
tmp + (fl->seekTablePos - offset), lenWrite);
output->pos += lenWrite;
fl->seekTablePos += lenWrite;
fl->seekTablePos += (U32)lenWrite;
if (lenWrite < 4) return ZSTD_seekable_seekTableSize(fl) - fl->seekTablePos;
}
@ -339,8 +335,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos;
if (fl->seekTablePos < seekTableLen - 4) {
BYTE sfd = 0;
sfd |= (fl->checksumFlag) << 7;
BYTE const sfd = (BYTE)((fl->checksumFlag) << 7);
((BYTE*)output->dst)[output->pos] = sfd;
output->pos++;

148
contrib/seekable_format/zstdseek_decompress.c
View File

@ -107,7 +107,8 @@ typedef struct {
static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
{
buffWrapper_t* buff = (buffWrapper_t*) opaque;
buffWrapper_t* const buff = (buffWrapper_t*)opaque;
assert(buff != NULL);
if (buff->pos + n > buff->size) return -1;
memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n);
buff->pos += n;
@ -118,15 +119,17 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
{
buffWrapper_t* const buff = (buffWrapper_t*) opaque;
unsigned long long newOffset;
assert(buff != NULL);
switch (origin) {
case SEEK_SET:
newOffset = offset;
assert(offset >= 0);
newOffset = (unsigned long long)offset;
break;
case SEEK_CUR:
newOffset = (unsigned long long)buff->pos + offset;
newOffset = (unsigned long long)((long long)buff->pos + offset);
break;
case SEEK_END:
newOffset = (unsigned long long)buff->size + offset;
newOffset = (unsigned long long)((long long)buff->size + offset);
break;
default:
assert(0); /* not possible */
@ -144,18 +147,18 @@ typedef struct {
U32 checksum;
} seekEntry_t;
typedef struct {
struct ZSTD_seekTable_s {
seekEntry_t* entries;
size_t tableLen;
int checksumFlag;
} seekTable_t;
};
#define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_MAX
struct ZSTD_seekable_s {
ZSTD_DStream* dstream;
seekTable_t seekTable;
ZSTD_seekTable seekTable;
ZSTD_seekable_customFile src;
U64 decompressedOffset;
@ -173,8 +176,7 @@ struct ZSTD_seekable_s {
ZSTD_seekable* ZSTD_seekable_create(void)
{
ZSTD_seekable* zs = malloc(sizeof(ZSTD_seekable));
ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable));
if (zs == NULL) return NULL;
/* also initializes stage to zsds_init */
@ -195,7 +197,35 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
ZSTD_freeDStream(zs->dstream);
free(zs->seekTable.entries);
free(zs);
return 0;
}
ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
{
ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable));
if (st==NULL) return NULL;
st->checksumFlag = zs->seekTable.checksumFlag;
st->tableLen = zs->seekTable.tableLen;
/* Allocate an extra entry at the end to match logic of initial allocation */
size_t const entriesSize = sizeof(seekEntry_t) * (zs->seekTable.tableLen + 1);
seekEntry_t* const entries = (seekEntry_t*)malloc(entriesSize);
if (entries==NULL) {
free(st);
return NULL;
}
memcpy(entries, zs->seekTable.entries, entriesSize);
st->entries = entries;
return st;
}
size_t ZSTD_seekTable_free(ZSTD_seekTable* st)
{
if (st == NULL) return 0; /* support free on null */
free(st->entries);
free(st);
return 0;
}
@ -203,19 +233,24 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
* Performs a binary search to find the last frame with a decompressed offset
* <= pos
* @return : the frame's index */
unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long pos)
{
return ZSTD_seekTable_offsetToFrameIndex(&zs->seekTable, pos);
}
unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long pos)
{
U32 lo = 0;
U32 hi = (U32)zs->seekTable.tableLen;
assert(zs->seekTable.tableLen <= UINT_MAX);
U32 hi = (U32)st->tableLen;
assert(st->tableLen <= UINT_MAX);
if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) {
return (U32)zs->seekTable.tableLen;
if (pos >= st->entries[st->tableLen].dOffset) {
return (unsigned)st->tableLen;
}
while (lo + 1 < hi) {
U32 const mid = lo + ((hi - lo) >> 1);
if (zs->seekTable.entries[mid].dOffset <= pos) {
if (st->entries[mid].dOffset <= pos) {
lo = mid;
} else {
hi = mid;
@ -224,36 +259,61 @@ unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long
return lo;
}
unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs)
{
assert(zs->seekTable.tableLen <= UINT_MAX);
return (unsigned)zs->seekTable.tableLen;
return ZSTD_seekTable_getNumFrames(&zs->seekTable);
}
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st)
{
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return zs->seekTable.entries[frameIndex].cOffset;
assert(st->tableLen <= UINT_MAX);
return (unsigned)st->tableLen;
}
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex)
unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex)
{
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return zs->seekTable.entries[frameIndex].dOffset;
return ZSTD_seekTable_getFrameCompressedOffset(&zs->seekTable, frameIndex);
}
size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
return zs->seekTable.entries[frameIndex + 1].cOffset -
zs->seekTable.entries[frameIndex].cOffset;
if (frameIndex >= st->tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return st->entries[frameIndex].cOffset;
}
size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex)
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex)
{
if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge);
return zs->seekTable.entries[frameIndex + 1].dOffset -
zs->seekTable.entries[frameIndex].dOffset;
return ZSTD_seekTable_getFrameDecompressedOffset(&zs->seekTable, frameIndex);
}
unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex >= st->tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return st->entries[frameIndex].dOffset;
}
size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex)
{
return ZSTD_seekTable_getFrameCompressedSize(&zs->seekTable, frameIndex);
}
size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex >= st->tableLen) return ERROR(frameIndex_tooLarge);
return st->entries[frameIndex + 1].cOffset -
st->entries[frameIndex].cOffset;
}
size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex)
{
return ZSTD_seekTable_getFrameDecompressedSize(&zs->seekTable, frameIndex);
}
size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex > st->tableLen) return ERROR(frameIndex_tooLarge);
return st->entries[frameIndex + 1].dOffset -
st->entries[frameIndex].dOffset;
}
static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
@ -274,8 +334,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
/* check reserved bits */
if ((checksumFlag >> 2) & 0x1f) {
return ERROR(corruption_detected);
}
}
} }
{ U32 const numFrames = MEM_readLE32(zs->inBuff);
U32 const sizePerEntry = 8 + (checksumFlag?4:0);
@ -283,12 +342,9 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
{
U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE);
{ U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE);
CHECK_IO(src.seek(src.opaque, -(S64)frameSize, SEEK_END));
CHECK_IO(src.read(src.opaque, zs->inBuff, toRead));
remaining -= toRead;
}
@ -301,19 +357,15 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
{ /* Allocate an extra entry at the end so that we can do size
* computations on the last element without special case */
seekEntry_t* entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1));
seekEntry_t* const entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1));
U32 idx = 0;
U32 pos = 8;
U64 cOffset = 0;
U64 dOffset = 0;
if (!entries) {
free(entries);
return ERROR(memory_allocation);
}
if (entries == NULL) return ERROR(memory_allocation);
/* compute cumulative positions */
for (; idx < numFrames; idx++) {
@ -390,8 +442,9 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset;
zs->curFrame = targetFrame;
assert(zs->seekTable.entries[targetFrame].cOffset < LLONG_MAX);
CHECK_IO(zs->src.seek(zs->src.opaque,
zs->seekTable.entries[targetFrame].cOffset,
(long long)zs->seekTable.entries[targetFrame].cOffset,
SEEK_SET));
zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0};
XXH64_reset(&zs->xxhState, 0);
@ -460,7 +513,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
zs->in.size = toRead;
zs->in.pos = 0;
}
}
} /* while (zs->decompressedOffset < offset + len) */
} while (zs->decompressedOffset != offset + len);
return len;
@ -472,8 +525,7 @@ size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSiz
return ERROR(frameIndex_tooLarge);
}
{
size_t const decompressedSize =
{ size_t const decompressedSize =
zs->seekTable.entries[frameIndex + 1].dOffset -
zs->seekTable.entries[frameIndex].dOffset;
if (dstSize < decompressedSize) {

2
lib/common/mem.h
View File

@ -308,7 +308,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
}
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)