sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

faster inflate() autodetection of zlib/zstd

Browse Source
This commit is contained in:
inikep 2016-09-23 14:59:46 +02:00
parent b88accfb5f
commit 57b9708054
3 changed files with 172 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
.travis.yml
View File

@ -3,9 +3,63 @@ compiler: gcc
matrix: matrix:
fast_finish: true fast_finish: true
include: include:
# OS X Mavericks
- os: osx
env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install"
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
- os: linux
sudo: false
language: cpp
install:
- export CXX="g++-4.8" CC="gcc-4.8"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
- g++-4.8
env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
- os: linux - os: linux
sudo: required sudo: required
env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest' env: PLATFORM="Ubuntu 12.04" CMD="make armtest"
addons:
apt:
packages:
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- linux-libc-dev-armel-cross
- binfmt-support
- qemu
- qemu-user-static
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make asan32"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- libc6-dev-i386
- gcc-multilib
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest"
addons: addons:
apt: apt:
packages: packages:
@ -13,7 +67,25 @@ matrix:
- os: linux - os: linux
dist: trusty dist: trusty
sudo: required sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
addons:
apt:
packages:
- libc6-dev-i386
- g++-multilib
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32"
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
addons: addons:
apt: apt:
sources: sources:

15
zlibWrapper/Makefile
View File

@ -1,8 +1,8 @@
# Makefile for example of using zstd wrapper for zlib # Makefile for example of using zstd wrapper for zlib
# #
# make - compiles statically and dynamically linked examples # make - compiles examples
# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on # make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
# make test test_d - runs statically and dynamically linked examples # make test - runs examples
# Paths to static and dynamic zlib and zstd libraries # Paths to static and dynamic zlib and zstd libraries
@ -22,7 +22,7 @@ LDFLAGS = $(LOC)
RM = rm -f RM = rm -f
all: clean fitblk example example_d zwrapbench all: clean fitblk example zwrapbench
test: example fitblk example_zstd fitblk_zstd zwrapbench test: example fitblk example_zstd fitblk_zstd zwrapbench
./example ./example
@ -34,11 +34,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench
./zwrapbench -qb1e5 ../zstd_compression_format.md ./zwrapbench -qb1e5 ../zstd_compression_format.md
./zwrapbench -qb1e5B1K ../zstd_compression_format.md ./zwrapbench -qb1e5B1K ../zstd_compression_format.md
test_d: example_d #valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
./example_d
valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
@echo "\n ---- valgrind tests ----" @echo "\n ---- valgrind tests ----"
$(VALGRIND) ./example $(VALGRIND) ./example
@ -83,5 +80,5 @@ $(ZSTDLIBDIR)/libzstd.so:
$(MAKE) -C $(ZSTDLIBDIR) all $(MAKE) -C $(ZSTDLIBDIR) all
clean: clean:
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench
@echo Cleaning completed @echo Cleaning completed

159
zlibWrapper/zstd_zlibwrapper.c
View File

@ -548,6 +548,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR; if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0); if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->decompState = Z_NEED_DICT;
if (strm->total_in == ZSTD_HEADERSIZE) { if (strm->total_in == ZSTD_HEADERSIZE) {
zwd->inBuffer.src = zwd->headerBuf; zwd->inBuffer.src = zwd->headerBuf;
@ -571,6 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
{ {
ZWRAP_DCtx* zwd;
int res; int res;
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
@ -581,60 +583,79 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
if (strm->avail_in > 0) { if (strm->avail_in > 0) {
size_t errorCode, srcSize; size_t errorCode, srcSize;
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (zwd == NULL) return Z_STREAM_ERROR;
if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
if (strm->total_in < ZLIB_HEADERSIZE) if (strm->total_in < ZLIB_HEADERSIZE) {
{ if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in); if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); if (zwd->windowBits)
strm->total_in += srcSize; errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
strm->next_in += srcSize; else
strm->avail_in -= srcSize; errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { strm->reserved = 0; /* mark as zlib stream */
z_stream strm2; errorCode = ZWRAP_freeDCtx(zwd);
strm2.next_in = strm->next_in; if (ZSTD_isError(errorCode)) goto error;
strm2.avail_in = strm->avail_in;
strm2.next_out = strm->next_out;
strm2.avail_out = strm->avail_out;
if (zwd->windowBits) if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size); else res = inflate(strm, flush);
else LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size); return res;
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode); }
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode); } else {
srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
/* inflate header */ if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
strm->next_in = (unsigned char*)zwd->headerBuf; z_stream strm2;
strm->avail_in = ZLIB_HEADERSIZE; strm2.next_in = strm->next_in;
strm->avail_out = 0; strm2.avail_in = strm->avail_in;
errorCode = inflate(strm, Z_NO_FLUSH); strm2.next_out = strm->next_out;
LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in); strm2.avail_out = strm->avail_out;
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
if (strm->avail_in > 0) goto error;
strm->next_in = strm2.next_in; if (zwd->windowBits)
strm->avail_in = strm2.avail_in; errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
strm->next_out = strm2.next_out; else
strm->avail_out = strm2.avail_out; errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
strm->reserved = 0; /* mark as zlib stream */ /* inflate header */
errorCode = ZWRAP_freeDCtx(zwd); strm->next_in = (unsigned char*)zwd->headerBuf;
if (ZSTD_isError(errorCode)) goto error; strm->avail_in = ZLIB_HEADERSIZE;
strm->avail_out = 0;
errorCode = inflate(strm, Z_NO_FLUSH);
LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
if (strm->avail_in > 0) goto error;
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); strm->next_in = strm2.next_in;
else res = inflate(strm, flush); strm->avail_in = strm2.avail_in;
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); strm->next_out = strm2.next_out;
return res; strm->avail_out = strm2.avail_out;
strm->reserved = 0; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) goto error;
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
}
} }
} }
if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
if (!zwd->zbd) { if (!zwd->zbd) {
zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; } if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
@ -642,31 +663,36 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
if (strm->total_in < ZSTD_HEADERSIZE) if (strm->total_in < ZSTD_HEADERSIZE)
{ {
srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in); if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize); if (zwd->decompState != Z_NEED_DICT) {
strm->total_in += srcSize; errorCode = ZSTD_initDStream(zwd->zbd);
strm->next_in += srcSize; if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
strm->avail_in -= srcSize; }
if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK; } else {
srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
errorCode = ZSTD_initDStream(zwd->zbd); errorCode = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; } if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } zwd->inBuffer.src = zwd->headerBuf;
zwd->inBuffer.size = ZSTD_HEADERSIZE;
zwd->inBuffer.src = zwd->headerBuf; zwd->inBuffer.pos = 0;
zwd->inBuffer.size = ZSTD_HEADERSIZE; zwd->outBuffer.dst = strm->next_out;
zwd->inBuffer.pos = 0; zwd->outBuffer.size = 0;
zwd->outBuffer.dst = strm->next_out; zwd->outBuffer.pos = 0;
zwd->outBuffer.size = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
zwd->outBuffer.pos = 0; LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); if (ZSTD_isError(errorCode)) {
LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
if (ZSTD_isError(errorCode)) { goto error;
LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); }
goto error; if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
} }
if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
} }
zwd->inBuffer.src = strm->next_in; zwd->inBuffer.src = strm->next_in;
@ -694,13 +720,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
zwd->decompState = Z_STREAM_END; zwd->decompState = Z_STREAM_END;
return Z_STREAM_END; return Z_STREAM_END;
} }
goto finish;
error:
return ZWRAPD_finishWithError(zwd, strm, 0);
} }
finish:
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
return Z_OK; return Z_OK;
error:
return ZWRAPD_finishWithError(zwd, strm, 0);
} }