diff --git a/.travis.yml b/.travis.yml index 463cb6efc..501afe9fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: c compiler: gcc -script: make test-travis +script: make $ZSTD_TRAVIS_CI_ENV before_install: - sudo apt-get update -qq - sudo apt-get install -qq gcc-arm-linux-gnueabi @@ -11,11 +11,13 @@ before_install: env: - ZSTD_TRAVIS_CI_ENV=travis-install - - ZSTD_TRAVIS_CI_ENV=test-all - ZSTD_TRAVIS_CI_ENV=clangtest - ZSTD_TRAVIS_CI_ENV=gpptest - ZSTD_TRAVIS_CI_ENV=armtest + - ZSTD_TRAVIS_CI_ENV=test + - ZSTD_TRAVIS_CI_ENV="-C programs test32" - ZSTD_TRAVIS_CI_ENV=sanitize + - ZSTD_TRAVIS_CI_ENV="-C programs memtest" matrix: fast_finish: true diff --git a/Makefile b/Makefile index c778cb722..1f7e6e823 100644 --- a/Makefile +++ b/Makefile @@ -32,40 +32,25 @@ # ################################################################ # Version number -export VERSION=0.1.0 -export RELEASE=r$(VERSION) +export VERSION=0.1.1 -DESTDIR?= -PREFIX ?= /usr - -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include PRGDIR = programs ZSTDDIR = lib -# Select test target for Travis CI's Build Matrix -ifneq (,$(filter test-%,$(ZSTD_TRAVIS_CI_ENV))) -TRAVIS_TARGET=prg-travis -else -TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV) -endif - - .PHONY: clean -default: zstdprograms +default: zstdprogram all: - @cd $(ZSTDDIR); $(MAKE) -e all - @cd $(PRGDIR); $(MAKE) -e all + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ -zstdprograms: - @cd $(PRGDIR); $(MAKE) -e +zstdprogram: + $(MAKE) -C $(PRGDIR) clean: - @cd $(PRGDIR); $(MAKE) clean - @cd $(ZSTDDIR); $(MAKE) clean -# @cd examples; $(MAKE) clean + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ @echo Cleaning completed @@ -74,23 +59,18 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) install: - @cd $(ZSTDDIR); $(MAKE) -e install - @cd $(PRGDIR); $(MAKE) -e install + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ uninstall: - @cd $(ZSTDDIR); $(MAKE) uninstall - @cd $(PRGDIR); $(MAKE) uninstall + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ travis-install: sudo $(MAKE) install test: - @cd $(PRGDIR); $(MAKE) -e test - -test-travis: $(TRAVIS_TARGET) - -prg-travis: - @cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV) + $(MAKE) -C $(PRGDIR) $@ clangtest: clean clang -v @@ -100,8 +80,8 @@ gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" armtest: clean - cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" - cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + $(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + $(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" sanitize: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" diff --git a/lib/Makefile b/lib/Makefile index 46000f7f9..142baae71 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -32,14 +32,14 @@ # ################################################################ # Version numbers -VERSION?= 0 +VERSION?= 0.1.1 LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) DESTDIR?= -PREFIX ?= /usr +PREFIX ?= /usr/local CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes LDFLAGS = -I. diff --git a/lib/fse.c b/lib/fse.c index 2c55a5635..8b1223dad 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -135,7 +135,7 @@ typedef signed long long S64; * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which generate assembly depending on alignment. + * It can generate buggy code on targets generating assembly depending on alignment. * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) @@ -1398,6 +1398,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); if (FSE_isError(errorCode)) return errorCode; if (errorCode == srcSize) return 1; + if (errorCode == 1) return 0; /* each symbol only present once */ if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); @@ -1529,12 +1530,14 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz */ static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits) { - return ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> 1) >> (((sizeof(bitD->bitContainer)*8)-1)-nbBits); + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); } static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { - return (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD->bitContainer)*8)-nbBits); + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); } static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits) @@ -1904,11 +1907,17 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (rankLast[nBitsToDecrease-1] == noOne) rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; // now there is one elt huffNode[rankLast[nBitsToDecrease]].nbBits ++; - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noOne; // rank list emptied + if (rankLast[nBitsToDecrease] == 0) + rankLast[nBitsToDecrease] = noOne; + else + { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noOne; // rank list emptied + } } - while (totalCost < 0) // Sometimes, cost correction overshoot + + while (totalCost < 0) /* Sometimes, cost correction overshoot */ { if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */ { diff --git a/lib/zstd.c b/lib/zstd.c index 462b335a0..23fdff73b 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1389,7 +1389,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream)); nbBits = offsetCode - 1; if (offsetCode==0) nbBits = 0; /* cmove */ - offset = ((size_t)1 << nbBits) + FSE_readBits(&(seqState->DStream), nbBits); + offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits); if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream)); if (offsetCode==0) offset = prevOffset; } diff --git a/programs/Makefile b/programs/Makefile index b90cfdcef..9bcb42797 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,21 +30,18 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE?= v0.1.0 +VERSION?= v0.1.1 DESTDIR?= -PREFIX ?= /usr +PREFIX ?= /usr/local CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\" +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\" LDFLAGS = -I../lib FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/share/man/man1 -ZSTDDIR=../lib - -TEST_FILES = COPYING -TEST_TARGETS=test-native +BINDIR = $(PREFIX)/bin +MANDIR = $(PREFIX)/share/man/man1 +ZSTDDIR = ../lib # Define *.exe as extension for Windows systems @@ -120,9 +117,12 @@ test: test-zstd test-fullbench test-fuzzer test32: test-zstd32 test-fullbench32 test-fuzzer32 -test-all: test test32 test-mem +test-all: test test32 memtest test-zstd: zstd datagen + @echo "*** zstd cli write error test ***" + echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi + @echo "*** zstd round-trip tests *** " ./datagen | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID) @@ -146,15 +146,15 @@ test-fuzzer: fuzzer test-fuzzer32: fuzzer32 ./fuzzer32 -test-mem: zstd datagen fuzzer fullbench +memtest: zstd datagen fuzzer fullbench @echo "\n ---- valgrind tests : memory analyzer ----" - valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) ./datagen -g16KB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID) ./datagen -g64MB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID) @rm tmp - valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1 + valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1 valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 endif diff --git a/programs/fileio.c b/programs/fileio.c index d1c4a7c7f..a496906dd 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -52,6 +52,7 @@ #include /* malloc, free */ #include /* strcmp, strlen */ #include /* clock */ +#include /* errno */ #include "fileio.h" #include "zstd_static.h" @@ -254,7 +255,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header"); + if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename); compressedfilesize += cSize; /* Main compression loop */ @@ -276,7 +277,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* Write cBlock */ sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block"); + if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename); compressedfilesize += cSize; inSlot += inSize; @@ -288,7 +289,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end"); sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end"); + if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename); compressedfilesize += cSize; /* Status */ @@ -299,9 +300,9 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* clean */ free(inBuff); free(outBuff); - fclose(finput); - fclose(foutput); ZSTD_freeCCtx(ctx); + fclose(finput); + if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); return compressedfilesize; } diff --git a/programs/fullbench.c b/programs/fullbench.c index a7dbac3b8..3ca219979 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -616,7 +616,7 @@ int main(int argc, char** argv) /* Modify Nb Iterations */ case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) + if ((argument[1] >='0') && (argument[1] <='9')) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); diff --git a/programs/unzstd.1 b/programs/unzstd.1 deleted file mode 100644 index 157fab765..000000000 --- a/programs/unzstd.1 +++ /dev/null @@ -1,31 +0,0 @@ -\" -\" unzstd.1: This is a manual page for 'unzstd' program. This file is part of -\" the zstd project. -\" - -\" No hyphenation -.hy 0 -.nr HY 0 - -.TH unzstd "1" "2014-06-20" "unzstd" "User Commands" -.SH NAME -\fBunzstd\fR - Utility based on zstd - -.SH SYNOPSIS -.TP 5 -\fBunzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] - -.SH DESCRIPTION -.PP -\fBunzstd\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm. - -\fBunzstd\fR decompress input file, it is equivalent to \fBzstd -d\fR, - -Available options are the same as \fBzstd\fR ones (man zstd). - - -.SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd/ - -.SH AUTHOR -Yann Collet diff --git a/programs/unzstd.1 b/programs/unzstd.1 new file mode 120000 index 000000000..e66191d76 --- /dev/null +++ b/programs/unzstd.1 @@ -0,0 +1 @@ +zstd.1 \ No newline at end of file diff --git a/programs/zstd.1 b/programs/zstd.1 index ce3602729..fdc8cc462 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,26 +1,35 @@ \" \" zstd.1: This is a manual page for 'zstd' program. This file is part of the -\" zstd project. +\" zstd project. +\" Author: Yann Collet \" \" No hyphenation .hy 0 .nr HY 0 -.TH zstd "1" "2015-01-22" "zstd" "User Commands" +.TH zstd "1" "2015-08-22" "zstd" "User Commands" .SH NAME -\fBzstd\fR - standard compression algorithm +\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files .SH SYNOPSIS .TP 5 \fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] +.PP +.B unzstd +is equivalent to +.BR "zstd \-d" +.br +.B zstdcat +is equivalent to +.BR "zstd \-dc" +.br .SH DESCRIPTION .PP -\fBzstd\fR is a fast lossless compression algorithm -with highly parametrable compression strength and memory usage. -It is based on the \fBLZ77\fR family, coupled with FSE entropy stage. -zstd offers compression speed of 200 MB/s per core. +\fBzstd\fR is a fast lossless compression algorithm. +It is based on the \fBLZ77\fR family, with FSE & huff0 entropy stage. +zstd offers compression speed > 200 MB/s per core. It also features a fast decoder, with speed > 500 MB/s per core. \fBzstd\fR supports the following options : @@ -35,13 +44,13 @@ It also features a fast decoder, with speed > 500 MB/s per core. .B \-f overwrite output without prompting .TP -.B \-h/\-H +.BR \-h/\-H ", " --help display help/long help and exit .TP -.B \-V +.BR \-V ", " --version display Version number and exit .TP -.B \-v +.BR \-v ", " --verbose verbose mode .TP .B \-q @@ -63,7 +72,7 @@ It also features a fast decoder, with speed > 500 MB/s per core. iteration loops [1-9](default : 3), benchmark mode only .SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd +Report bugs at:- https://github.com/Cyan4973/zstd/issues .SH AUTHOR Yann Collet diff --git a/programs/zstdcat.1 b/programs/zstdcat.1 deleted file mode 100644 index 036ac0711..000000000 --- a/programs/zstdcat.1 +++ /dev/null @@ -1,32 +0,0 @@ -\" -\" zstdcat.1: This is a manual page for 'zstdcat' program. This file is part of -\" the zstd project. -\" - -\" No hyphenation -.hy 0 -.nr HY 0 - -.TH zstdcat "1" "2014-06-20" "zstdcat" "User Commands" -.SH NAME -\fBzstdcat\fR - Utility based on zstd - -.SH SYNOPSIS -.TP 5 -\fBzstdcat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] - -.SH DESCRIPTION -.PP -\fBzstdcat\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm. - -\fBzstdcat\fR decompress input file or stream, redirecting its output to the console. -It is equivalent to \fBzstd -cd\fR, - -Available options are the same as \fBzstd\fR ones (man zstd). - - -.SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd/ - -.SH AUTHOR -Yann Collet diff --git a/programs/zstdcat.1 b/programs/zstdcat.1 new file mode 120000 index 000000000..e66191d76 --- /dev/null +++ b/programs/zstdcat.1 @@ -0,0 +1 @@ +zstd.1 \ No newline at end of file diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7fa85c855..e16b0ecde 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -86,8 +86,9 @@ /************************************** * Display Macros **************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static FILE* displayOut; static unsigned displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information @@ -137,7 +138,7 @@ static int usage_advanced(const char* programName) //DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); - DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); + DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); return 0; } @@ -173,30 +174,32 @@ int main(int argc, char** argv) char* dynNameSpace = NULL; char extension[] = ZSTD_EXTENSION; + displayOut = stderr; /* Pick out basename component. Don't rely on stdlib because of conflicting behaviour. */ for (i = (int)strlen(programName); i > 0; i--) { - if (programName[i] == '/') - { - i++; - break; - } + if (programName[i] == '/') { i++; break; } } programName += i; - /* zstdcat behavior */ + /* zstdcat preset behavior */ if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } - /* unzstd behavior */ + /* unzstd preset behavior */ if (!strcmp(programName, ZSTD_UNZSTD)) decode=1; - // command switches + /* command switches */ for(i=1; i