From 5b147600f9ece047159abfad03a94c0e2cd86e6a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2015 17:41:46 +0100 Subject: [PATCH 01/10] Fixed #42 --- programs/Makefile | 3 +++ programs/fileio.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index b90cfdcef..525832453 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -123,6 +123,9 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32 test-all: test test32 test-mem test-zstd: zstd datagen + @echo "*** zstd cli write error test ***" + echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi + @echo "*** zstd round-trip tests *** " ./datagen | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID) diff --git a/programs/fileio.c b/programs/fileio.c index d1c4a7c7f..82fce992e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -52,6 +52,7 @@ #include /* malloc, free */ #include /* strcmp, strlen */ #include /* clock */ +#include /* errno */ #include "fileio.h" #include "zstd_static.h" @@ -299,9 +300,9 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* clean */ free(inBuff); free(outBuff); - fclose(finput); - fclose(foutput); ZSTD_freeCCtx(ctx); + fclose(finput); + if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close output"); return compressedfilesize; } From 61d08c52a58b4befc9c5102326c1a030ff9f9d2b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2015 18:13:32 +0100 Subject: [PATCH 02/10] More informative write error messages --- programs/fileio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 82fce992e..a496906dd 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -255,7 +255,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header"); + if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename); compressedfilesize += cSize; /* Main compression loop */ @@ -277,7 +277,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* /* Write cBlock */ sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block"); + if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename); compressedfilesize += cSize; inSlot += inSize; @@ -289,7 +289,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end"); sizeCheck = fwrite(outBuff, 1, cSize, foutput); - if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end"); + if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename); compressedfilesize += cSize; /* Status */ @@ -302,7 +302,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char* free(outBuff); ZSTD_freeCCtx(ctx); fclose(finput); - if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close output"); + if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); return compressedfilesize; } From c7055675225e9d67e4094019d58eabe9275acd97 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2015 22:56:42 +0100 Subject: [PATCH 03/10] Fixed : compression error on some specific distribution --- lib/fse.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index 2c55a5635..be3919f45 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -1904,11 +1904,17 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (rankLast[nBitsToDecrease-1] == noOne) rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; // now there is one elt huffNode[rankLast[nBitsToDecrease]].nbBits ++; - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noOne; // rank list emptied + if (rankLast[nBitsToDecrease] > 0) + { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noOne; // rank list emptied + } + else + rankLast[nBitsToDecrease] = noOne; // rank list emptied } - while (totalCost < 0) // Sometimes, cost correction overshoot + + while (totalCost < 0) /* Sometimes, cost correction overshoot */ { if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */ { From f44b2b02b40e99bfbf076d98c24121d6573a2296 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 25 Aug 2015 23:32:45 +0100 Subject: [PATCH 04/10] Zstd cli : --version and --help write to stdout --- Makefile | 5 ++--- lib/fse.c | 6 +++--- programs/Makefile | 5 ++--- programs/zstdcli.c | 28 +++++++++++++++------------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index c778cb722..0995d85c7 100644 --- a/Makefile +++ b/Makefile @@ -32,11 +32,10 @@ # ################################################################ # Version number -export VERSION=0.1.0 -export RELEASE=r$(VERSION) +export VERSION=0.1.1 DESTDIR?= -PREFIX ?= /usr +PREFIX ?= /usr/local LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include diff --git a/lib/fse.c b/lib/fse.c index be3919f45..6871a19e3 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -1904,14 +1904,14 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (rankLast[nBitsToDecrease-1] == noOne) rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; // now there is one elt huffNode[rankLast[nBitsToDecrease]].nbBits ++; - if (rankLast[nBitsToDecrease] > 0) + if (rankLast[nBitsToDecrease] == 0) + rankLast[nBitsToDecrease] = noOne; + else { rankLast[nBitsToDecrease]--; if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) rankLast[nBitsToDecrease] = noOne; // rank list emptied } - else - rankLast[nBitsToDecrease] = noOne; // rank list emptied } while (totalCost < 0) /* Sometimes, cost correction overshoot */ diff --git a/programs/Makefile b/programs/Makefile index 525832453..277a83c67 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -30,12 +30,12 @@ # fullbench32: Same as fullbench, but forced to compile in 32-bits mode # ########################################################################## -RELEASE?= v0.1.0 +VERSION?= v0.1.1 DESTDIR?= PREFIX ?= /usr CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(RELEASE)\" +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\" LDFLAGS = -I../lib FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -43,7 +43,6 @@ BINDIR=$(PREFIX)/bin MANDIR=$(PREFIX)/share/man/man1 ZSTDDIR=../lib -TEST_FILES = COPYING TEST_TARGETS=test-native diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7fa85c855..f3780a49f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -86,8 +86,9 @@ /************************************** * Display Macros **************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static FILE* displayOut; static unsigned displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information @@ -137,7 +138,7 @@ static int usage_advanced(const char* programName) //DISPLAY( " -t : test compressed file integrity\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); - DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); + DISPLAY( " -i# : iteration loops [1-9](default : 3)\n"); return 0; } @@ -173,30 +174,31 @@ int main(int argc, char** argv) char* dynNameSpace = NULL; char extension[] = ZSTD_EXTENSION; + displayOut = stderr; /* Pick out basename component. Don't rely on stdlib because of conflicting behaviour. */ for (i = (int)strlen(programName); i > 0; i--) { - if (programName[i] == '/') - { - i++; - break; - } + if (programName[i] == '/') { i++; break; } } programName += i; - /* zstdcat behavior */ + /* zstdcat preset behavior */ if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; } - /* unzstd behavior */ + /* unzstd preset behavior */ if (!strcmp(programName, ZSTD_UNZSTD)) decode=1; - // command switches + /* command switches */ for(i=1; i Date: Wed, 26 Aug 2015 00:19:06 +0100 Subject: [PATCH 05/10] Refactor Makefile and .travis tests --- .travis.yml | 6 ++++-- Makefile | 47 ++++++++++++++--------------------------------- lib/Makefile | 4 ++-- programs/Makefile | 22 ++++++++++------------ 4 files changed, 30 insertions(+), 49 deletions(-) diff --git a/.travis.yml b/.travis.yml index 463cb6efc..501afe9fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: c compiler: gcc -script: make test-travis +script: make $ZSTD_TRAVIS_CI_ENV before_install: - sudo apt-get update -qq - sudo apt-get install -qq gcc-arm-linux-gnueabi @@ -11,11 +11,13 @@ before_install: env: - ZSTD_TRAVIS_CI_ENV=travis-install - - ZSTD_TRAVIS_CI_ENV=test-all - ZSTD_TRAVIS_CI_ENV=clangtest - ZSTD_TRAVIS_CI_ENV=gpptest - ZSTD_TRAVIS_CI_ENV=armtest + - ZSTD_TRAVIS_CI_ENV=test + - ZSTD_TRAVIS_CI_ENV="-C programs test32" - ZSTD_TRAVIS_CI_ENV=sanitize + - ZSTD_TRAVIS_CI_ENV="-C programs memtest" matrix: fast_finish: true diff --git a/Makefile b/Makefile index 0995d85c7..1f7e6e823 100644 --- a/Makefile +++ b/Makefile @@ -34,37 +34,23 @@ # Version number export VERSION=0.1.1 -DESTDIR?= -PREFIX ?= /usr/local - -LIBDIR ?= $(PREFIX)/lib -INCLUDEDIR=$(PREFIX)/include PRGDIR = programs ZSTDDIR = lib -# Select test target for Travis CI's Build Matrix -ifneq (,$(filter test-%,$(ZSTD_TRAVIS_CI_ENV))) -TRAVIS_TARGET=prg-travis -else -TRAVIS_TARGET=$(ZSTD_TRAVIS_CI_ENV) -endif - - .PHONY: clean -default: zstdprograms +default: zstdprogram all: - @cd $(ZSTDDIR); $(MAKE) -e all - @cd $(PRGDIR); $(MAKE) -e all + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ -zstdprograms: - @cd $(PRGDIR); $(MAKE) -e +zstdprogram: + $(MAKE) -C $(PRGDIR) clean: - @cd $(PRGDIR); $(MAKE) clean - @cd $(ZSTDDIR); $(MAKE) clean -# @cd examples; $(MAKE) clean + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ @echo Cleaning completed @@ -73,23 +59,18 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) install: - @cd $(ZSTDDIR); $(MAKE) -e install - @cd $(PRGDIR); $(MAKE) -e install + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ uninstall: - @cd $(ZSTDDIR); $(MAKE) uninstall - @cd $(PRGDIR); $(MAKE) uninstall + $(MAKE) -C $(ZSTDDIR) $@ + $(MAKE) -C $(PRGDIR) $@ travis-install: sudo $(MAKE) install test: - @cd $(PRGDIR); $(MAKE) -e test - -test-travis: $(TRAVIS_TARGET) - -prg-travis: - @cd $(PRGDIR); $(MAKE) -e $(ZSTD_TRAVIS_CI_ENV) + $(MAKE) -C $(PRGDIR) $@ clangtest: clean clang -v @@ -99,8 +80,8 @@ gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" armtest: clean - cd $(ZSTDDIR); $(MAKE) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" - cd $(PRGDIR); $(MAKE) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + $(MAKE) -C $(ZSTDDIR) -e all CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + $(MAKE) -C $(PRGDIR) -e CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" sanitize: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" diff --git a/lib/Makefile b/lib/Makefile index 46000f7f9..142baae71 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -32,14 +32,14 @@ # ################################################################ # Version numbers -VERSION?= 0 +VERSION?= 0.1.1 LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h` LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) DESTDIR?= -PREFIX ?= /usr +PREFIX ?= /usr/local CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes LDFLAGS = -I. diff --git a/programs/Makefile b/programs/Makefile index 277a83c67..9bcb42797 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -33,17 +33,15 @@ VERSION?= v0.1.1 DESTDIR?= -PREFIX ?= /usr +PREFIX ?= /usr/local CFLAGS ?= -O3 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\" LDFLAGS = -I../lib FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/share/man/man1 -ZSTDDIR=../lib - -TEST_TARGETS=test-native +BINDIR = $(PREFIX)/bin +MANDIR = $(PREFIX)/share/man/man1 +ZSTDDIR = ../lib # Define *.exe as extension for Windows systems @@ -119,7 +117,7 @@ test: test-zstd test-fullbench test-fuzzer test32: test-zstd32 test-fullbench32 test-fuzzer32 -test-all: test test32 test-mem +test-all: test test32 memtest test-zstd: zstd datagen @echo "*** zstd cli write error test ***" @@ -148,15 +146,15 @@ test-fuzzer: fuzzer test-fuzzer32: fuzzer32 ./fuzzer32 -test-mem: zstd datagen fuzzer fullbench +memtest: zstd datagen fuzzer fullbench @echo "\n ---- valgrind tests : memory analyzer ----" - valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) ./datagen -g16KB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID) ./datagen -g64MB > tmp - valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp /dev/null + valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID) @rm tmp - valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i128 -t1 + valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1 valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 endif From 50b6f941e824d8c697b5151db7a1677b938ccc8d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 26 Aug 2015 10:32:17 +0100 Subject: [PATCH 06/10] added : --verbose command --- programs/zstdcli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f3780a49f..e16b0ecde 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -199,6 +199,7 @@ int main(int argc, char** argv) /* long commands (--long-word) */ if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); return 0; } if (!strcmp(argument, "--help")) { displayOut=stdout; return usage_advanced(programName); } + if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } /* Decode commands (note : aggregated commands are allowed) */ if (argument[0]=='-') @@ -208,6 +209,7 @@ int main(int argc, char** argv) { if (!inFileName) inFileName=stdinmark; else outFileName=stdoutmark; + continue; } argument++; From ddf39e91c5d02cbaf872f27ff821f10897adaa46 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 26 Aug 2015 12:04:58 +0200 Subject: [PATCH 07/10] modified man page --- programs/unzstd.1 | 32 +------------------------------- programs/zstd.1 | 31 ++++++++++++++++++++----------- programs/zstdcat.1 | 33 +-------------------------------- 3 files changed, 22 insertions(+), 74 deletions(-) mode change 100644 => 120000 programs/unzstd.1 mode change 100644 => 120000 programs/zstdcat.1 diff --git a/programs/unzstd.1 b/programs/unzstd.1 deleted file mode 100644 index 157fab765..000000000 --- a/programs/unzstd.1 +++ /dev/null @@ -1,31 +0,0 @@ -\" -\" unzstd.1: This is a manual page for 'unzstd' program. This file is part of -\" the zstd project. -\" - -\" No hyphenation -.hy 0 -.nr HY 0 - -.TH unzstd "1" "2014-06-20" "unzstd" "User Commands" -.SH NAME -\fBunzstd\fR - Utility based on zstd - -.SH SYNOPSIS -.TP 5 -\fBunzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] - -.SH DESCRIPTION -.PP -\fBunzstd\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm. - -\fBunzstd\fR decompress input file, it is equivalent to \fBzstd -d\fR, - -Available options are the same as \fBzstd\fR ones (man zstd). - - -.SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd/ - -.SH AUTHOR -Yann Collet diff --git a/programs/unzstd.1 b/programs/unzstd.1 new file mode 120000 index 000000000..e66191d76 --- /dev/null +++ b/programs/unzstd.1 @@ -0,0 +1 @@ +zstd.1 \ No newline at end of file diff --git a/programs/zstd.1 b/programs/zstd.1 index ce3602729..fdc8cc462 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,26 +1,35 @@ \" \" zstd.1: This is a manual page for 'zstd' program. This file is part of the -\" zstd project. +\" zstd project. +\" Author: Yann Collet \" \" No hyphenation .hy 0 .nr HY 0 -.TH zstd "1" "2015-01-22" "zstd" "User Commands" +.TH zstd "1" "2015-08-22" "zstd" "User Commands" .SH NAME -\fBzstd\fR - standard compression algorithm +\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files .SH SYNOPSIS .TP 5 \fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] +.PP +.B unzstd +is equivalent to +.BR "zstd \-d" +.br +.B zstdcat +is equivalent to +.BR "zstd \-dc" +.br .SH DESCRIPTION .PP -\fBzstd\fR is a fast lossless compression algorithm -with highly parametrable compression strength and memory usage. -It is based on the \fBLZ77\fR family, coupled with FSE entropy stage. -zstd offers compression speed of 200 MB/s per core. +\fBzstd\fR is a fast lossless compression algorithm. +It is based on the \fBLZ77\fR family, with FSE & huff0 entropy stage. +zstd offers compression speed > 200 MB/s per core. It also features a fast decoder, with speed > 500 MB/s per core. \fBzstd\fR supports the following options : @@ -35,13 +44,13 @@ It also features a fast decoder, with speed > 500 MB/s per core. .B \-f overwrite output without prompting .TP -.B \-h/\-H +.BR \-h/\-H ", " --help display help/long help and exit .TP -.B \-V +.BR \-V ", " --version display Version number and exit .TP -.B \-v +.BR \-v ", " --verbose verbose mode .TP .B \-q @@ -63,7 +72,7 @@ It also features a fast decoder, with speed > 500 MB/s per core. iteration loops [1-9](default : 3), benchmark mode only .SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd +Report bugs at:- https://github.com/Cyan4973/zstd/issues .SH AUTHOR Yann Collet diff --git a/programs/zstdcat.1 b/programs/zstdcat.1 deleted file mode 100644 index 036ac0711..000000000 --- a/programs/zstdcat.1 +++ /dev/null @@ -1,32 +0,0 @@ -\" -\" zstdcat.1: This is a manual page for 'zstdcat' program. This file is part of -\" the zstd project. -\" - -\" No hyphenation -.hy 0 -.nr HY 0 - -.TH zstdcat "1" "2014-06-20" "zstdcat" "User Commands" -.SH NAME -\fBzstdcat\fR - Utility based on zstd - -.SH SYNOPSIS -.TP 5 -\fBzstdcat\fR [\fBOPTIONS\fR] [-|INPUT-FILE] - -.SH DESCRIPTION -.PP -\fBzstdcat\fR is an utility based on \fBzstd\fR, a fast lossless compression algorithm. - -\fBzstdcat\fR decompress input file or stream, redirecting its output to the console. -It is equivalent to \fBzstd -cd\fR, - -Available options are the same as \fBzstd\fR ones (man zstd). - - -.SH BUGS -Report bugs at:- https://github.com/Cyan4973/zstd/ - -.SH AUTHOR -Yann Collet diff --git a/programs/zstdcat.1 b/programs/zstdcat.1 new file mode 120000 index 000000000..e66191d76 --- /dev/null +++ b/programs/zstdcat.1 @@ -0,0 +1 @@ +zstd.1 \ No newline at end of file From 6b5198fe3abf77b25d978e2f1cc448086de1dd64 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 26 Aug 2015 19:22:01 +0100 Subject: [PATCH 08/10] Updated FSE --- lib/fse.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index 6871a19e3..9221151ac 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -135,7 +135,7 @@ typedef signed long long S64; * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which generate assembly depending on alignment. + * It can generate buggy code on targets generating assembly depending on alignment. * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) @@ -1398,6 +1398,7 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); if (FSE_isError(errorCode)) return errorCode; if (errorCode == srcSize) return 1; + if (errorCode == 1) return 0; /* each symbol only present once */ if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); @@ -1534,7 +1535,7 @@ static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits) static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { - return (bitD->bitContainer << bitD->bitsConsumed) >> ((sizeof(bitD->bitContainer)*8)-nbBits); + return (bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> ((sizeof(bitD->bitContainer)*8)-nbBits); } static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits) From 352ab88aea1ea56434c4ba6052f051627b4417b2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 26 Aug 2015 22:45:23 +0100 Subject: [PATCH 09/10] fixed minor issue on decoding invalid data --- lib/fse.c | 6 ++++-- lib/zstd.c | 2 +- programs/fullbench.c | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/fse.c b/lib/fse.c index 9221151ac..d008d30c4 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -1530,12 +1530,14 @@ size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSiz */ static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits) { - return ((bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> 1) >> (((sizeof(bitD->bitContainer)*8)-1)-nbBits); + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); } static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { - return (bitD->bitContainer << (bitD->bitsConsumed & ((sizeof(bitD->bitContainer)*8)-1))) >> ((sizeof(bitD->bitContainer)*8)-nbBits); + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (-nbBits & bitMask); } static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits) diff --git a/lib/zstd.c b/lib/zstd.c index 462b335a0..23fdff73b 100644 --- a/lib/zstd.c +++ b/lib/zstd.c @@ -1389,7 +1389,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream)); nbBits = offsetCode - 1; if (offsetCode==0) nbBits = 0; /* cmove */ - offset = ((size_t)1 << nbBits) + FSE_readBits(&(seqState->DStream), nbBits); + offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits); if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream)); if (offsetCode==0) offset = prevOffset; } diff --git a/programs/fullbench.c b/programs/fullbench.c index a7dbac3b8..3ca219979 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -616,7 +616,7 @@ int main(int argc, char** argv) /* Modify Nb Iterations */ case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) + if ((argument[1] >='0') && (argument[1] <='9')) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); From 866a3c3158c826381bfe3dfb8118df90210739ee Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 26 Aug 2015 23:10:09 +0100 Subject: [PATCH 10/10] Fixed minor visual warning --- lib/fse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fse.c b/lib/fse.c index d008d30c4..8b1223dad 100644 --- a/lib/fse.c +++ b/lib/fse.c @@ -1537,7 +1537,7 @@ static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits) static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ { const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (-nbBits & bitMask); + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); } static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits)