diff --git a/Makefile b/Makefile index 96f5a9ad8..a57587341 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,12 @@ ZWRAPDIR = zlibWrapper TESTDIR = tests # Define nul output -ifneq (,$(filter Windows%,$(OS))) -VOID = nul -else VOID = /dev/null + +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = endif .PHONY: default @@ -35,7 +37,7 @@ lib: zstd: @$(MAKE) -C $(PRGDIR) - cp $(PRGDIR)/zstd . + cp $(PRGDIR)/zstd$(EXT) . .PHONY: zlibwrapper zlibwrapper: @@ -51,7 +53,7 @@ clean: @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(TESTDIR) $@ > $(VOID) @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) - @$(RM) zstd + @$(RM) zstd$(EXT) @echo Cleaning completed diff --git a/lib/Makefile b/lib/Makefile index a7df44f2f..a928c60c0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -73,9 +73,11 @@ $(LIBZSTD): LDFLAGS += -shared -fPIC $(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ +ifeq (,$(filter Windows%,$(OS))) @echo creating versioned links @ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT_MAJOR) @ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT) +endif libzstd : $(LIBZSTD) diff --git a/programs/Makefile b/programs/Makefile index 64aeb668d..b72cebdb1 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -56,7 +56,6 @@ endif # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) EXT =.exe -VOID = nul RES64_FILE = windres\zstd64.res RES32_FILE = windres\zstd32.res ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine))) @@ -66,7 +65,6 @@ else endif else EXT = -VOID = /dev/null endif diff --git a/programs/bench.c b/programs/bench.c index c477b2682..7b6e25206 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -33,7 +33,7 @@ # define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) #endif -#define NBLOOPS 3 +#define NBSECONDS 3 #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ #define COOLPERIOD_SEC 10 @@ -82,7 +82,7 @@ static clock_t g_time = 0; /* ************************************* * Benchmark Parameters ***************************************/ -static U32 g_nbIterations = NBLOOPS; +static U32 g_nbSeconds = NBSECONDS; static size_t g_blockSize = 0; int g_additionalParam = 0; @@ -90,10 +90,10 @@ void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } -void BMK_SetNbIterations(unsigned nbLoops) +void BMK_SetNbSeconds(unsigned nbSeconds) { - g_nbIterations = nbLoops; - DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations); + g_nbSeconds = nbSeconds; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); } void BMK_SetBlockSize(size_t blockSize) @@ -175,7 +175,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); UTIL_time_t coolTime; - U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100; + U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100; U64 totalCTime=0, totalDTime=0; U32 cCompleted=0, dCompleted=0; # define NB_MARKS 4 @@ -188,7 +188,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, DISPLAYLEVEL(2, "\r%79s\r", ""); while (!cCompleted | !dCompleted) { UTIL_time_t clockStart; - U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1; + U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; /* overheat protection */ if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) { @@ -306,7 +306,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, break; } } /* CRC Checking */ #endif - } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ + } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ if (g_displayLevel == 1) { double cSpeed = (double)srcSize / fastestC; @@ -361,7 +361,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, SET_HIGH_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/programs/bench.h b/programs/bench.h index 7350fd43d..1e3e3812b 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -17,7 +17,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast); /* Set Parameters */ -void BMK_SetNbIterations(unsigned nbLoops); +void BMK_SetNbSeconds(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); void BMK_setAdditionalParam(int additionalParam); void BMK_setNotificationLevel(unsigned level); diff --git a/programs/fileio.c b/programs/fileio.c index fe321015e..a17a78f30 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -375,14 +375,18 @@ static int FIO_compressFilename_dstFile(cRess_t ress, const char* dstFileName, const char* srcFileName) { int result; + stat_t statbuf; + int stat_result = 0; ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ + if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ + else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf); return result; } @@ -426,7 +430,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile SET_BINARY_MODE(stdout); for (u=0; u /* features.h with _POSIX_C_SOURCE, malloc */ #include /* fprintf */ -#include /* stat */ +#include /* stat, utime */ #include /* stat */ +#if defined(_MSC_VER) + #include /* utime */ + #include /* _chmod */ +#else + #include /* chown, stat */ + #include /* utime */ +#endif +#include /* time */ +#include #include "mem.h" /* U32, U64 */ @@ -142,6 +151,48 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ +#if defined(_MSC_VER) + #define chmod _chmod + typedef struct _stat64 stat_t; +#else + typedef struct stat stat_t; +#endif + + +UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) +{ + int res = 0; + struct utimbuf timebuf; + + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); /* set access and modification times */ + +#if !defined(_WIN32) + res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ +#endif + + res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ + + errno = 0; + return -res; /* number of errors is returned */ +} + + +UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) +{ + int r; +#if defined(_MSC_VER) + r = _stat64(infilename, statbuf); + if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ +#else + r = stat(infilename, statbuf); + if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ +#endif + return 1; +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; @@ -278,7 +329,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ -# include UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 38eef2b56..a4b8486bf 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -28,7 +28,7 @@ #include /* errno */ #include "fileio.h" #ifndef ZSTD_NOBENCH -# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ +# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */ #endif #ifndef ZSTD_NODICT # include "dibio.h" @@ -383,7 +383,7 @@ int main(int argCount, const char* argv[]) argument++; { U32 const iters = readU32FromChar(&argument); BMK_setNotificationLevel(displayLevel); - BMK_SetNbIterations(iters); + BMK_SetNbSeconds(iters); } break; diff --git a/tests/Makefile b/tests/Makefile index 80e11d6b0..26ae60078 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -50,12 +50,11 @@ ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) EXT =.exe -VOID = nul else EXT = -VOID = /dev/null endif +VOID = /dev/null ZBUFFTEST = -T2mn FUZZERTEST= -T5mn ZSTDRTTEST= --test-large-data diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index b36a90d61..48dc1a5b6 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -1,7 +1,7 @@ # Makefile for example of using zstd wrapper for zlib # # make - compiles examples -# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on +# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on # make test - runs examples @@ -15,12 +15,10 @@ ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md -CC ?= gcc -CFLAGS ?= -O3 -CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -LDFLAGS = $(LOC) -RM = rm -f + +CPPFLAGS = -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) +CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wstrict-aliasing=1 -Wundef all: clean fitblk example zwrapbench @@ -48,8 +46,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE) $(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< +#.c.o: +# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) @@ -69,10 +67,10 @@ zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c + $(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c + $(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZSTDLIBDIR)/libzstd.a: $(MAKE) -C $(ZSTDLIBDIR) libzstd.a diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index d16fcfdd5..ebf76e48f 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -130,7 +130,7 @@ void BMK_SetBlockSize(size_t blockSize) **********************************************************/ typedef struct { - const char* srcPtr; + z_const char* srcPtr; size_t srcSize; char* cPtr; size_t cRoom; @@ -145,7 +145,7 @@ typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZS #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) -static int BMK_benchMem(const void* srcBuffer, size_t srcSize, +static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor) @@ -171,7 +171,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init blockTable data */ - { const char* srcPtr = (const char*)srcBuffer; + { z_const char* srcPtr = (z_const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; U32 fileNb; @@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ } - def.next_in = (const void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const void*) blockTable[blockNb].srcPtr; def.avail_in = blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; @@ -338,7 +338,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); } - def.next_in = (const void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const void*) blockTable[blockNb].srcPtr; def.avail_in = blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; @@ -443,7 +443,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, else ret = inflateReset(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); - inf.next_in = (const void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const void*) blockTable[blockNb].cPtr; inf.avail_in = blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; @@ -475,7 +475,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, inf.opaque = Z_NULL; ret = inflateInit(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); - inf.next_in = (const void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const void*) blockTable[blockNb].cPtr; inf.avail_in = blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 31e784a80..326fea260 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -10,7 +10,9 @@ #include /* vsprintf */ #include /* va_list, for z_gzprintf */ -#include +#define NO_DUMMY_DECL +#define ZLIB_CONST +#include /* without #define Z_PREFIX */ #include "zstd_zlibwrapper.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */ #include "zstd.h" @@ -87,6 +89,9 @@ typedef struct { unsigned long long pledgedSrcSize; } ZWRAP_CCtx; +typedef ZWRAP_CCtx internal_state; + + size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) { diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index 873413907..45d15bac0 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -15,16 +15,14 @@ extern "C" { #endif +#define ZLIB_CONST #define Z_PREFIX #include #if !defined(z_const) -#if ZLIB_VERNUM >= 0x1260 - #define z_const const -#else #define z_const #endif -#endif + /* returns a string with version of zstd library */ const char * zstdVersion(void);