sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-11-04 00:02:59 -05:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2370 from facebook/libzstd_autoconf

Browse Source 
Automatic configuration detection
This commit is contained in:
Yann Collet 2020-10-26 15:16:04 -07:00 committed by GitHub
commit 284d84ecf7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 211 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Makefile
View File

@ -31,9 +31,9 @@ VOID = /dev/null
TARGET_SYSTEM ?= $(OS)
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
EXT =.exe
EXT =.exe
else
EXT =
EXT =
endif
## default: Build lib-release and zstd-release
@ -65,7 +65,7 @@ lib lib-release lib-all :
.PHONY: zstd zstd-release
zstd zstd-release:
$(Q)$(MAKE) -C $(PRGDIR) $@
$(Q)cp $(PRGDIR)/zstd$(EXT) .
$(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT)
.PHONY: zstdmt
zstdmt:
@ -79,9 +79,9 @@ zlibwrapper: lib
## test: run long-duration tests
.PHONY: test
DEBUGLEVEL ?= 1
test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
test: MOREFLAGS += -g -Werror
test:
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@
ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@

2
appveyor.yml
View File

@ -194,7 +194,7 @@
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
SCRIPT: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x86"

65
lib/Makefile
View File

@ -54,17 +54,18 @@ else
CFLAGS += -O3
endif
CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
DEBUGLEVEL ?= 0
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
endif
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef -Wpointer-arith \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wmissing-prototypes -Wc++-compat
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS)
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS)
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
GREP_OPTIONS ?=
@ -170,7 +171,29 @@ ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o)
ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated
vpath %.c $(ZSTD_SUBDIR)
BUILD_DIR ?= obj
UNAME := $(shell uname)
ifeq ($(UNAME), Darwin)
HASH ?= md5
endif
ifeq ($(UNAME), FreeBSD)
HASH ?= gmd5sum
endif
ifeq ($(UNAME), OpenBSD)
HASH ?= md5
endif
HASH ?= md5sum
HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
ifeq ($(HAVE_HASH), 1)
HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | head -c 16)
HASH_DIR := conf_$(HASH_VALUE)
else
$(info warning : could not find hash function to differentiate builds with different flags)
HASH_DIR := 0
endif
BUILD_DIR ?= obj/$(HASH_DIR)
ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynlib
ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ))
ZSTD_STATLIB_DIR := $(BUILD_DIR)/statlib
@ -178,7 +201,7 @@ ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ))
# macOS linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
ifeq ($(shell uname), Darwin)
ifeq ($(UNAME), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
@ -200,14 +223,20 @@ lib-all: all
all: lib
libzstd.a: ARFLAGS = rcs
libzstd.a: $(ZSTD_STATLIB_OBJ)
$(ZSTD_STATLIB_DIR)/libzstd.a: ARFLAGS = rcs
$(ZSTD_STATLIB_DIR)/libzstd.a: | $(ZSTD_STATLIB_DIR)
$(ZSTD_STATLIB_DIR)/libzstd.a: $(ZSTD_STATLIB_OBJ)
@echo compiling static library
$(Q)$(AR) $(ARFLAGS) $@ $^
.PHONY: libzstd.a # must be run every time
libzstd.a: $(ZSTD_STATLIB_DIR)/libzstd.a
$(Q)ln -sf $< $@
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
LIBZSTD = dll\libzstd.dll
$(LIBZSTD):
$(LIBZSTD): $(ZSTD_FILES)
@echo compiling dynamic library $(LIBVER)
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@
@ -215,15 +244,20 @@ $(LIBZSTD): $(ZSTD_FILES)
else
LIBZSTD = libzstd.$(SHARED_EXT_VER)
$(LIBZSTD): CFLAGS += -fPIC
$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden
$(LIBZSTD): $(ZSTD_DYNLIB_OBJ)
$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): CFLAGS += -fPIC
$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden
$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): | $(ZSTD_DYNLIB_DIR)
$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_DYNLIB_OBJ)
@echo compiling dynamic library $(LIBVER)
$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
@echo creating versioned links
$(Q)ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
$(Q)ln -sf $@ libzstd.$(SHARED_EXT)
.PHONY: $(LIBZSTD) # must be run every time
$(LIBZSTD): $(ZSTD_DYNLIB_DIR)/$(LIBZSTD)
$(Q)ln -sf $< $@
endif
.PHONY: libzstd
@ -281,14 +315,13 @@ clean:
$(Q)$(RM) -r *.dSYM # macOS-specific
$(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
$(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
$(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
$(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/* $(ZSTD_DYNLIB_DIR)/*
$(Q)$(RM) -r obj/*
@echo Cleaning library completed
#-----------------------------------------------------------------------------
# make install is validated only for below listed environments
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
all: libzstd.pc
@ -324,13 +357,13 @@ $(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(PREFI
endif
endif
ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
endif
ifneq (,$(filter $(shell uname),SunOS))
ifneq (,$(filter $(UNAME),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install

20
lib/README.md
View File

@ -168,6 +168,26 @@ file it should be linked with `dll\libzstd.dll`. For example:
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
#### Advanced Build options
The build system requires a hash function in order to
separate object files created with different compilation flags.
By default, it tries to use `md5sum` or equivalent.
The hash function can be manually switched by setting the `HASH` variable.
For example : `make HASH=xxhsum`
The hash function needs to generate at least 64-bit using hexadecimal format.
When no hash function is found,
the Makefile just generates all object files into the same default directory,
irrespective of compilation flags.
This functionality only matters if `libzstd` is compiled multiple times
with different build flags.
The build directory, where object files are stored
can also be manually controlled using variable `BUILD_DIR`,
for example `make BUILD_DIR=objectDir/v1`.
In which case, the hash function doesn't matter.
#### Deprecated API
Obsolete API on their way out are stored in directory `lib/deprecated`.

146
programs/Makefile
View File

@ -36,19 +36,20 @@ ZSTD_VERSION = $(LIBVER)
HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
GREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
GREP_OPTIONS += --color=never
GREP_OPTIONS += --color=never
endif
GREP = grep $(GREP_OPTIONS)
ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
ALIGN_LOOP = -falign-loops=32
ALIGN_LOOP = -falign-loops=32
else
ALIGN_LOOP =
ALIGN_LOOP =
endif
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
DEBUGLEVEL ?= 0
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL)
ifeq ($(OS),Windows_NT) # MinGW assumed
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
endif
CFLAGS ?= -O3
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
@ -65,11 +66,7 @@ ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress
ZDICT_DIR := $(ZSTDDIR)/dictBuilder
ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy
vpath %.c $(ZSTDLIB_COMMON)
vpath %.c $(ZSTDLIB_COMPRESS)
vpath %.c $(ZSTDLIB_DECOMPRESS)
vpath %.c $(ZDICT_DIR)
vpath %.c $(ZSTDLEGACY_DIR)
vpath %.c $(ZSTDLIB_COMMON) $(ZSTDLIB_COMPRESS) $(ZSTDLIB_DECOMPRESS) $(ZDICT_DIR) $(ZSTDLEGACY_DIR)
ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c)
ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c)
@ -81,7 +78,7 @@ ZSTD_LEGACY_SUPPORT ?= 5
ZSTDLEGACY_SRC :=
ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
endif
endif
@ -94,19 +91,44 @@ ZSTD_CLI_SRC := $(wildcard *.c)
ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o)
UNAME := $(shell uname)
ifeq ($(UNAME), Darwin)
HASH ?= md5
endif
ifeq ($(UNAME), FreeBSD)
HASH ?= gmd5sum
endif
ifeq ($(UNAME), OpenBSD)
HASH ?= md5
endif
HASH ?= md5sum
HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
ifeq ($(HAVE_HASH), 1)
HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) | $(HASH) | head -c 16)
HASH_DIR := conf_$(HASH_VALUE)
else
$(info warning : could not find hash function to differentiate builds with different flags)
HASH_DIR := 0
endif
BUILD_DIR ?= obj/$(HASH_DIR)
ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ))
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
RES64_FILE = windres/zstd64.res
RES32_FILE = windres/zstd32.res
EXT =.exe
RES64_FILE = windres/zstd64.res
RES32_FILE = windres/zstd32.res
ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
RES_FILE = $(RES64_FILE)
else
RES_FILE = $(RES32_FILE)
endif
else
EXT =
EXT =
endif
VOID = /dev/null
@ -119,54 +141,54 @@ NO_THREAD_MSG := ==> no threads, building without multithreading support
HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
ifeq ($(HAVE_THREAD), 1)
THREAD_MSG := ==> building with threading support
THREAD_CPP := -DZSTD_MULTITHREAD
THREAD_LD := -pthread
THREAD_MSG := ==> building with threading support
THREAD_CPP := -DZSTD_MULTITHREAD
THREAD_LD := -pthread
else
THREAD_MSG := $(NO_THREAD_MSG)
THREAD_MSG := $(NO_THREAD_MSG)
endif
# zlib detection
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
ifeq ($(HAVE_ZLIB), 1)
ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
ZLIBLD = -lz
ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
ZLIBLD = -lz
else
ZLIB_MSG := $(NO_ZLIB_MSG)
ZLIB_MSG := $(NO_ZLIB_MSG)
endif
# lzma detection
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
ifeq ($(HAVE_LZMA), 1)
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
LZMALD = -llzma
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
LZMALD = -llzma
else
LZMA_MSG := $(NO_LZMA_MSG)
LZMA_MSG := $(NO_LZMA_MSG)
endif
# lz4 detection
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
ifeq ($(HAVE_LZ4), 1)
LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
LZ4LD = -llz4
LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
LZ4LD = -llz4
else
LZ4_MSG := $(NO_LZ4_MSG)
LZ4_MSG := $(NO_LZ4_MSG)
endif
# explicit backtrace enable/disable for Linux & Darwin
ifeq ($(BACKTRACE), 0)
DEBUGFLAGS += -DBACKTRACE_ENABLE=0
DEBUGFLAGS += -DBACKTRACE_ENABLE=0
endif
ifeq (,$(filter Windows%, $(OS)))
ifeq ($(BACKTRACE), 1)
DEBUGFLAGS += -DBACKTRACE_ENABLE=1
DEBUGFLAGS_LD += -rdynamic
DEBUGFLAGS += -DBACKTRACE_ENABLE=1
DEBUGFLAGS_LD += -rdynamic
endif
endif
@ -180,18 +202,23 @@ all: zstd
.PHONY: allVariants
allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder
$(BUILD_DIR)/zstd : $(ZSTD_OBJ)
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@echo "$(LZ4_MSG)"
$(Q)$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
.PHONY: zstd
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
ifneq (,$(filter Windows%,$(OS)))
zstd : $(RES_FILE)
endif
zstd : $(ZSTDLIB_LOCAL_OBJ) $(ZSTD_CLI_OBJ)
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@echo "$(LZ4_MSG)"
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd : $(BUILD_DIR)/zstd
$(Q)ln -sf $< $@
@echo zstd build completed
.PHONY: zstd-release
zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
@ -292,6 +319,7 @@ clean:
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT)
$(Q)$(RM) -r obj/*
@echo Cleaning completed
MD2ROFF = ronn
@ -324,34 +352,32 @@ preview-man: clean-man man
# Generate .h dependencies automatically
MKDIR = mkdir
DEPFLAGS = -MT $@ -MMD -MP -MF
DEPDIR := .deps
DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d
$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR)
@echo $@
$(Q)$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $<
COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c
MKDIR ?= mkdir
$(BUILD_DIR): ; $(Q)$(MKDIR) -p $@
%.o : %.c
%.o : %.c $(DEPDIR)/%.d | $(DEPDIR)
$(COMPILE.c) $(OUTPUT_OPTION) $<
$(DEPDIR): ; $(Q)$(MKDIR) -p $@
DEPFILES := $(ZSTD_ALL_SRC:%.c=$(DEPDIR)/%.d)
DEPFILES := $(ZSTD_OBJ:.o=.d)
$(DEPFILES):
include $(wildcard $(DEPFILES))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
UNAME := $(shell uname)
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
ifeq ($HAVE_COLORNEVER, 1)
EGREP_OPTIONS += --color=never
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
AWK = awk
@ -389,17 +415,17 @@ datarootdir ?= $(PREFIX)/share
mandir ?= $(datarootdir)/man
man1dir ?= $(mandir)/man1
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS))
MANDIR ?= $(PREFIX)/man
MAN1DIR ?= $(MANDIR)/man1
ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS))
MANDIR ?= $(PREFIX)/man
MAN1DIR ?= $(MANDIR)/man1
else
MAN1DIR ?= $(man1dir)
MAN1DIR ?= $(man1dir)
endif
ifneq (,$(filter $(shell uname),SunOS))
INSTALL ?= ginstall
ifneq (,$(filter $(UNAME),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
INSTALL ?= install
endif
INSTALL_PROGRAM ?= $(INSTALL)

63
programs/README.md
View File

@ -3,7 +3,7 @@ Command Line Interface for Zstandard library
Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
There are however other Makefile targets that create different variations of CLI:
- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd formats
- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and supports decompression of legacy zstd formats
- `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats
- `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats
- `zstd-compress` : version of CLI which can only compress into zstd format
@ -147,76 +147,91 @@ FILE : a filename
Arguments :
-# : # compression level (1-19, default: 3)
-d : decompression
-D file: use `file` as Dictionary
-o file: result stored into `file` (only if 1 input file)
-f : overwrite output without prompting and (de)compress links
-D DICT: use DICT as Dictionary for compression or decompression
-o file: result stored into `file` (only 1 output file)
-f : overwrite output without prompting, also (de)compress links
--rm : remove source file(s) after successful de/compression
-k : preserve source file(s) (default)
-h/-H : display help/long help and exit
Advanced arguments :
-V : display Version number and exit
-c : force write to standard output, even if it is the console
-v : verbose mode; specify multiple times to increase verbosity
-q : suppress warnings; specify twice to suppress errors too
-c : force write to standard output, even if it is the console
-l : print information about zstd compressed files
--exclude-compressed: only compress files that are not previously compressed
--no-progress : do not display the progress counter
-r : operate recursively on directories
--filelist FILE : read list of files to operate upon from FILE
--output-dir-flat DIR : processed files are stored into DIR
--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure
--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled). If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate).
-- : All arguments after "--" are treated as files
Advanced compression arguments :
--ultra : enable levels beyond 19, up to 22 (requires more memory)
--long[=#]: enable long distance matching with given window log (default: 27)
--fast[=#]: switch to very fast compression levels (default: 1)
--adapt : dynamically adapt compression level to I/O conditions
--stream-size=# : optimize compression parameters for streaming input of given number of bytes
--size-hint=# optimize compression parameters for streaming input of approximately this size
--target-compressed-block-size=# : make compressed block near targeted size
-T# : spawns # compression threads (default: 1, 0==# cores)
-B# : select size of each job (default: 0==automatic)
--single-thread : use a single thread for both I/O and compression (result slightly different than -T1)
--rsyncable : compress using a rsync-friendly method (-B sets block size)
--no-dictID : don't write dictID into header (dictionary compression)
--[no-]check : integrity check (default: enabled)
--exclude-compressed: only compress files that are not already compressed
--stream-size=# : specify size of streaming input from `stdin`
--size-hint=# optimize compression parameters for streaming input of approximately this size
--target-compressed-block-size=# : generate compressed block of approximately targeted size
--no-dictID : don't write dictID into header (dictionary compression only)
--[no-]compress-literals : force (un)compressed literals
-r : operate recursively on directories
--output-dir-flat[=directory]: all resulting files stored into `directory`.
--format=zstd : compress files to the .zst format (default)
--format=gzip : compress files to the .gz format
--format=xz : compress files to the .xz format
--format=lzma : compress files to the .lzma format
--format=lz4 : compress files to the .lz4 format
Advanced decompression arguments :
-l : print information about zstd compressed files
--test : test compressed file integrity
--[no-]sparse : sparse mode (default: disabled)
-M# : Set a memory usage limit for decompression
--no-progress : do not display the progress bar
-- : All arguments after "--" are treated as files
--[no-]sparse : sparse mode (default: disabled)
Dictionary builder :
--train ## : create a dictionary from a training set of files
--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args
--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
-o file : `file` is dictionary name (default: dictionary)
-o DICT : DICT is dictionary name (default: dictionary)
--maxdict=# : limit dictionary to specified size (default: 112640)
--dictID=# : force dictionary ID to specified value (default: random)
Benchmark arguments :
-b# : benchmark file(s), using # compression level (default: 3)
-e# : test all compression levels from -bX to # (default: 1)
-e# : test all compression levels successively from -b# to -e# (default: 1)
-i# : minimum evaluation time in seconds (default: 3s)
-B# : cut file into independent blocks of size # (default: no block)
-S : output one benchmark result per input file (default: consolidated result)
--priority=rt : set process priority to real-time
```
### Passing parameters through Environment Variables
There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner.
Using environment variables for this purpose has security implications.
Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`.
`ZSTD_CLEVEL` can be used to modify the default compression level of `zstd`
(usually set to `3`) to another value between 1 and 19 (the "normal" range).
`ZSTD_NBTHREADS` can be used to specify number of threads that `zstd` will use during compression, which by default is `1`.
`ZSTD_NBTHREADS` can be used to specify a number of threads
that `zstd` will use for compression, which by default is `1`.
This functionality only exists when `zstd` is compiled with multithread support.
`0` means "use as many threads as detected cpu cores on local system".
The max # of threads is capped at: `ZSTDMT_NBWORKERS_MAX==200`.
This functionality can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments.
One such scenario is `tar --zstd`.
As `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` only replace the default compression level
and number of threads, respectively, they can both be overridden by corresponding command line arguments:
and number of threads respectively, they can both be overridden by corresponding command line arguments:
`-#` for compression level and `-T#` for number of threads.
There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner.
Using environment variables for this purpose has security implications.
Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`.
### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve

21
tests/Makefile
View File

@ -94,23 +94,23 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
# note : broken : requires symbols unavailable from dynamic library
dll: fuzzer-dll zstreamtest-dll
PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update
PHONY: zstd zstd32 zstd-nolegacy # phony: only external makefile knows how to build or update them
zstd zstd32 zstd-nolegacy:
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL)
gzstd:
$(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)"
$(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL)
.PHONY: libzstd
libzstd :
$(MAKE) -C $(ZSTDDIR) libzstd
$(MAKE) -C $(ZSTDDIR) libzstd DEBUGLEVEL=$(DEBUGLEVEL)
%-dll : libzstd
%-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
.PHONY: zstd-staticLib
zstd-staticLib :
$(MAKE) -C $(ZSTDDIR) libzstd.a
$(MAKE) -C $(ZSTDDIR) libzstd.a DEBUGLEVEL=$(DEBUGLEVEL)
zstdm_%.o : $(ZSTDDIR)/common/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
@ -255,7 +255,8 @@ clean:
#----------------------------------------------------------------------------------
# valgrind tests are validated only for some posix platforms
#----------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
UNAME := $(shell uname)
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
HOST_OS = POSIX
valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
@ -275,8 +276,8 @@ valgrindTest: zstd datagen fuzzer fullbench
endif
ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
HOST_OS = MSYS
ifneq (,$(filter MINGW% MSYS%,$(UNAME)))
HOST_OS = MSYS
endif
@ -286,8 +287,8 @@ endif
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
DIFF:=diff
ifneq (,$(filter $(shell uname),SunOS))
DIFF:=gdiff
ifneq (,$(filter $(UNAME),SunOS))
DIFF:=gdiff
endif
.PHONY: list

1
tests/libzstd_partial_builds.sh
View File

@ -22,7 +22,6 @@ mustBeAbsent() {
}
# default compilation : all features enabled
make clean > /dev/null
$ECHO "testing default library compilation"
CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog