mirror of
https://github.com/facebook/zstd.git
synced 2025-12-05 00:03:19 -05:00
Merge pull request #2457 from facebook/cli-dll
zstd CLI can now be linked to libzstd dynamic library
This commit is contained in:
Yann Collet
GitHub
commit
33b73db33c
@ -971,16 +971,11 @@ static size_t ZDICT_addEntropyTablesFromBuffer_advanced(
|
||||
return MIN(dictBufferCapacity, hSize+dictContentSize);
|
||||
}
|
||||
|
||||
/* Hidden declaration for dbio.c */
|
||||
size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
void* dictBuffer, size_t maxDictSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t params);
|
||||
/*! ZDICT_trainFromBuffer_unsafe_legacy() :
|
||||
* Warning : `samplesBuffer` must be followed by noisy guard band.
|
||||
* Warning : `samplesBuffer` must be followed by noisy guard band !!!
|
||||
* @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
static size_t ZDICT_trainFromBuffer_unsafe_legacy(
|
||||
void* dictBuffer, size_t maxDictSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t params)
|
||||
|
||||
@ -264,10 +264,11 @@ typedef struct {
|
||||
* Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0.
|
||||
*/
|
||||
ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy(
|
||||
void *dictBuffer, size_t dictBufferCapacity,
|
||||
const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples,
|
||||
void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t parameters);
|
||||
|
||||
|
||||
/* Deprecation warnings */
|
||||
/* It is generally possible to disable deprecation warnings from compiler,
|
||||
for example with -Wno-deprecated-declarations for gcc
|
||||
|
||||
1
programs/.gitignore
vendored
1
programs/.gitignore
vendored
@ -8,6 +8,7 @@ zstd-frugal
|
||||
zstd-small
|
||||
zstd-nolegacy
|
||||
zstd-dictBuilder
|
||||
zstd-dll
|
||||
|
||||
# Object files
|
||||
*.o
|
||||
|
||||
@ -90,13 +90,13 @@ endif
|
||||
|
||||
# Sort files in alphabetical order for reproducible builds
|
||||
ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
|
||||
ZSTDLIB_LOCAL_SRC := $(notdir $(ZSTDLIB_FULL_SRC))
|
||||
ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC))
|
||||
ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o)
|
||||
|
||||
ZSTD_CLI_SRC := $(wildcard *.c)
|
||||
ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
|
||||
|
||||
ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
|
||||
ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
|
||||
ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o)
|
||||
|
||||
UNAME := $(shell uname)
|
||||
@ -111,7 +111,7 @@ HASH ?= md5sum
|
||||
HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
|
||||
|
||||
ifndef BUILD_DIR
|
||||
HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ")
|
||||
HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDLIBS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ")
|
||||
ifeq ($(HAVE_HASH),0)
|
||||
$(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
|
||||
BUILD_DIR := obj/generic_noconf
|
||||
@ -199,7 +199,8 @@ SET_CACHE_DIRECTORY = \
|
||||
CPPFLAGS="$(CPPFLAGS)" \
|
||||
CFLAGS="$(CFLAGS)" \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
LDLIBS="$(LDLIBS)"
|
||||
LDLIBS="$(LDLIBS)" \
|
||||
ZSTD_ALL_SRC="$(ZSTD_ALL_SRC)"
|
||||
|
||||
|
||||
.PHONY: all
|
||||
@ -288,18 +289,12 @@ zstd-noxz : LZMALD :=
|
||||
zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
|
||||
zstd-noxz : zstd
|
||||
|
||||
## zstd-dll: zstd executable linked to dynamic library libzstd (must already exist)
|
||||
# note : the following target doesn't link
|
||||
# because zstd uses non-public symbols from libzstd
|
||||
# such as XXH64 (for benchmark),
|
||||
# ZDICT_trainFromBuffer_unsafe_legacy (for dictionary builder)
|
||||
# and ZSTD_cycleLog (likely for --patch-from).
|
||||
# It's unclear at this stage if this is a scenario that must be supported
|
||||
## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version)
|
||||
.PHONY: zstd-dll
|
||||
zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
|
||||
zstd-dll : ZSTDLIB_FULL_SRC =
|
||||
zstd-dll : $(ZSTD_CLI_OBJ)
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
|
||||
zstd-dll : LDFLAGS+= -L$(ZSTDDIR)
|
||||
zstd-dll : LDLIBS += -lzstd
|
||||
zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c
|
||||
zstd-dll : zstd
|
||||
|
||||
|
||||
## zstd-pgo: zstd executable optimized with PGO.
|
||||
@ -350,9 +345,11 @@ endif
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
|
||||
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
|
||||
zstd$(EXT) zstd32$(EXT) zstd-dll$(EXT) \
|
||||
zstd-compress$(EXT) zstd-decompress$(EXT) \
|
||||
zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \
|
||||
zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT)
|
||||
zstd-dictBuilder$(EXT) \
|
||||
*.gcda default*.profraw default.profdata have_zlib$(EXT)
|
||||
$(RM) -r obj/*
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
@ -255,18 +255,6 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si
|
||||
}
|
||||
|
||||
|
||||
/*! ZDICT_trainFromBuffer_unsafe_legacy() :
|
||||
Strictly Internal use only !!
|
||||
Same as ZDICT_trainFromBuffer_legacy(), but does not control `samplesBuffer`.
|
||||
`samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||
or an error code.
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_legacy_params_t parameters);
|
||||
|
||||
|
||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
||||
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
|
||||
@ -319,9 +307,9 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
{ size_t dictSize;
|
||||
if (params) {
|
||||
DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
||||
dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize,
|
||||
srcBuffer, sampleSizes, fs.nbSamples,
|
||||
*params);
|
||||
dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize,
|
||||
srcBuffer, sampleSizes, fs.nbSamples,
|
||||
*params);
|
||||
} else if (coverParams) {
|
||||
if (optimize) {
|
||||
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
|
||||
|
||||
@ -103,6 +103,24 @@ typedef enum { cover, fastCover, legacy } dictType;
|
||||
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Check Version (when CLI linked to dynamic library)
|
||||
**************************************/
|
||||
|
||||
/* Due to usage of experimental symbols and capabilities by the CLI,
|
||||
* the CLI must be linked against a dynamic library of same version */
|
||||
static void checkLibVersion(void)
|
||||
{
|
||||
if (strcmp(ZSTD_VERSION_STRING, ZSTD_versionString())) {
|
||||
DISPLAYLEVEL(1, "Error : incorrect library version (expecting : %s ; actual : %s ) \n",
|
||||
ZSTD_VERSION_STRING, ZSTD_versionString());
|
||||
DISPLAYLEVEL(1, "Please update library to version %s, or use stand-alone zstd binary \n",
|
||||
ZSTD_VERSION_STRING);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Command Line
|
||||
**************************************/
|
||||
@ -753,6 +771,7 @@ int main(int const argCount, const char* argv[])
|
||||
|
||||
|
||||
/* init */
|
||||
checkLibVersion();
|
||||
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
|
||||
(void)memLimit;
|
||||
assert(argCount >= 1);
|
||||
@ -1281,15 +1300,15 @@ int main(int const argCount, const char* argv[])
|
||||
DISPLAY("error : can't use --patch-from=# on multiple files \n");
|
||||
CLEAN_RETURN(1);
|
||||
}
|
||||
|
||||
/* No status message in pipe mode (stdin - stdout) */
|
||||
|
||||
/* No status message in pipe mode (stdin - stdout) */
|
||||
hasStdout = outFileName && !strcmp(outFileName,stdoutmark);
|
||||
|
||||
if (hasStdout && (g_displayLevel==2)) g_displayLevel=1;
|
||||
|
||||
/* IO Stream/File */
|
||||
FIO_setHasStdoutOutput(fCtx, hasStdout);
|
||||
FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize);
|
||||
FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize);
|
||||
FIO_determineHasStdinInput(fCtx, filenames);
|
||||
FIO_setNotificationLevel(g_displayLevel);
|
||||
FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user