sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #920 from facebook/benchSeparate

Browse Source 
Bench multiple files
This commit is contained in:
Yann Collet 2017-11-17 13:25:12 -08:00 committed by GitHub
commit c8b3e08535
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 67 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -130,11 +130,6 @@ uninstall:
travis-install: travis-install:
$(MAKE) install PREFIX=~/install_test_dir $(MAKE) install PREFIX=~/install_test_dir
.PHONY: gppbuild
gppbuild: clean
g++ -v
CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
.PHONY: gcc5build .PHONY: gcc5build
gcc5build: clean gcc5build: clean
gcc-5 -v gcc-5 -v

2
circle.yml
View File

@ -24,7 +24,7 @@ test:
: :
parallel: true parallel: true
- ? | - ? |
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gppbuild && make clean; fi && if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi &&
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi
: :
parallel: true parallel: true

66
programs/bench.c
View File

@ -129,6 +129,17 @@ void BMK_setNbThreads(unsigned nbThreads) {
#endif #endif
g_nbThreads = nbThreads; g_nbThreads = nbThreads;
} }
static U32 g_realTime = 0;
void BMK_setRealTime(unsigned priority) {
g_realTime = (priority>0);
}
static U32 g_separateFiles = 0;
void BMK_setSeparateFiles(unsigned separate) {
g_separateFiles = (separate>0);
}
static U32 g_ldmFlag = 0; static U32 g_ldmFlag = 0;
void BMK_setLdmFlag(unsigned ldmFlag) { void BMK_setLdmFlag(unsigned ldmFlag) {
g_ldmFlag = ldmFlag; g_ldmFlag = ldmFlag;
@ -476,11 +487,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem); return (size_t)(requiredMem);
} }
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast, const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize, const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *compressionParams, int setRealTimePrio) const ZSTD_compressionParameters* const compressionParams)
{ {
int l; int l;
@ -488,8 +499,8 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1; if (pch) displayName = pch+1;
if (setRealTimePrio) { if (g_realTime) {
DISPLAYLEVEL(2, "Note : switching to a real-time priority \n"); DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY; SET_REALTIME_PRIORITY;
} }
@ -512,7 +523,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
At most, fills `buffer` entirely */ At most, fills `buffer` entirely */
static void BMK_loadFiles(void* buffer, size_t bufferSize, static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes, size_t* fileSizes,
const char** fileNamesTable, unsigned nbFiles) const char* const * const fileNamesTable, unsigned nbFiles)
{ {
size_t pos = 0, totalSize = 0; size_t pos = 0, totalSize = 0;
unsigned n; unsigned n;
@ -544,16 +555,17 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench"); if (totalSize == 0) EXM_THROW(12, "no data to bench");
} }
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) const char* const dictFileName,
int const cLevel, int const cLevelLast,
const ZSTD_compressionParameters* const compressionParams)
{ {
void* srcBuffer; void* srcBuffer;
size_t benchedSize; size_t benchedSize;
void* dictBuffer = NULL; void* dictBuffer = NULL;
size_t dictBufferSize = 0; size_t dictBufferSize = 0;
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
@ -582,13 +594,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
/* Bench */ /* Bench */
if (g_separateFiles) {
const BYTE* srcPtr = (const BYTE*)srcBuffer;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
size_t const fileSize = fileSizes[fileNb];
BMK_benchCLevel(srcPtr, fileSize,
fileNamesTable[fileNb], cLevel, cLevelLast,
fileSizes+fileNb, 1,
dictBuffer, dictBufferSize, compressionParams);
srcPtr += fileSize;
}
} else {
char mfName[20] = {0};
snprintf (mfName, sizeof(mfName), " %u files", nbFiles); snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize, BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast, displayName, cLevel, cLevelLast,
fileSizes, nbFiles, fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams, setRealTimePrio); dictBuffer, dictBufferSize, compressionParams);
} } }
/* clean up */ /* clean up */
free(srcBuffer); free(srcBuffer);
@ -597,7 +622,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
} }
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, const ZSTD_compressionParameters* compressionParams)
{ {
char name[20] = {0}; char name[20] = {0};
size_t benchedSize = 10000000; size_t benchedSize = 10000000;
@ -611,15 +636,17 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */ /* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio); BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
/* clean up */ /* clean up */
free(srcBuffer); free(srcBuffer);
} }
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) const char* dictFileName,
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams)
{ {
double const compressibility = (double)g_compressibilityDefault / 100; double const compressibility = (double)g_compressibilityDefault / 100;
@ -627,11 +654,12 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (cLevelLast > cLevel)
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0) if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio); BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
else else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio); BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
return 0; return 0;
} }

6
programs/bench.h
View File

@ -16,14 +16,16 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio); int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams);
/* Set Parameters */ /* Set Parameters */
void BMK_setNbSeconds(unsigned nbLoops); void BMK_setNbSeconds(unsigned nbLoops);
void BMK_setBlockSize(size_t blockSize); void BMK_setBlockSize(size_t blockSize);
void BMK_setNbThreads(unsigned nbThreads); void BMK_setNbThreads(unsigned nbThreads);
void BMK_setRealTime(unsigned priority);
void BMK_setNotificationLevel(unsigned level); void BMK_setNotificationLevel(unsigned level);
void BMK_setSeparateFiles(unsigned separate);
void BMK_setAdditionalParam(int additionalParam); void BMK_setAdditionalParam(int additionalParam);
void BMK_setDecodeOnlyMode(unsigned decodeFlag); void BMK_setDecodeOnlyMode(unsigned decodeFlag);
void BMK_setLdmFlag(unsigned ldmFlag); void BMK_setLdmFlag(unsigned ldmFlag);

2
programs/util.h
View File

@ -336,7 +336,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
} }
UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
{ {
U64 total = 0; U64 total = 0;
int error = 0; int error = 0;

11
programs/zstdcli.c
View File

@ -377,6 +377,7 @@ int main(int argCount, const char* argv[])
lastCommand = 0, lastCommand = 0,
nbThreads = 1, nbThreads = 1,
setRealTimePrio = 0, setRealTimePrio = 0,
separateFiles = 0,
ldmFlag = 0; ldmFlag = 0;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0; size_t blockSize = 0;
@ -633,6 +634,12 @@ int main(int argCount, const char* argv[])
blockSize = readU32FromChar(&argument); blockSize = readU32FromChar(&argument);
break; break;
/* benchmark files separately (hidden option) */
case 'S':
argument++;
separateFiles = 1;
break;
#endif /* ZSTD_NOBENCH */ #endif /* ZSTD_NOBENCH */
/* nb of threads (hidden option) */ /* nb of threads (hidden option) */
@ -751,8 +758,10 @@ int main(int argCount, const char* argv[])
if (operation==zom_bench) { if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(g_displayLevel); BMK_setNotificationLevel(g_displayLevel);
BMK_setSeparateFiles(separateFiles);
BMK_setBlockSize(blockSize); BMK_setBlockSize(blockSize);
BMK_setNbThreads(nbThreads); BMK_setNbThreads(nbThreads);
BMK_setRealTime(setRealTimePrio);
BMK_setNbSeconds(bench_nbSeconds); BMK_setNbSeconds(bench_nbSeconds);
BMK_setLdmFlag(ldmFlag); BMK_setLdmFlag(ldmFlag);
BMK_setLdmMinMatch(g_ldmMinMatch); BMK_setLdmMinMatch(g_ldmMinMatch);
@ -763,7 +772,7 @@ int main(int argCount, const char* argv[])
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
BMK_setLdmHashEveryLog(g_ldmHashEveryLog); BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
} }
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio); BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
#endif #endif
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio;
goto _end; goto _end;