sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-29 00:05:38 -04:00
Code Issues Projects Releases Wiki Activity

reduced paramgrill dependency to C standard lib only

Browse Source
This commit is contained in:
Yann Collet 2016-07-13 16:45:24 +02:00
parent 2b1a3638e6
commit 158e7703bb
1 changed files with 67 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
programs/paramgrill.c
View File

@ -22,30 +22,16 @@
- zstd homepage : http://www.zstd.net/ - zstd homepage : http://www.zstd.net/
*/ */
/*-************************************
* Compiler Options
**************************************/
/* gettimeofday() are not supported by MSVC */
#if defined(_MSC_VER) || defined(_WIN32)
# define BMK_LEGACY_TIMER 1
#endif
/*-************************************ /*-************************************
* Dependencies * Dependencies
**************************************/ **************************************/
#include "util.h" /* Compiler options, UTIL_GetFileSize */ #include "util.h" /* Compiler options, UTIL_GetFileSize */
#include <stdlib.h> /* malloc */ #include <stdlib.h> /* malloc */
#include <stdio.h> /* fprintf, fopen, ftello64 */ #include <stdio.h> /* fprintf, fopen, ftello64 */
#include <string.h> /* strcmp */ #include <string.h> /* strcmp */
#include <math.h> /* log */ #include <math.h> /* log */
#include <time.h> /* clock_t */
/* Use ftime() if gettimeofday() is not available on your target */
#if defined(BMK_LEGACY_TIMER)
# include <sys/timeb.h> /* timeb, ftime */
#else
# include <sys/time.h> /* gettimeofday */
#endif
#include "mem.h" #include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */
@ -67,7 +53,7 @@
#define GB *(1ULL<<30) #define GB *(1ULL<<30)
#define NBLOOPS 2 #define NBLOOPS 2
#define TIMELOOP 2000 #define TIMELOOP (2 * CLOCKS_PER_SEC)
#define NB_LEVELS_TRACKED 30 #define NB_LEVELS_TRACKED 30
@ -76,9 +62,9 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t
#define COMPRESSIBILITY_DEFAULT 0.50 #define COMPRESSIBILITY_DEFAULT 0.50
static const size_t sampleSize = 10000000; static const size_t sampleSize = 10000000;
static const int g_grillDuration = 50000000; /* about 13 hours */ static const U32 g_grillDuration_s = 60000; /* about 16 hours */
static const int g_maxParamTime = 15000; /* 15 sec */ static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC;
static const int g_maxVariationTime = 60000; /* 60 sec */ static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC;
static const int g_maxNbVariations = 64; static const int g_maxNbVariations = 64;
@ -111,49 +97,15 @@ void BMK_SetNbIterations(int nbLoops)
* Private functions * Private functions
*********************************************************/ *********************************************************/
#if defined(BMK_LEGACY_TIMER) static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; } /* works even if overflow ; max span ~ 30 mn */
static int BMK_GetMilliStart(void) static U32 BMK_timeSpan(time_t tStart) { return (U32)difftime(time(NULL), tStart); } /* accuracy in seconds only, span can be multiple years */
{
/* Based on Legacy ftime()
* Rolls over every ~ 12.1 days (0x100000/24/60/60)
* Use GetMilliSpan to correct for rollover */
struct timeb tb;
int nCount;
ftime( &tb );
nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
return nCount;
}
#else
static int BMK_GetMilliStart(void)
{
/* Based on newer gettimeofday()
* Use GetMilliSpan to correct for rollover */
struct timeval tv;
int nCount;
gettimeofday(&tv, NULL);
nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
return nCount;
}
#endif
static int BMK_GetMilliSpan( int nTimeStart )
{
int nSpan = BMK_GetMilliStart() - nTimeStart;
if ( nSpan < 0 )
nSpan += 0x100000 * 1000;
return nSpan;
}
static size_t BMK_findMaxMem(U64 requiredMem) static size_t BMK_findMaxMem(U64 requiredMem)
{ {
size_t step = 64 MB; size_t const step = 64 MB;
BYTE* testmem=NULL; void* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26); requiredMem = (((requiredMem >> 26) + 1) << 26);
if (requiredMem > maxMemory) requiredMem = maxMemory; if (requiredMem > maxMemory) requiredMem = maxMemory;
@ -161,7 +113,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
requiredMem += 2*step; requiredMem += 2*step;
while (!testmem) { while (!testmem) {
requiredMem -= step; requiredMem -= step;
testmem = (BYTE*) malloc ((size_t)requiredMem); testmem = malloc ((size_t)requiredMem);
} }
free (testmem); free (testmem);
@ -188,8 +140,8 @@ U32 FUZ_rand(U32* src)
*********************************************************/ *********************************************************/
typedef struct { typedef struct {
size_t cSize; size_t cSize;
U32 cSpeed; double cSpeed;
U32 dSpeed; double dSpeed;
} BMK_result_t; } BMK_result_t;
typedef struct typedef struct
@ -265,36 +217,33 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.10, 1); RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.10, 1);
/* Bench */ /* Bench */
{ { U32 loopNb;
U32 loopNb;
size_t cSize = 0; size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.; double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.; double ratio = 0.;
U64 crcCheck = 0; U64 crcCheck = 0;
const int startTime =BMK_GetMilliStart(); time_t const benchStart = clock();
DISPLAY("\r%79s\r", ""); DISPLAY("\r%79s\r", "");
memset(&params, 0, sizeof(params)); memset(&params, 0, sizeof(params));
params.cParams = cParams; params.cParams = cParams;
params.fParams.contentSizeFlag = 0;
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
int nbLoops; int nbLoops;
int milliTime;
U32 blockNb; U32 blockNb;
const int totalTime = BMK_GetMilliSpan(startTime); clock_t roundStart, roundClock;
/* early break (slow params) */ { clock_t const benchTime = BMK_clockSpan(benchStart);
if (totalTime > g_maxParamTime) break; if (benchTime > g_maxParamTime) break; }
/* Compression */ /* Compression */
DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize); DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize); memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0; nbLoops = 0;
milliTime = BMK_GetMilliStart(); roundStart = clock();
while (BMK_GetMilliStart() == milliTime); while (clock() == roundStart);
milliTime = BMK_GetMilliStart(); roundStart = clock();
while (BMK_GetMilliSpan(milliTime) < TIMELOOP) { while (BMK_clockSpan(roundStart) < TIMELOOP) {
for (blockNb=0; blockNb<nbBlocks; blockNb++) for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx, blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
@ -303,40 +252,40 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
params); params);
nbLoops++; nbLoops++;
} }
milliTime = BMK_GetMilliSpan(milliTime); roundClock = BMK_clockSpan(roundStart);
cSize = 0; cSize = 0;
for (blockNb=0; blockNb<nbBlocks; blockNb++) for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize; cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops; if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
ratio = (double)srcSize / (double)cSize; ratio = (double)srcSize / (double)cSize;
DISPLAY("\r"); DISPLAY("\r");
DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize); DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize);
DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.); DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
resultPtr->cSize = cSize; resultPtr->cSize = cSize;
resultPtr->cSpeed = (U32)((double)srcSize / fastestC); resultPtr->cSpeed = (double)srcSize / fastestC;
#if 1 #if 1
/* Decompression */ /* Decompression */
memset(resultBuffer, 0xD6, srcSize); memset(resultBuffer, 0xD6, srcSize);
nbLoops = 0; nbLoops = 0;
milliTime = BMK_GetMilliStart(); roundStart = clock();
while (BMK_GetMilliStart() == milliTime); while (clock() == roundStart);
milliTime = BMK_GetMilliStart(); roundStart = clock();
for ( ; BMK_GetMilliSpan(milliTime) < TIMELOOP; nbLoops++) { for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) {
for (blockNb=0; blockNb<nbBlocks; blockNb++) for (blockNb=0; blockNb<nbBlocks; blockNb++)
blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize, blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
blockTable[blockNb].cPtr, blockTable[blockNb].cSize); blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
} }
milliTime = BMK_GetMilliSpan(milliTime); roundClock = BMK_clockSpan(roundStart);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; if ((double)roundClock < fastestD * CLOCKS_PER_SEC * nbLoops) fastestD = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops;
DISPLAY("\r"); DISPLAY("\r");
DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize); DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize);
DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000.); DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.);
DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000.); DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000000.);
resultPtr->dSpeed = (U32)((double)srcSize / fastestD); resultPtr->dSpeed = (double)srcSize / fastestD;
/* CRC Checking */ /* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0); crcCheck = XXH64(resultBuffer, srcSize, 0);
@ -378,11 +327,11 @@ static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compr
params.targetLength, g_stratName[(U32)(params.strategy)]); params.targetLength, g_stratName[(U32)(params.strategy)]);
fprintf(f, fprintf(f,
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", "/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
cLevel, (double)srcSize / result.cSize, (double)result.cSpeed / 1000., (double)result.dSpeed / 1000.); cLevel, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.);
} }
static U32 g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0 }; /* NB_LEVELS_TRACKED : checked at main() */ static double g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0. }; /* NB_LEVELS_TRACKED : checked at main() */
typedef struct { typedef struct {
BMK_result_t result; BMK_result_t result;
@ -447,11 +396,11 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log((double)testResult.cSpeed); double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed);
double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log((double)winners[cLevel].result.cSpeed); double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed);
double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log((double)testResult.dSpeed); double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed);
double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed); double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed);
if (W_DMemUsed_note < O_DMemUsed_note) { if (W_DMemUsed_note < O_DMemUsed_note) {
/* uses too much Decompression memory for too little benefit */ /* uses too much Decompression memory for too little benefit */
@ -473,16 +422,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
/* too large compression speed difference for the compression benefit */ /* too large compression speed difference for the compression benefit */
if (W_ratio > O_ratio) if (W_ratio > O_ratio)
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
W_ratio, (double)(testResult.cSpeed) / 1000., W_ratio, testResult.cSpeed / 1000000,
O_ratio, (double)(winners[cLevel].result.cSpeed) / 1000., cLevel); O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel);
continue; continue;
} }
if (W_DSpeed_note < O_DSpeed_note ) { if (W_DSpeed_note < O_DSpeed_note ) {
/* too large decompression speed difference for the compression benefit */ /* too large decompression speed difference for the compression benefit */
if (W_ratio > O_ratio) if (W_ratio > O_ratio)
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
W_ratio, (double)(testResult.dSpeed) / 1000., W_ratio, testResult.dSpeed / 1000000.,
O_ratio, (double)(winners[cLevel].result.dSpeed) / 1000., cLevel); O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel);
continue; continue;
} }
@ -578,9 +527,9 @@ static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_CCtx* ctx) ZSTD_CCtx* ctx)
{ {
int nbVariations = 0; int nbVariations = 0;
const int startTime = BMK_GetMilliStart(); clock_t const clockStart = clock();
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) { while (BMK_clockSpan(clockStart) < g_maxVariationTime) {
ZSTD_compressionParameters p = params; ZSTD_compressionParameters p = params;
if (nbVariations++ > g_maxNbVariations) break; if (nbVariations++ > g_maxNbVariations) break;
@ -641,11 +590,15 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_CCtx* const ctx = ZSTD_createCCtx();
ZSTD_compressionParameters params; ZSTD_compressionParameters params;
winnerInfo_t winners[NB_LEVELS_TRACKED]; winnerInfo_t winners[NB_LEVELS_TRACKED];
unsigned u; const char* const rfName = "grillResults.txt";
const char* rfName = "grillResults.txt";
FILE* const f = fopen(rfName, "w"); FILE* const f = fopen(rfName, "w");
const size_t blockSize = g_blockSize ? g_blockSize : srcSize; const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
/* init */
if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); }
memset(winners, 0, sizeof(winners));
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
if (g_singleRun) { if (g_singleRun) {
BMK_result_t testResult; BMK_result_t testResult;
g_params = ZSTD_adjustCParams(g_params, srcSize, 0); g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
@ -654,24 +607,21 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
return; return;
} }
/* init */
if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); }
memset(winners, 0, sizeof(winners));
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
if (g_target) if (g_target)
g_cSpeedTarget[1] = g_target * 1000; g_cSpeedTarget[1] = g_target * 1000000;
else { else {
/* baseline config for level 1 */ /* baseline config for level 1 */
BMK_result_t testResult; BMK_result_t testResult;
params = ZSTD_getCParams(1, blockSize, 0); params = ZSTD_getCParams(1, blockSize, 0);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5; g_cSpeedTarget[1] = (testResult.cSpeed * 31) / 32;
} }
/* establish speed objectives (relative to level 1) */ /* establish speed objectives (relative to level 1) */
for (u=2; u<=ZSTD_maxCLevel(); u++) { unsigned u;
g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) >> 5; for (u=2; u<=ZSTD_maxCLevel(); u++)
g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) / 32;
}
/* populate initial solution */ /* populate initial solution */
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
@ -683,10 +633,10 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
BMK_printWinners(f, winners, srcSize); BMK_printWinners(f, winners, srcSize);
/* start tests */ /* start tests */
{ const int milliStart = BMK_GetMilliStart(); { const time_t grillStart = time(NULL);
do { do {
BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx);
} while (BMK_GetMilliSpan(milliStart) < g_grillDuration); } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
} }
/* end summary */ /* end summary */
@ -843,7 +793,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
/* start tests */ /* start tests */
{ const int milliStart = BMK_GetMilliStart(); { time_t const grillStart = time(NULL);
do { do {
params = winner.params; params = winner.params;
paramVariation(&params); paramVariation(&params);
@ -865,7 +815,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed)
winner.result = candidate; winner.result = candidate;
BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize);
} }
} while (BMK_GetMilliSpan(milliStart) < g_grillDuration); } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
} }
/* end summary */ /* end summary */