mirror of
https://github.com/facebook/zstd.git
synced 2025-10-08 00:04:02 -04:00
Yann Collet
commit
3f73a27655
36
lib/zbuff.c
36
lib/zbuff.c
@ -136,7 +136,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize);
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
|
||||
}
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
@ -222,7 +222,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
/* prepare next block */
|
||||
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
|
||||
if (zbc->inBuffTarget > zbc->inBuffSize)
|
||||
{ zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; } /* note : inBuffSize >= blockSize */
|
||||
zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */
|
||||
zbc->inToCompress = zbc->inBuffPos;
|
||||
if (cDst == op) { op += cSize; break; } /* no need to flush */
|
||||
zbc->outBuffContentSize = cSize;
|
||||
@ -326,10 +326,11 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
|
||||
struct ZBUFF_DCtx_s {
|
||||
ZSTD_DCtx* zc;
|
||||
ZSTD_frameParams fParams;
|
||||
char* inBuff;
|
||||
size_t blockSize;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
char* outBuff;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
@ -405,20 +406,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
} }
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */
|
||||
if (zbc->inBuffSize < neededInSize) {
|
||||
{ size_t const blockSize = MIN(1 << zbc->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
|
||||
zbc->blockSize = blockSize;
|
||||
if (zbc->inBuffSize < blockSize) {
|
||||
free(zbc->inBuff);
|
||||
zbc->inBuffSize = neededInSize;
|
||||
zbc->inBuff = (char*)malloc(neededInSize);
|
||||
zbc->inBuffSize = blockSize;
|
||||
zbc->inBuff = (char*)malloc(blockSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
{ size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog;
|
||||
if (zbc->outBuffSize < neededOutSize) {
|
||||
free(zbc->outBuff);
|
||||
zbc->outBuffSize = neededOutSize;
|
||||
zbc->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
}
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbc->fParams.windowLog) + blockSize;
|
||||
if (zbc->outBuffSize < neededOutSize) {
|
||||
free(zbc->outBuff);
|
||||
zbc->outBuffSize = neededOutSize;
|
||||
zbc->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} } }
|
||||
zbc->stage = ZBUFFds_read;
|
||||
|
||||
case ZBUFFds_read:
|
||||
@ -471,7 +473,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
zbc->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) {
|
||||
zbc->stage = ZBUFFds_read;
|
||||
if (zbc->outStart + ZSTD_BLOCKSIZE_MAX > zbc->outBuffSize)
|
||||
if (zbc->outStart + zbc->blockSize > zbc->outBuffSize)
|
||||
zbc->outStart = zbc->outEnd = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
@ -164,8 +164,8 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
|
||||
if (srcSize > (1ULL << ZSTD_WINDOWLOG_MIN)) return ZSTD_checkCParams(cParams);
|
||||
if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported);
|
||||
if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */
|
||||
if (srcSize <= (1ULL << cParams.hashLog)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */
|
||||
if (srcSize <= (1ULL << cParams.contentLog)) cParams.contentLog = ZSTD_CONTENTLOG_MIN; /* fake value - temporary work around */
|
||||
if ((srcSize <= (1ULL << cParams.hashLog)) && ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */
|
||||
return ZSTD_checkCParams(cParams);
|
||||
}
|
||||
|
||||
@ -193,6 +193,7 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t
|
||||
if (params->contentLog > maxContentLog) params->contentLog = maxContentLog; } /* <= ZSTD_CONTENTLOG_MAX */
|
||||
|
||||
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||
if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -686,18 +686,18 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
|
||||
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
BYTE* const oend_8 = oend-8;
|
||||
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* match = oLitEnd - sequence.offset;
|
||||
|
||||
/* check */
|
||||
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
||||
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
*litPtr = iLitEnd; /* update for next sequence */
|
||||
|
||||
/* copy Match */
|
||||
if (sequence.offset > (size_t)(oLitEnd - base)) {
|
||||
@ -761,7 +761,6 @@ static size_t ZSTD_decompressSequences(
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
int nbSeq;
|
||||
U32* DTableLL = dctx->LLTable;
|
||||
U32* DTableML = dctx->MLTable;
|
||||
U32* DTableOffb = dctx->OffTable;
|
||||
@ -769,13 +768,14 @@ static size_t ZSTD_decompressSequences(
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
const U32 mls = dctx->fParams.mml;
|
||||
int nbSeq;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
{ size_t const errorCode = ZSTD_decodeSeqHeaders(&nbSeq,
|
||||
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq,
|
||||
DTableLL, DTableML, DTableOffb,
|
||||
ip, seqSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
ip += errorCode;
|
||||
if (ZSTD_isError(seqHSize)) return seqHSize;
|
||||
ip += seqHSize;
|
||||
}
|
||||
|
||||
/* Regen sequences */
|
||||
|
||||
519
programs/result
519
programs/result
@ -1,519 +0,0 @@
|
||||
/*
|
||||
bench.c - Demo module to benchmark open-source compression algorithms
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/* **************************************
|
||||
* Compiler Options
|
||||
****************************************/
|
||||
/* Disable some Visual warning messages */
|
||||
#ifdef _MSC_VER
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fopen */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
/* Unix Large Files support (>4GB) */
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
|
||||
# define _LARGEFILE_SOURCE
|
||||
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
|
||||
# define _LARGEFILE64_SOURCE
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <sys/types.h> /* stat64 */
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
/* sleep : posix - windows - others */
|
||||
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
# include <unistd.h> /* sleep */
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# define BMK_sleep(s) sleep(s)
|
||||
# define HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
#elif defined(_WIN32)
|
||||
# include <windows.h>
|
||||
# define BMK_sleep(s) Sleep(1000*s)
|
||||
# define HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS);
|
||||
#else
|
||||
# define BMK_sleep(s) /* disabled */
|
||||
# define HIGH_PRIORITY
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd_static.h"
|
||||
#include "xxhash.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Compiler specifics
|
||||
***************************************/
|
||||
#if !defined(S_ISREG)
|
||||
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define snprintf sprintf_s
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define NBLOOPS 3
|
||||
#define TIMELOOP_S 1
|
||||
#define ACTIVEPERIOD_S 70
|
||||
#define COOLPERIOD_S 10
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
#define DEFAULT_CHUNKSIZE (4 MB)
|
||||
|
||||
static U32 g_compressibilityDefault = 50;
|
||||
|
||||
|
||||
/* *************************************
|
||||
* console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, "\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Benchmark Parameters
|
||||
***************************************/
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static size_t g_blockSize = 0;
|
||||
|
||||
void BMK_SetNbIterations(unsigned nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
void BMK_SetBlockSize(size_t blockSize)
|
||||
{
|
||||
g_blockSize = blockSize;
|
||||
DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10));
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Private functions
|
||||
**********************************************************/
|
||||
static clock_t BMK_clockSpan( clock_t clockStart )
|
||||
{
|
||||
return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */
|
||||
}
|
||||
|
||||
|
||||
static U64 BMK_getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
#endif
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Bench functions
|
||||
**********************************************************/
|
||||
typedef struct
|
||||
{
|
||||
const char* srcPtr;
|
||||
size_t srcSize;
|
||||
char* cPtr;
|
||||
size_t cRoom;
|
||||
size_t cSize;
|
||||
char* resPtr;
|
||||
size_t resSize;
|
||||
} blockParam_t;
|
||||
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, U32 nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
|
||||
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
ZSTD_CCtx* refCtx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
U32 nbBlocks;
|
||||
|
||||
/* checks */
|
||||
if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx)
|
||||
EXM_THROW(31, "not enough memory");
|
||||
|
||||
/* init */
|
||||
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
|
||||
HIGH_PRIORITY;
|
||||
|
||||
/* Init blockTable data */
|
||||
{ const char* srcPtr = (const char*)srcBuffer;
|
||||
char* cPtr = (char*)compressedBuffer;
|
||||
char* resPtr = (char*)resultBuffer;
|
||||
U32 fileNb;
|
||||
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||
size_t remaining = fileSizes[fileNb];
|
||||
U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
|
||||
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
|
||||
for ( ; nbBlocks<blockEnd; nbBlocks++) {
|
||||
size_t const thisBlockSize = MIN(remaining, blockSize);
|
||||
blockTable[nbBlocks].srcPtr = srcPtr;
|
||||
blockTable[nbBlocks].cPtr = cPtr;
|
||||
blockTable[nbBlocks].resPtr = resPtr;
|
||||
blockTable[nbBlocks].srcSize = thisBlockSize;
|
||||
blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
|
||||
srcPtr += thisBlockSize;
|
||||
cPtr += blockTable[nbBlocks].cRoom;
|
||||
resPtr += thisBlockSize;
|
||||
remaining -= thisBlockSize;
|
||||
} } }
|
||||
|
||||
/* warmimg up memory */
|
||||
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
|
||||
|
||||
/* Bench */
|
||||
{ double fastestC = 100000000., fastestD = 100000000.;
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
clock_t coolTime = clock();
|
||||
U32 testNb;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
|
||||
size_t cSize;
|
||||
double ratio = 0.;
|
||||
clock_t clockStart;
|
||||
clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10;
|
||||
|
||||
/* overheat protection */
|
||||
if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) {
|
||||
DISPLAY("\rcooling down ... \r");
|
||||
BMK_sleep(COOLPERIOD_S);
|
||||
coolTime = clock();
|
||||
}
|
||||
|
||||
/* Compression */
|
||||
DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
|
||||
memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
} }
|
||||
{ clock_t const clockSpan = BMK_clockSpan(clockStart);
|
||||
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) );
|
||||
|
||||
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
#if 1
|
||||
/* Decompression */
|
||||
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
|
||||
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
if (ZSTD_isError(regenSize)) {
|
||||
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
|
||||
blockNb, ZSTD_getErrorName(regenSize));
|
||||
clockStart -= clockLoop+1; /* force immediate test end */
|
||||
break;
|
||||
}
|
||||
blockTable[blockNb].resSize = regenSize;
|
||||
} }
|
||||
{ clock_t const clockSpan = BMK_clockSpan(clockStart);
|
||||
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC),
|
||||
(double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) );
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
|
||||
U32 segNb, bNb, pos;
|
||||
size_t bacc = 0;
|
||||
DISPLAY("Decoding error at pos %u ", (U32)u);
|
||||
for (segNb = 0; segNb < nbBlocks; segNb++) {
|
||||
if (bacc + blockTable[segNb].srcSize > u) break;
|
||||
bacc += blockTable[segNb].srcSize;
|
||||
}
|
||||
pos = (U32)(u - bacc);
|
||||
bNb = pos / (128 KB);
|
||||
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
|
||||
break;
|
||||
}
|
||||
if (u==srcSize-1) { /* should never happen */
|
||||
DISPLAY("no difference detected\n");
|
||||
} }
|
||||
break;
|
||||
} } /* CRC Checking */
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
|
||||
DISPLAY("%2i#\n", cLevel);
|
||||
} /* Bench */
|
||||
|
||||
/* clean up */
|
||||
free(compressedBuffer);
|
||||
free(resultBuffer);
|
||||
ZSTD_freeCCtx(refCtx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(refDCtx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t const step = 64 MB;
|
||||
BYTE* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
requiredMem += step;
|
||||
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
||||
|
||||
do {
|
||||
testmem = (BYTE*)malloc((size_t)requiredMem);
|
||||
requiredMem -= step;
|
||||
} while (!testmem);
|
||||
|
||||
free(testmem);
|
||||
return (size_t)(requiredMem);
|
||||
}
|
||||
|
||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
if (cLevel < 0) { /* range mode : test all levels from 1 to l */
|
||||
int l;
|
||||
for (l=1; l <= -cLevel; l++) {
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, l,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
}
|
||||
return;
|
||||
}
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, cLevel,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
}
|
||||
|
||||
static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++)
|
||||
total += BMK_getFileSize(fileNamesTable[n]);
|
||||
return total;
|
||||
}
|
||||
|
||||
/*! BMK_loadFiles() :
|
||||
Loads `buffer` with content of files listed within `fileNamesTable`.
|
||||
At most, fills `buffer` entirely */
|
||||
static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0;
|
||||
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
|
||||
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize; }
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
}
|
||||
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel)
|
||||
{
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
void* dictBuffer = NULL;
|
||||
size_t dictBufferSize = 0;
|
||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
char mfName[20] = {0};
|
||||
const char* displayName = NULL;
|
||||
|
||||
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
|
||||
|
||||
/* Load dictionary */
|
||||
if (dictFileName != NULL) {
|
||||
U64 dictFileSize = BMK_getFileSize(dictFileName);
|
||||
if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
|
||||
dictBufferSize = (size_t)dictFileSize;
|
||||
dictBuffer = malloc(dictBufferSize);
|
||||
if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
|
||||
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
|
||||
}
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
|
||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||
if (benchedSize < totalSizeToLoad)
|
||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||
srcBuffer = malloc(benchedSize);
|
||||
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
||||
|
||||
/* Load input buffer */
|
||||
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
|
||||
/* Bench */
|
||||
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
|
||||
if (nbFiles > 1) displayName = mfName;
|
||||
else displayName = fileNamesTable[0];
|
||||
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
free(dictBuffer);
|
||||
free(fileSizes);
|
||||
}
|
||||
|
||||
|
||||
static void BMK_syntheticTest(int cLevel, double compressibility)
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
void* srcBuffer = malloc(benchedSize);
|
||||
|
||||
/* Memory allocation */
|
||||
if (!srcBuffer) EXM_THROW(21, "not enough memory");
|
||||
|
||||
/* Fill input buffer */
|
||||
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
|
||||
|
||||
/* Bench */
|
||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
}
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, compressibility);
|
||||
else
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel);
|
||||
return 0;
|
||||
}
|
||||
|
||||
BIN
programs/tmp.mgz
BIN
programs/tmp.mgz
Binary file not shown.
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user