sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

added some tests for correctness, time, and compression ratio

Browse Source
This commit is contained in:
Paul Cruz 2017-07-13 13:50:23 -07:00
parent 766663f1f1
commit 9165e97fc6
4 changed files with 381 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
contrib/adaptive-compression/Makefile
View File

@ -19,13 +19,24 @@ CFLAGS += $(DEBUGFLAGS)
CFLAGS += $(MOREFLAGS) CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
all: adapt all: adapt datagen
adapt: $(ZSTD_FILES) adapt.c adapt: $(ZSTD_FILES) adapt.c
$(CC) $(FLAGS) $^ -o $@ $(CC) $(FLAGS) $^ -o $@
datagen : $(PRGDIR)/datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
test-adapt-correctness: datagen adapt
@./test-correctness.sh
@echo "test correctness complete"
test-adapt-performance: datagen adapt
@./test-performance.sh
@echo "test performance complete"
clean: clean:
@$(RM) -f adapt @$(RM) -f adapt datagen
@$(RM) -rf *.dSYM @$(RM) -rf *.dSYM
@$(RM) -f tmp* @$(RM) -f tmp*
@$(RM) -f tests/*.zst @$(RM) -f tests/*.zst

129
contrib/adaptive-compression/datagencli.c Normal file
View File

@ -0,0 +1,129 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/*-************************************
* Dependencies
**************************************/
#include "util.h" /* Compiler options */
#include <stdio.h> /* fprintf, stderr */
#include "datagen.h" /* RDG_generate */
/*-************************************
* Constants
**************************************/
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define SIZE_DEFAULT ((64 KB) + 1)
#define SEED_DEFAULT 0
#define COMPRESSIBILITY_DEFAULT 50
/*-************************************
* Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned displayLevel = 2;
/*-*******************************************************
* Command line
*********************************************************/
static int usage(const char* programName)
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n",
COMPRESSIBILITY_DEFAULT);
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, const char** argv)
{
unsigned probaU32 = COMPRESSIBILITY_DEFAULT;
double litProba = 0.0;
U64 size = SIZE_DEFAULT;
U32 seed = SEED_DEFAULT;
const char* const programName = argv[0];
int argNb;
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Handle commands. Aggregated commands are allowed */
if (*argument=='-') {
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return usage(programName);
case 'g':
argument++;
size=0;
while ((*argument>='0') && (*argument<='9'))
size *= 10, size += *argument++ - '0';
if (*argument=='K') { size <<= 10; argument++; }
if (*argument=='M') { size <<= 20; argument++; }
if (*argument=='G') { size <<= 30; argument++; }
if (*argument=='B') { argument++; }
break;
case 's':
argument++;
seed=0;
while ((*argument>='0') && (*argument<='9'))
seed *= 10, seed += *argument++ - '0';
break;
case 'P':
argument++;
probaU32 = 0;
while ((*argument>='0') && (*argument<='9'))
probaU32 *= 10, probaU32 += *argument++ - '0';
if (probaU32>100) probaU32 = 100;
break;
case 'L': /* hidden argument : Literal distribution probability */
argument++;
litProba=0.;
while ((*argument>='0') && (*argument<='9'))
litProba *= 10, litProba += *argument++ - '0';
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default:
return usage(programName);
}
} } } /* for(argNb=1; argNb<argc; argNb++) */
DISPLAYLEVEL(4, "Compressible data Generator \n");
if (probaU32!=COMPRESSIBILITY_DEFAULT)
DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
RDG_genStdout(size, (double)probaU32/100, litProba, seed);
DISPLAYLEVEL(1, "\n");
return 0;
}

205
contrib/adaptive-compression/test-correctness.sh Executable file
View File

@ -0,0 +1,205 @@
echo "correctness tests -- general"
./datagen -g1GB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g500MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g250MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g125MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g50MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g25MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g5MB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g500KB > tmp
./adapt -otmp.zst tmp
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
echo -e "\ncorrectness tests -- streaming"
./datagen -g1GB > tmp
cat tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100MB > tmp
cat tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10MB > tmp
cat tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g1MB > tmp
cat tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100KB > tmp
cat tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10KB > tmp
cat tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
echo -e "\ncorrectness tests -- read limit"
./datagen -g1GB > tmp
pv -L 50m -q tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100MB > tmp
pv -L 50m -q tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10MB > tmp
pv -L 50m -q tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g1MB > tmp
pv -L 50m -q tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100KB > tmp
pv -L 50m -q tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10KB > tmp
pv -L 50m -q tmp | ./adapt > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
echo -e "\ncorrectness tests -- write limit"
./datagen -g1GB > tmp
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100MB > tmp
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10MB > tmp
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g1MB > tmp
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100KB > tmp
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10KB > tmp
pv -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
echo -e "\ncorrectness tests -- read and write limits"
./datagen -g1GB > tmp
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100MB > tmp
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10MB > tmp
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g1MB > tmp
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g100KB > tmp
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
./datagen -g10KB > tmp
pv -L 50m -q tmp | ./adapt | pv -L 5m -q > tmp.zst
zstd -d tmp.zst -o tmp2
diff -q tmp tmp2
rm tmp*
make clean

34
contrib/adaptive-compression/test-performance.sh Executable file
View File

@ -0,0 +1,34 @@
echo "testing time"
./datagen -g1GB > tmp
time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp
rm tmp*
./datagen -g2GB > tmp
time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp
rm tmp*
./datagen -g4GB > tmp
time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp
rm tmp*
echo -e "\ntesting compression ratio"
./datagen -g1GB > tmp
time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp
ls -l tmp1.zst tmp2.zst
rm tmp*
./datagen -g2GB > tmp
time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp
ls -l tmp1.zst tmp2.zst
rm tmp*
./datagen -g4GB > tmp
time ./adapt -otmp1.zst tmp
time zstd -1 -o tmp2.zst tmp
ls -l tmp1.zst tmp2.zst
rm tmp*