sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-19 00:05:29 -04:00
Code Issues Projects Releases Wiki Activity

Remove clevel and update documentation

Browse Source
This commit is contained in:
Jennifer Liu 2018-07-13 17:38:53 -07:00
parent a23a3b95f9
commit 31731df4da
3 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
contrib/randomDictBuilder/README.md
View File

@ -1,12 +1,17 @@
Random Dictionary Builder Random Dictionary Builder
### Permitted Arguments: ### Permitted Arguments:
Input Files (in=fileName): files used to build dictionary, can include multiple files, each following "in=", required Input File/Directory (in=fileName): required; file/directory used to build dictionary; if directory, will operate recursively for files inside directory; can include multiple files/directories, each following "in="
Output Dictionary (out=dictName): if not provided, default to defaultDict Output Dictionary (out=dictName): if not provided, default to defaultDict
Dictionary ID (dictID=#): positive number, if not provided, default to 0 Dictionary ID (dictID=#): nonnegative number; if not provided, default to 0
Maximum Dictionary Size (maxdict=#): positive number, in bytes, if not provided, default to 110KB Maximum Dictionary Size (maxdict=#): positive number; in bytes, if not provided, default to 110KB
Size of Randomly Selected Segment (k=#): positive number, in bytes, if not provided, default to 200 Size of Randomly Selected Segment (k=#): positive number; in bytes; if not provided, default to 200
Compression Level (c=#): positive number, if not provided, default to 3 Compression Level (c=#): positive number; if not provided, default to 3
###Usage:
To build a random dictionary with the provided arguments: make run ARG= followed by arguments
### Examples: ### Examples:
make run ARG="in=../../lib/dictBuilder out=dict100 dictID=520" make run ARG="in=../../lib/dictBuilder out=dict100 dictID=520"

11
contrib/randomDictBuilder/main.c
View File

@ -63,7 +63,7 @@ int main(int argCount, const char* argv[])
const char* programName = argv[0]; const char* programName = argv[0];
int operationResult = 0; int operationResult = 0;
unsigned cLevel = DEFAULT_CLEVEL; /* Initialize parameters with default value */
char* inputFile = DEFAULT_INPUTFILE; char* inputFile = DEFAULT_INPUTFILE;
unsigned k = DEFAULT_k; unsigned k = DEFAULT_k;
char* outputFile = DEFAULT_OUTPUTFILE; char* outputFile = DEFAULT_OUTPUTFILE;
@ -76,10 +76,10 @@ int main(int argCount, const char* argv[])
for (int i = 1; i < argCount; i++) { for (int i = 1; i < argCount; i++) {
const char* argument = argv[i]; const char* argument = argv[i];
if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "k=")) { k = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "c=")) { cLevel = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "dictID=")) { dictID = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "in=")) { if (longCommandWArg(&argument, "in=")) {
/* Allow multiple input files */
inputFile = malloc(strlen(argument) + 1); inputFile = malloc(strlen(argument) + 1);
strcpy(inputFile, argument); strcpy(inputFile, argument);
filenameTable[filenameIdx] = inputFile; filenameTable[filenameIdx] = inputFile;
@ -96,6 +96,11 @@ int main(int argCount, const char* argv[])
return operationResult; return operationResult;
} }
if (maxDictSize == 0) {
DISPLAYLEVEL(1, "maxDictSize should not be 0.\n");
operationResult = 1;
return operationResult;
}
char* fileNamesBuf = NULL; char* fileNamesBuf = NULL;
unsigned fileNamesNb = filenameIdx; unsigned fileNamesNb = filenameIdx;
@ -114,7 +119,7 @@ int main(int argCount, const char* argv[])
ZDICT_random_params_t params; ZDICT_random_params_t params;
ZDICT_params_t zParams; ZDICT_params_t zParams;
zParams.compressionLevel = cLevel; zParams.compressionLevel = DEFAULT_CLEVEL;
zParams.notificationLevel = displayLevel; zParams.notificationLevel = displayLevel;
zParams.dictID = dictID; zParams.dictID = dictID;
params.zParams = zParams; params.zParams = zParams;

8
contrib/randomDictBuilder/test.sh
View File

@ -1,8 +1,8 @@
echo "Building random dictionary with c=5 in=../../lib/common k=200 out=dict1" echo "Building random dictionary with in=../../lib/common k=200 out=dict1"
./main c=5 in=../../lib/common k=200 out=dict1 ./main in=../../lib/common k=200 out=dict1
zstd -be3 -D dict1 -r ../../lib/common -q zstd -be3 -D dict1 -r ../../lib/common -q
echo "Building random dictionary with c=9 in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000" echo "Building random dictionary with in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000"
./main c=9 in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000 ./main in=../../lib/common k=500 out=dict2 dictID=100 maxdict=140000
zstd -be3 -D dict2 -r ../../lib/common -q zstd -be3 -D dict2 -r ../../lib/common -q
echo "Building random dictionary with 2 sample sources" echo "Building random dictionary with 2 sample sources"
./main in=../../lib/common in=../../lib/compress out=dict3 ./main in=../../lib/common in=../../lib/compress out=dict3