sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-10 00:03:36 -04:00
Code Issues Projects Releases Wiki Activity

write summary for --list on multiple files

Browse Source
This commit is contained in:
Yann Collet 2017-09-26 11:21:36 -07:00
parent 8a4d0abc1c
commit 56f1f0e3dd
1 changed files with 71 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
programs/fileio.c
View File

@ -347,13 +347,16 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
fileHandle = fopen(fileName, "rb"); fileHandle = fopen(fileName, "rb");
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
fileSize = UTIL_getFileSize(fileName); fileSize = UTIL_getFileSize(fileName);
if (fileSize > DICTSIZE_MAX) if (fileSize > DICTSIZE_MAX) {
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */ fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
}
*bufferPtr = malloc((size_t)fileSize); *bufferPtr = malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); } if (readSize!=fileSize)
EXM_THROW(35, "Error reading dictionary file %s", fileName);
}
fclose(fileHandle); fclose(fileHandle);
return (size_t)fileSize; return (size_t)fileSize;
} }
@ -970,7 +973,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
char* dstFileName = (char*)malloc(FNSPACE); char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0; size_t const suffixSize = suffix ? strlen(suffix) : 0;
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]); int const isRegularFile = (nbFiles > 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]); /* won't write frame content size when nbFiles > 1 */
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams); cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
/* init */ /* init */
@ -1700,10 +1703,11 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
typedef struct { typedef struct {
int numActualFrames; int numActualFrames;
int numSkippableFrames; int numSkippableFrames;
unsigned long long decompressedSize; U64 decompressedSize;
int decompUnavailable; int decompUnavailable;
unsigned long long compressedSize; U64 compressedSize;
int usesCheck; int usesCheck;
U32 nbFiles;
} fileInfo_t; } fileInfo_t;
/** getFileInfo() : /** getFileInfo() :
@ -1720,14 +1724,16 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
DISPLAY("Error: could not open source file %s\n", inFileName); DISPLAY("Error: could not open source file %s\n", inFileName);
return 3; return 3;
} }
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName); info->compressedSize = UTIL_getFileSize(inFileName);
/* begin analyzing frame */ /* begin analyzing frame */
for ( ; ; ) { for ( ; ; ) {
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
if (numBytesRead < ZSTD_frameHeaderSize_min) { if (numBytesRead < ZSTD_frameHeaderSize_min) {
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) { if ( feof(srcFile)
&& (numBytesRead == 0)
&& (info->compressedSize > 0) ) {
break; break;
} }
else if (feof(srcFile)) { else if (feof(srcFile)) {
@ -1829,6 +1835,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
} }
} /* end analyzing frame */ } /* end analyzing frame */
fclose(srcFile); fclose(srcFile);
info->nbFiles = 1;
return detectError; return detectError;
} }
@ -1841,16 +1848,17 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
const char* const checkString = (info->usesCheck ? "XXH64" : "None"); const char* const checkString = (info->usesCheck ? "XXH64" : "None");
if (displayLevel <= 2) { if (displayLevel <= 2) {
if (!info->decompUnavailable) { if (!info->decompUnavailable) {
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n"); DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", info->numSkippableFrames + info->numActualFrames,
info->numSkippableFrames, info->numActualFrames, info->numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, inFileName); ratio, checkString, inFileName);
} else { } else {
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n"); DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n",
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n", info->numSkippableFrames + info->numActualFrames,
info->numSkippableFrames, info->numActualFrames, info->numSkippableFrames,
compressedSizeUnit, checkString, inFileName); compressedSizeUnit, unitStr,
checkString, inFileName);
} }
} else { } else {
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
@ -1867,33 +1875,40 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
} }
} }
static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
{
fileInfo_t total;
total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames;
total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames;
total.compressedSize = fi1.compressedSize + fi2.compressedSize;
total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize;
total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable;
total.usesCheck = fi1.usesCheck & fi2.usesCheck;
total.nbFiles = fi1.nbFiles + fi2.nbFiles;
return total;
}
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){ static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
/* initialize info to avoid warnings */ /* initialize info to avoid warnings */
fileInfo_t info; fileInfo_t info;
memset(&info, 0, sizeof(info)); memset(&info, 0, sizeof(info));
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles); { int const error = getFileInfo(&info, inFileName);
{
int const error = getFileInfo(&info, inFileName);
if (error == 1) { if (error == 1) {
/* display error, but provide output */ /* display error, but provide output */
DISPLAY("An error occurred with getting file info\n"); DISPLAY("An error occurred while getting file info \n");
} }
else if (error == 2) { else if (error == 2) {
DISPLAYOUT("File %s not compressed with zstd\n", inFileName); DISPLAYOUT("File %s not compressed by zstd \n", inFileName);
if (displayLevel > 2) { if (displayLevel > 2) DISPLAYOUT("\n");
DISPLAYOUT("\n");
}
return 1; return 1;
} }
else if (error == 3) { else if (error == 3) {
/* error occurred with opening the file */ /* error occurred while opening the file */
if (displayLevel > 2) { if (displayLevel > 2) DISPLAYOUT("\n");
DISPLAYOUT("\n");
}
return 1; return 1;
} }
displayInfo(inFileName, &info, displayLevel); displayInfo(inFileName, &info, displayLevel);
*total = FIO_addFInfo(*total, info);
return error; return error;
} }
} }
@ -1903,15 +1918,36 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
DISPLAYOUT("No files given\n"); DISPLAYOUT("No files given\n");
return 0; return 0;
} }
DISPLAYOUT("===========================================\n"); DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n");
DISPLAYOUT("Printing information about compressed files\n"); { int error = 0;
DISPLAYOUT("===========================================\n");
DISPLAYOUT("Number of files listed: %u\n", numFiles);
{
int error = 0;
unsigned u; unsigned u;
fileInfo_t total;
memset(&total, 0, sizeof(total));
total.usesCheck = 1;
for (u=0; u<numFiles;u++) { for (u=0; u<numFiles;u++) {
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles); error |= FIO_listFile(&total, filenameTable[u], displayLevel);
}
if (numFiles > 1) {
unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
double const compressedSizeUnit = (double)total.compressedSize / unit;
double const decompressedSizeUnit = (double)total.decompressedSize / unit;
double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize;
const char* const checkString = (total.usesCheck ? "XXH64" : "");
DISPLAYOUT("----------------------------------------------------------------- \n");
if (total.decompUnavailable) {
DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n",
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr,
checkString, total.nbFiles);
} else {
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n",
total.numSkippableFrames + total.numActualFrames,
total.numSkippableFrames,
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
ratio, checkString, total.nbFiles);
}
} }
return error; return error;
} }