From 26fab1d963b70b5673e7a7405735818f7b6e30bd Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 20:25:31 -0700 Subject: [PATCH 01/26] Make the CLI output the file sizes in human readable format --- programs/fileio.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a67c0fbfb..9d3054b1c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1527,6 +1527,26 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, return compressedfilesize; } +char* human_size(long size, char* str) { + if (size > 1125899906842624L) { + snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); + } else if (size > 1099511627776L) { + snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); + } else if (size > 1073741824L) { + snprintf(str, 7, "%.1fG", (float)size / 1073741824L); + } else if (size > 1048576L) { + snprintf(str, 7, "%.1fM", (float)size / 1048576L); + } else if (size > 1024) { + snprintf(str, 7, "%.1fK", (float)size / 1024); + } else if (size >= 0) { + snprintf(str, 7, "%dB", size); + } else { + str[0] = '\0'; + } + + return str; +} + /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, @@ -1598,10 +1618,16 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + char input_size_str[8] = ""; + human_size((unsigned long long) readsize, input_size_str); + + char output_size_str[8] = ""; + human_size((unsigned long long) compressedfilesize, output_size_str); + + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%s => %s, %s) \n", srcFileName, (double)compressedfilesize / (double)readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, + input_size_str, output_size_str, dstFileName); } } From b70175e5ec51294115f311db3d77c104b557a14e Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 20:28:55 -0700 Subject: [PATCH 02/26] Put the human_size() function in util.c --- programs/fileio.c | 20 -------------------- programs/util.c | 21 +++++++++++++++++++++ programs/util.h | 2 ++ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 9d3054b1c..b29295586 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1527,26 +1527,6 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, return compressedfilesize; } -char* human_size(long size, char* str) { - if (size > 1125899906842624L) { - snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); - } else if (size > 1099511627776L) { - snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); - } else if (size > 1073741824L) { - snprintf(str, 7, "%.1fG", (float)size / 1073741824L); - } else if (size > 1048576L) { - snprintf(str, 7, "%.1fM", (float)size / 1048576L); - } else if (size > 1024) { - snprintf(str, 7, "%.1fK", (float)size / 1024); - } else if (size >= 0) { - snprintf(str, 7, "%dB", size); - } else { - str[0] = '\0'; - } - - return str; -} - /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, diff --git a/programs/util.c b/programs/util.c index 8d190c62c..d942d3732 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,6 +121,27 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ +char* human_size(long size, char* str) { + if (size > 1125899906842624L) { + snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); + } else if (size > 1099511627776L) { + snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); + } else if (size > 1073741824L) { + snprintf(str, 7, "%.1fG", (float)size / 1073741824L); + } else if (size > 1048576L) { + snprintf(str, 7, "%.1fM", (float)size / 1048576L); + } else if (size > 1024) { + snprintf(str, 7, "%.1fK", (float)size / 1024); + } else if (size >= 0) { + snprintf(str, 7, "%dB", size); + } else { + str[0] = '\0'; + } + + return str; +} + + int UTIL_stat(const char* filename, stat_t* statbuf) { #if defined(_MSC_VER) diff --git a/programs/util.h b/programs/util.h index 24cce4480..415b01f1b 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,6 +122,8 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif +char* human_size(long size, char* str); + /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. * Returns success (1) or failure (0). From b6b23dfe64e2ff52e52d2eeed7e68ebc70249178 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 21:44:40 -0700 Subject: [PATCH 03/26] Convert names to CamelCase --- programs/fileio.c | 10 +++++----- programs/util.c | 2 +- programs/util.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b29295586..b5beb145c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1598,16 +1598,16 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); } else { - char input_size_str[8] = ""; - human_size((unsigned long long) readsize, input_size_str); + char inputSizeStr[8] = ""; + humanSize((unsigned long long) readsize, inputSizeStr); - char output_size_str[8] = ""; - human_size((unsigned long long) compressedfilesize, output_size_str); + char outputSizeStr[8] = ""; + humanSize((unsigned long long) compressedfilesize, outputSizeStr); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%s => %s, %s) \n", srcFileName, (double)compressedfilesize / (double)readsize * 100, - input_size_str, output_size_str, + inputSizeStr, outputSizeStr, dstFileName); } } diff --git a/programs/util.c b/programs/util.c index d942d3732..5aec11184 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ -char* human_size(long size, char* str) { +char* humanSize(long size, char* str) { if (size > 1125899906842624L) { snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); } else if (size > 1099511627776L) { diff --git a/programs/util.h b/programs/util.h index 415b01f1b..aaaf2522c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,7 +122,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif -char* human_size(long size, char* str); +char* humanSize(long size, char* str); /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. From eefdbcd93aa296df9a72ba75a53201bb10a3a2ae Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 22:02:32 -0700 Subject: [PATCH 04/26] Make the variable types match --- programs/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 5aec11184..6d04b8fb7 100644 --- a/programs/util.c +++ b/programs/util.c @@ -133,7 +133,7 @@ char* humanSize(long size, char* str) { } else if (size > 1024) { snprintf(str, 7, "%.1fK", (float)size / 1024); } else if (size >= 0) { - snprintf(str, 7, "%dB", size); + snprintf(str, 7, "%dB", 0); } else { str[0] = '\0'; } From 4e0d9f1cc83a0795a2eb8401dc92369a899614c1 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 22:03:33 -0700 Subject: [PATCH 05/26] Move the variable declarations to the top --- programs/fileio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b5beb145c..b347da423 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1544,6 +1544,9 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, U64 readsize = 0; U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); + char inputSizeStr[8] = ""; + char outputSizeStr[8] = ""; + DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); /* compression format selection */ @@ -1598,10 +1601,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); } else { - char inputSizeStr[8] = ""; humanSize((unsigned long long) readsize, inputSizeStr); - - char outputSizeStr[8] = ""; humanSize((unsigned long long) compressedfilesize, outputSizeStr); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%s => %s, %s) \n", From 894698d3b61583fb539f355584db576b7e0635c5 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 22:09:40 -0700 Subject: [PATCH 06/26] Use human_size() in the benchmark output also --- programs/benchzstd.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 49c03490d..954c9282d 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -388,6 +388,8 @@ BMK_benchMemAdvancedNoAlloc( # define NB_MARKS 4 const char* marks[NB_MARKS] = { " |", " /", " =", " \\" }; U32 markNb = 0; + char inputSizeStr[8] = ""; + char outputSizeStr[8] = ""; int compressionCompleted = (adv->mode == BMK_decodeOnly); int decompressionCompleted = (adv->mode == BMK_compressOnly); BMK_benchParams_t cbp, dbp; @@ -429,8 +431,10 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; + humanSize((unsigned)srcSize, inputSizeStr); + DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize); + DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> \r", marks[markNb], displayName, inputSizeStr); while (!(compressionCompleted && decompressionCompleted)) { if (!compressionCompleted) { @@ -451,9 +455,13 @@ BMK_benchMemAdvancedNoAlloc( } } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", + + humanSize((unsigned)srcSize, inputSizeStr); + humanSize((unsigned)cSize, outputSizeStr); + + DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, - (unsigned)srcSize, (unsigned)cSize, + inputSizeStr, outputSizeStr, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); } @@ -474,9 +482,13 @@ BMK_benchMemAdvancedNoAlloc( } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + + humanSize((unsigned)srcSize, inputSizeStr); + humanSize((unsigned)cSize, outputSizeStr); + + DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, - (unsigned)srcSize, (unsigned)cSize, + inputSizeStr, outputSizeStr, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, (double)benchResult.dSpeed / MB_UNIT); From 77001f00fb30ee63f10c96b89b4d6ccfb00078b4 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 4 Jun 2021 22:21:00 -0700 Subject: [PATCH 07/26] Use human_size() on the "multiple files compressed" output also --- programs/fileio.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b347da423..7432348c0 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1841,6 +1841,8 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, { int status; int error = 0; + char inputSizeStr[8] = ""; + char outputSizeStr[8] = ""; cRess_t ress = FIO_createCResources(prefs, dictFileName, FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), compressionLevel, comprParams); @@ -1896,10 +1898,13 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, } if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { + humanSize((unsigned long long) fCtx->totalBytesInput, inputSizeStr); + humanSize((unsigned long long) fCtx->totalBytesOutput, outputSizeStr); + DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, + DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%s => %s bytes)\n", fCtx->nbFilesProcessed, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, - fCtx->totalBytesInput, fCtx->totalBytesOutput); + inputSizeStr, outputSizeStr); } FIO_freeCResources(&ress); From 35576e63ce5f770b99306c5730244e310c0f0aed Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Sat, 5 Jun 2021 07:41:26 -0700 Subject: [PATCH 08/26] Convert tabs to spaces --- programs/benchzstd.c | 14 +++++++------- programs/fileio.c | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 954c9282d..b4c73d73d 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -388,8 +388,8 @@ BMK_benchMemAdvancedNoAlloc( # define NB_MARKS 4 const char* marks[NB_MARKS] = { " |", " /", " =", " \\" }; U32 markNb = 0; - char inputSizeStr[8] = ""; - char outputSizeStr[8] = ""; + char inputSizeStr[8] = ""; + char outputSizeStr[8] = ""; int compressionCompleted = (adv->mode == BMK_decodeOnly); int decompressionCompleted = (adv->mode == BMK_compressOnly); BMK_benchParams_t cbp, dbp; @@ -431,7 +431,7 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - humanSize((unsigned)srcSize, inputSizeStr); + humanSize((unsigned)srcSize, inputSizeStr); DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> \r", marks[markNb], displayName, inputSizeStr); @@ -456,8 +456,8 @@ BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - humanSize((unsigned)srcSize, inputSizeStr); - humanSize((unsigned)cSize, outputSizeStr); + humanSize((unsigned)srcSize, inputSizeStr); + humanSize((unsigned)cSize, outputSizeStr); DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, @@ -483,8 +483,8 @@ BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - humanSize((unsigned)srcSize, inputSizeStr); - humanSize((unsigned)cSize, outputSizeStr); + humanSize((unsigned)srcSize, inputSizeStr); + humanSize((unsigned)cSize, outputSizeStr); DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, diff --git a/programs/fileio.c b/programs/fileio.c index 7432348c0..cf65c2828 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1544,8 +1544,8 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, U64 readsize = 0; U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); - char inputSizeStr[8] = ""; - char outputSizeStr[8] = ""; + char inputSizeStr[8] = ""; + char outputSizeStr[8] = ""; DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); @@ -1601,8 +1601,8 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); } else { - humanSize((unsigned long long) readsize, inputSizeStr); - humanSize((unsigned long long) compressedfilesize, outputSizeStr); + humanSize((unsigned long long) readsize, inputSizeStr); + humanSize((unsigned long long) compressedfilesize, outputSizeStr); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%s => %s, %s) \n", srcFileName, @@ -1841,8 +1841,8 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, { int status; int error = 0; - char inputSizeStr[8] = ""; - char outputSizeStr[8] = ""; + char inputSizeStr[8] = ""; + char outputSizeStr[8] = ""; cRess_t ress = FIO_createCResources(prefs, dictFileName, FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), compressionLevel, comprParams); @@ -1898,8 +1898,8 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, } if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { - humanSize((unsigned long long) fCtx->totalBytesInput, inputSizeStr); - humanSize((unsigned long long) fCtx->totalBytesOutput, outputSizeStr); + humanSize((unsigned long long) fCtx->totalBytesInput, inputSizeStr); + humanSize((unsigned long long) fCtx->totalBytesOutput, outputSizeStr); DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%s => %s bytes)\n", fCtx->nbFilesProcessed, From e5fc830795f2966d0198bf3b6523e86984fbf4fa Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Sat, 5 Jun 2021 08:45:35 -0700 Subject: [PATCH 09/26] human_size() should use size_t --- programs/util.c | 6 +++--- programs/util.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/util.c b/programs/util.c index 6d04b8fb7..cdfda1ca0 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ -char* humanSize(long size, char* str) { +char* humanSize(size_t size, char* str) { if (size > 1125899906842624L) { snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); } else if (size > 1099511627776L) { @@ -132,8 +132,8 @@ char* humanSize(long size, char* str) { snprintf(str, 7, "%.1fM", (float)size / 1048576L); } else if (size > 1024) { snprintf(str, 7, "%.1fK", (float)size / 1024); - } else if (size >= 0) { - snprintf(str, 7, "%dB", 0); + } else if (size <= 1024) { + snprintf(str, 7, "%luB", size); } else { str[0] = '\0'; } diff --git a/programs/util.h b/programs/util.h index aaaf2522c..8ac930d8c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,7 +122,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif -char* humanSize(long size, char* str); +char* humanSize(size_t size, char* str); /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. From 1ef6f3d079b2415c6bdea61c944a1a0cea3a9a28 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Sat, 5 Jun 2021 10:06:40 -0700 Subject: [PATCH 10/26] Use unsigned long instead to help with some tests --- programs/util.c | 2 +- programs/util.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/util.c b/programs/util.c index cdfda1ca0..e7acc6cbb 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ -char* humanSize(size_t size, char* str) { +char* humanSize(unsigned long size, char* str) { if (size > 1125899906842624L) { snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); } else if (size > 1099511627776L) { diff --git a/programs/util.h b/programs/util.h index 8ac930d8c..21e93d7d7 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,7 +122,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif -char* humanSize(size_t size, char* str); +char* humanSize(unsigned long size, char* str); /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. From 64385ef7cbb3388ceacb7611f29b2f0a03bdf477 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Sat, 5 Jun 2021 10:30:21 -0700 Subject: [PATCH 11/26] Update humanSize() to skip the big numbers (it requires 64 bit) --- programs/util.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index e7acc6cbb..c2883ba7d 100644 --- a/programs/util.c +++ b/programs/util.c @@ -122,11 +122,15 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, ***************************************/ char* humanSize(unsigned long size, char* str) { + /* This only works on 64 bit platforms so I commented it out for now */ + /* if (size > 1125899906842624L) { snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); } else if (size > 1099511627776L) { snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); - } else if (size > 1073741824L) { + */ + + if (size > 1073741824L) { snprintf(str, 7, "%.1fG", (float)size / 1073741824L); } else if (size > 1048576L) { snprintf(str, 7, "%.1fM", (float)size / 1048576L); From 20b9b00b413c799bef54bfeb2a36256f11d65b06 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Sat, 5 Jun 2021 11:11:09 -0700 Subject: [PATCH 12/26] Try unsigned long long --- programs/util.c | 4 ++-- programs/util.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/util.c b/programs/util.c index c2883ba7d..1ca8eec19 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,7 +121,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ -char* humanSize(unsigned long size, char* str) { +char* humanSize(unsigned long long size, char* str) { /* This only works on 64 bit platforms so I commented it out for now */ /* if (size > 1125899906842624L) { @@ -137,7 +137,7 @@ char* humanSize(unsigned long size, char* str) { } else if (size > 1024) { snprintf(str, 7, "%.1fK", (float)size / 1024); } else if (size <= 1024) { - snprintf(str, 7, "%luB", size); + snprintf(str, 7, "%lluB", size); } else { str[0] = '\0'; } diff --git a/programs/util.h b/programs/util.h index 21e93d7d7..0eb64c013 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,7 +122,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif -char* humanSize(unsigned long size, char* str); +char* humanSize(unsigned long long size, char* str); /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. From 376a2730a8fbcefce94b33ed485a8b5ea606eb5e Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Sat, 5 Jun 2021 11:12:09 -0700 Subject: [PATCH 13/26] Try enabling the BIG strings now the unsigned long long is in effect --- programs/util.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/programs/util.c b/programs/util.c index 1ca8eec19..f99bf82c5 100644 --- a/programs/util.c +++ b/programs/util.c @@ -122,27 +122,23 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, ***************************************/ char* humanSize(unsigned long long size, char* str) { - /* This only works on 64 bit platforms so I commented it out for now */ - /* - if (size > 1125899906842624L) { - snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); - } else if (size > 1099511627776L) { - snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); - */ + if (size > 1125899906842624L) { + snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); + } else if (size > 1099511627776L) { + snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); + } else if (size > 1073741824L) { + snprintf(str, 7, "%.1fG", (float)size / 1073741824L); + } else if (size > 1048576L) { + snprintf(str, 7, "%.1fM", (float)size / 1048576L); + } else if (size > 1024) { + snprintf(str, 7, "%.1fK", (float)size / 1024); + } else if (size <= 1024) { + snprintf(str, 7, "%lluB", size); + } else { + str[0] = '\0'; + } - if (size > 1073741824L) { - snprintf(str, 7, "%.1fG", (float)size / 1073741824L); - } else if (size > 1048576L) { - snprintf(str, 7, "%.1fM", (float)size / 1048576L); - } else if (size > 1024) { - snprintf(str, 7, "%.1fK", (float)size / 1024); - } else if (size <= 1024) { - snprintf(str, 7, "%lluB", size); - } else { - str[0] = '\0'; - } - - return str; + return str; } From 1eb852854b48f7258b74d4e781e036db1b260607 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Mon, 7 Jun 2021 09:31:38 -0700 Subject: [PATCH 14/26] Some fixes to address things @felixhandte found --- programs/util.c | 10 +++++++--- programs/util.h | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/programs/util.c b/programs/util.c index f99bf82c5..934ec526d 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,8 +121,14 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ +/* + * Take a size in bytes and output a human readable string. Maximum + * buffer size is 8 but it's usually 7. Example: "123.4G" +*/ char* humanSize(unsigned long long size, char* str) { - if (size > 1125899906842624L) { + if (size > 1152921504606846976L) { + snprintf(str, 7, "%.1fE", (float)size / 1152921504606846976L); + } else if (size > 1125899906842624L) { snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); } else if (size > 1099511627776L) { snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); @@ -134,8 +140,6 @@ char* humanSize(unsigned long long size, char* str) { snprintf(str, 7, "%.1fK", (float)size / 1024); } else if (size <= 1024) { snprintf(str, 7, "%lluB", size); - } else { - str[0] = '\0'; } return str; diff --git a/programs/util.h b/programs/util.h index 0eb64c013..ba2ae13c8 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,6 +122,10 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif +/* + * Take a size in bytes and output a human readable string. Maximum + * buffer size is 8 but it's usually 7. Example: "123.4G" +*/ char* humanSize(unsigned long long size, char* str); /** From 8e0a9695d7a28087c14ca62ed15570021311a263 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Mon, 7 Jun 2021 13:53:55 -0700 Subject: [PATCH 15/26] Attempt to fix a failing test with help from @aqrit --- programs/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 934ec526d..691613940 100644 --- a/programs/util.c +++ b/programs/util.c @@ -139,7 +139,7 @@ char* humanSize(unsigned long long size, char* str) { } else if (size > 1024) { snprintf(str, 7, "%.1fK", (float)size / 1024); } else if (size <= 1024) { - snprintf(str, 7, "%lluB", size); + snprintf(str, 7, "%uB", (unsigned)size); } return str; From bbb81c8801006d8a4bcc10d2605da2cb1eda1662 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 13:05:44 -0400 Subject: [PATCH 16/26] Avoid `snprintf()` in Preparing Human-Readable Sizes; Improve Formatting This produces the following formatting: Size | `zstd` | `ls -lh` ---------- | ------ | -------- 1 | 1 | 1 12 | 12 | 12 123 | 123 | 123 1234 | 1.21K | 1.3K 12345 | 12.1K | 13K 123456 | 121K | 121K 1234567 | 1.18M | 1.2M 12345678 | 11.8M | 12M 123456789 | 118M | 118M 1234567890 | 1.15G | 1.2G 999 | 999 | 999 1000 | 1000 | 1000 1001 | 1001 | 1001 1023 | 1023 | 1023 1024 | 1.000K | 1.0K 1025 | 1.00K | 1.1K 999999 | 977K | 977K 1000000 | 977K | 977K 1000001 | 977K | 977K 1023999 | 1000K | 1000K 1024000 | 1000K | 1000K 1024001 | 1000K | 1001K 1048575 | 1024K | 1.0M 1048576 | 1.000M | 1.0M 1048577 | 1.00M | 1.1M This was produced with the following invocation: ``` for N in 1 12 123 1234 12345 123456 1234567 12345678 123456789 1234567890 999 1000 1001 1023 1024 1025 999999 1000000 1000001 1023999 1024000 1024001 1048575 1048576 1048577; do head -c $N /dev/urandom > r$N done ./zstd -i1 -b1 -S r1 r12 r123 r1234 r12345 r123456 r1234567 r12345678 r123456789 r1234567890 r999 r1000 r1001 r1023 r1024 r1025 r999999 r1000000 r1000001 r1023999 r1024000 r1024001 r1048575 r1048576 r1048577 ``` --- programs/benchzstd.c | 28 +++++++++----------- programs/fileio.c | 24 ++++++++--------- programs/util.c | 62 ++++++++++++++++++++++++++------------------ programs/util.h | 18 +++++++++---- 4 files changed, 73 insertions(+), 59 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index b4c73d73d..db051781a 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -388,13 +388,13 @@ BMK_benchMemAdvancedNoAlloc( # define NB_MARKS 4 const char* marks[NB_MARKS] = { " |", " /", " =", " \\" }; U32 markNb = 0; - char inputSizeStr[8] = ""; - char outputSizeStr[8] = ""; int compressionCompleted = (adv->mode == BMK_decodeOnly); int decompressionCompleted = (adv->mode == BMK_compressOnly); BMK_benchParams_t cbp, dbp; BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; + UTIL_HumanReadableSize_t hr_isize; + UTIL_HumanReadableSize_t hr_osize; cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; @@ -431,10 +431,10 @@ BMK_benchMemAdvancedNoAlloc( dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; - humanSize((unsigned)srcSize, inputSizeStr); + hr_isize = UTIL_makeHumanReadableSize((U64) srcSize); DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ - DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> \r", marks[markNb], displayName, inputSizeStr); + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix); while (!(compressionCompleted && decompressionCompleted)) { if (!compressionCompleted) { @@ -455,13 +455,11 @@ BMK_benchMemAdvancedNoAlloc( } } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - - humanSize((unsigned)srcSize, inputSizeStr); - humanSize((unsigned)cSize, outputSizeStr); - - DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s\r", + hr_osize = UTIL_makeHumanReadableSize((U64) cSize); + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, - inputSizeStr, outputSizeStr, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); } @@ -482,13 +480,11 @@ BMK_benchMemAdvancedNoAlloc( } { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - - humanSize((unsigned)srcSize, inputSizeStr); - humanSize((unsigned)cSize, outputSizeStr); - - DISPLAYLEVEL(2, "%2s-%-17.17s : %s -> %s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + hr_osize = UTIL_makeHumanReadableSize((U64) cSize); + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, - inputSizeStr, outputSizeStr, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, ratioAccuracy, ratio, benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, (double)benchResult.dSpeed / MB_UNIT); diff --git a/programs/fileio.c b/programs/fileio.c index cf65c2828..c83c4fffd 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1544,9 +1544,6 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, U64 readsize = 0; U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); - char inputSizeStr[8] = ""; - char outputSizeStr[8] = ""; - DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); /* compression format selection */ @@ -1601,13 +1598,14 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); } else { - humanSize((unsigned long long) readsize, inputSizeStr); - humanSize((unsigned long long) compressedfilesize, outputSizeStr); + UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); + UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%s => %s, %s) \n", + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%.*f%s => %.*f%s, %s) \n", srcFileName, (double)compressedfilesize / (double)readsize * 100, - inputSizeStr, outputSizeStr, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, dstFileName); } } @@ -1841,8 +1839,6 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, { int status; int error = 0; - char inputSizeStr[8] = ""; - char outputSizeStr[8] = ""; cRess_t ress = FIO_createCResources(prefs, dictFileName, FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), compressionLevel, comprParams); @@ -1898,13 +1894,15 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, } if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { - humanSize((unsigned long long) fCtx->totalBytesInput, inputSizeStr); - humanSize((unsigned long long) fCtx->totalBytesOutput, outputSizeStr); + UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); + UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%s => %s bytes)\n", fCtx->nbFilesProcessed, + DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%.*f%s => %.*f%s bytes)\n", + fCtx->nbFilesProcessed, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, - inputSizeStr, outputSizeStr); + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix); } FIO_freeCResources(&ress); diff --git a/programs/util.c b/programs/util.c index 691613940..347f5cc2f 100644 --- a/programs/util.c +++ b/programs/util.c @@ -121,31 +121,6 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, * Functions ***************************************/ -/* - * Take a size in bytes and output a human readable string. Maximum - * buffer size is 8 but it's usually 7. Example: "123.4G" -*/ -char* humanSize(unsigned long long size, char* str) { - if (size > 1152921504606846976L) { - snprintf(str, 7, "%.1fE", (float)size / 1152921504606846976L); - } else if (size > 1125899906842624L) { - snprintf(str, 7, "%.1fP", (float)size / 1125899906842624L); - } else if (size > 1099511627776L) { - snprintf(str, 7, "%.1fT", (float)size / 1099511627776L); - } else if (size > 1073741824L) { - snprintf(str, 7, "%.1fG", (float)size / 1073741824L); - } else if (size > 1048576L) { - snprintf(str, 7, "%.1fM", (float)size / 1048576L); - } else if (size > 1024) { - snprintf(str, 7, "%.1fK", (float)size / 1024); - } else if (size <= 1024) { - snprintf(str, 7, "%uB", (unsigned)size); - } - - return str; -} - - int UTIL_stat(const char* filename, stat_t* statbuf) { #if defined(_MSC_VER) @@ -328,6 +303,43 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) return (U64)statbuf->st_size; } +UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { + UTIL_HumanReadableSize_t hrs; + if (size >= (1L << 60)) { + hrs.value = (float)size / (1L << 60); + hrs.suffix = "E"; + } else if (size >= (1L << 50)) { + hrs.value = (float)size / (1L << 50); + hrs.suffix = "P"; + } else if (size >= (1L << 40)) { + hrs.value = (float)size / (1L << 40); + hrs.suffix = "T"; + } else if (size >= (1L << 30)) { + hrs.value = (float)size / (1L << 30); + hrs.suffix = "G"; + } else if (size >= (1L << 20)) { + hrs.value = (float)size / (1L << 20); + hrs.suffix = "M"; + } else if (size >= (1L << 10)) { + hrs.value = (float)size / (1L << 10); + hrs.suffix = "K"; + } else { + hrs.value = (float)size; + hrs.suffix = ""; + } + + if (hrs.value >= 100 || (U64)hrs.value == size) { + hrs.precision = 0; + } else if (hrs.value > 10) { + hrs.precision = 1; + } else if (hrs.value > 1) { + hrs.precision = 2; + } else { + hrs.precision = 3; + } + + return hrs; +} U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) { diff --git a/programs/util.h b/programs/util.h index ba2ae13c8..63cd0c378 100644 --- a/programs/util.h +++ b/programs/util.h @@ -122,11 +122,6 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const #define STRDUP(s) strdup(s) #endif -/* - * Take a size in bytes and output a human readable string. Maximum - * buffer size is 8 but it's usually 7. Example: "123.4G" -*/ -char* humanSize(unsigned long long size, char* str); /** * Calls platform's equivalent of stat() on filename and writes info to statbuf. @@ -176,6 +171,19 @@ int UTIL_isFIFO(const char* infilename); U64 UTIL_getFileSize(const char* infilename); U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); +/** + * Take a size in bytes and prepare the components to pretty-print it in a + * scaled way. The components in the returned struct should be passed in + * precision, value, suffix order to a "%.*f%s" format string. + */ +typedef struct { + float value; + int precision; + const char* suffix; +} UTIL_HumanReadableSize_t; + +UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size); + int UTIL_compareStr(const void *p1, const void *p2); const char* UTIL_getFileExtension(const char* infilename); void UTIL_mirrorSourceFilesDirectories(const char** fileNamesTable, unsigned int nbFiles, const char *outDirName); From 9b67219b1e961574724f2c4fb9f2796430e7fccf Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 13:27:57 -0400 Subject: [PATCH 17/26] Fix Integer Constants; Fix Comparison --- programs/util.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/programs/util.c b/programs/util.c index 347f5cc2f..964d60ee2 100644 --- a/programs/util.c +++ b/programs/util.c @@ -305,23 +305,23 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { UTIL_HumanReadableSize_t hrs; - if (size >= (1L << 60)) { - hrs.value = (float)size / (1L << 60); + if (size >= (1ull << 60)) { + hrs.value = (float)size / (1ull << 60); hrs.suffix = "E"; - } else if (size >= (1L << 50)) { - hrs.value = (float)size / (1L << 50); + } else if (size >= (1ull << 50)) { + hrs.value = (float)size / (1ull << 50); hrs.suffix = "P"; - } else if (size >= (1L << 40)) { - hrs.value = (float)size / (1L << 40); + } else if (size >= (1ull << 40)) { + hrs.value = (float)size / (1ull << 40); hrs.suffix = "T"; - } else if (size >= (1L << 30)) { - hrs.value = (float)size / (1L << 30); + } else if (size >= (1ull << 30)) { + hrs.value = (float)size / (1ull << 30); hrs.suffix = "G"; - } else if (size >= (1L << 20)) { - hrs.value = (float)size / (1L << 20); + } else if (size >= (1ull << 20)) { + hrs.value = (float)size / (1ull << 20); hrs.suffix = "M"; - } else if (size >= (1L << 10)) { - hrs.value = (float)size / (1L << 10); + } else if (size >= (1ull << 10)) { + hrs.value = (float)size / (1ull << 10); hrs.suffix = "K"; } else { hrs.value = (float)size; @@ -330,7 +330,7 @@ UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { if (hrs.value >= 100 || (U64)hrs.value == size) { hrs.precision = 0; - } else if (hrs.value > 10) { + } else if (hrs.value >= 10) { hrs.precision = 1; } else if (hrs.value > 1) { hrs.precision = 2; From 464bfb022ef2d1f44778717b71979b86a3ffbea5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 15:22:59 -0400 Subject: [PATCH 18/26] In Verbose Mode, Preserve Full Precision Where Possible --- programs/util.c | 76 ++++++++++++++++++++++++++++------------------ programs/util.h | 2 +- programs/zstdcli.c | 3 +- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/programs/util.c b/programs/util.c index 964d60ee2..c8f6d1886 100644 --- a/programs/util.c +++ b/programs/util.c @@ -305,37 +305,55 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { UTIL_HumanReadableSize_t hrs; - if (size >= (1ull << 60)) { - hrs.value = (float)size / (1ull << 60); - hrs.suffix = "E"; - } else if (size >= (1ull << 50)) { - hrs.value = (float)size / (1ull << 50); - hrs.suffix = "P"; - } else if (size >= (1ull << 40)) { - hrs.value = (float)size / (1ull << 40); - hrs.suffix = "T"; - } else if (size >= (1ull << 30)) { - hrs.value = (float)size / (1ull << 30); - hrs.suffix = "G"; - } else if (size >= (1ull << 20)) { - hrs.value = (float)size / (1ull << 20); - hrs.suffix = "M"; - } else if (size >= (1ull << 10)) { - hrs.value = (float)size / (1ull << 10); - hrs.suffix = "K"; - } else { - hrs.value = (float)size; - hrs.suffix = ""; - } - if (hrs.value >= 100 || (U64)hrs.value == size) { - hrs.precision = 0; - } else if (hrs.value >= 10) { - hrs.precision = 1; - } else if (hrs.value > 1) { - hrs.precision = 2; + if (g_utilDisplayLevel > 2) { + /* In verbose mode, do not scale sizes down, except in the case of + * values that exceed the integral precision of a double. */ + if (size >= (1ull << 53)) { + hrs.value = (double)size / (1ull << 20); + hrs.suffix = "M"; + /* At worst, a double representation of a maximal size will be + * accurate to better than tens of kilobytes. */ + hrs.precision = 2; + } else { + hrs.value = (double)size; + hrs.suffix = ""; + hrs.precision = 0; + } } else { - hrs.precision = 3; + /* In regular mode, scale sizes down and use suffixes. */ + if (size >= (1ull << 60)) { + hrs.value = (double)size / (1ull << 60); + hrs.suffix = "E"; + } else if (size >= (1ull << 50)) { + hrs.value = (double)size / (1ull << 50); + hrs.suffix = "P"; + } else if (size >= (1ull << 40)) { + hrs.value = (double)size / (1ull << 40); + hrs.suffix = "T"; + } else if (size >= (1ull << 30)) { + hrs.value = (double)size / (1ull << 30); + hrs.suffix = "G"; + } else if (size >= (1ull << 20)) { + hrs.value = (double)size / (1ull << 20); + hrs.suffix = "M"; + } else if (size >= (1ull << 10)) { + hrs.value = (double)size / (1ull << 10); + hrs.suffix = "K"; + } else { + hrs.value = (double)size; + hrs.suffix = ""; + } + + if (hrs.value >= 100 || (U64)hrs.value == size) { + hrs.precision = 0; + } else if (hrs.value >= 10) { + hrs.precision = 1; + } else if (hrs.value > 1) { + hrs.precision = 2; + } else { + hrs.precision = 3; + } } return hrs; diff --git a/programs/util.h b/programs/util.h index 63cd0c378..f146f3a08 100644 --- a/programs/util.h +++ b/programs/util.h @@ -177,7 +177,7 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles); * precision, value, suffix order to a "%.*f%s" format string. */ typedef struct { - float value; + double value; int precision; const char* suffix; } UTIL_HumanReadableSize_t; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9e2133c4f..32cd5af86 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1146,8 +1146,9 @@ int main(int const argCount, const char* argv[]) (void)singleThread; (void)nbWorkers; #endif -#ifdef UTIL_HAS_CREATEFILELIST g_utilDisplayLevel = g_displayLevel; + +#ifdef UTIL_HAS_CREATEFILELIST if (!followLinks) { unsigned u, fileNamesNb; unsigned const nbFilenames = (unsigned)filenames->tableSize; From 93bb368d744ac56e4536b739e188aeadb2aaee8e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 15:26:16 -0400 Subject: [PATCH 19/26] Change Suffix (e.g., "G" -> " GB") --- programs/fileio.c | 2 +- programs/util.c | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c83c4fffd..d149c5a64 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1898,7 +1898,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%.*f%s => %.*f%s bytes)\n", + DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%.*f%s => %.*f%s)\n", fCtx->nbFilesProcessed, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, hr_isize.precision, hr_isize.value, hr_isize.suffix, diff --git a/programs/util.c b/programs/util.c index c8f6d1886..838da8304 100644 --- a/programs/util.c +++ b/programs/util.c @@ -311,38 +311,38 @@ UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { * values that exceed the integral precision of a double. */ if (size >= (1ull << 53)) { hrs.value = (double)size / (1ull << 20); - hrs.suffix = "M"; + hrs.suffix = " MB"; /* At worst, a double representation of a maximal size will be * accurate to better than tens of kilobytes. */ hrs.precision = 2; } else { hrs.value = (double)size; - hrs.suffix = ""; + hrs.suffix = " B"; hrs.precision = 0; } } else { /* In regular mode, scale sizes down and use suffixes. */ if (size >= (1ull << 60)) { hrs.value = (double)size / (1ull << 60); - hrs.suffix = "E"; + hrs.suffix = " EB"; } else if (size >= (1ull << 50)) { hrs.value = (double)size / (1ull << 50); - hrs.suffix = "P"; + hrs.suffix = " PB"; } else if (size >= (1ull << 40)) { hrs.value = (double)size / (1ull << 40); - hrs.suffix = "T"; + hrs.suffix = " TB"; } else if (size >= (1ull << 30)) { hrs.value = (double)size / (1ull << 30); - hrs.suffix = "G"; + hrs.suffix = " GB"; } else if (size >= (1ull << 20)) { hrs.value = (double)size / (1ull << 20); - hrs.suffix = "M"; + hrs.suffix = " MB"; } else if (size >= (1ull << 10)) { hrs.value = (double)size / (1ull << 10); - hrs.suffix = "K"; + hrs.suffix = " KB"; } else { hrs.value = (double)size; - hrs.suffix = ""; + hrs.suffix = " B"; } if (hrs.value >= 100 || (U64)hrs.value == size) { From 7e0058848ca45d1ff8c7e846955b0f15a4635073 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 15:28:12 -0400 Subject: [PATCH 20/26] Fix Whitespace --- programs/benchzstd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index db051781a..72793f07b 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -456,7 +456,7 @@ BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; hr_osize = UTIL_makeHumanReadableSize((U64) cSize); - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f),%6.*f MB/s\r", + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s\r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix, hr_osize.precision, hr_osize.value, hr_osize.suffix, @@ -481,7 +481,7 @@ BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; hr_osize = UTIL_makeHumanReadableSize((U64) cSize); - DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", + DISPLAYLEVEL(2, "%2s-%-17.17s : %.*f%s -> %.*f%s (%5.*f), %6.*f MB/s, %6.1f MB/s \r", marks[markNb], displayName, hr_isize.precision, hr_isize.value, hr_isize.suffix, hr_osize.precision, hr_osize.value, hr_osize.suffix, From bc46b6efe43d2cf521e14806b67ec26d4b8fb33e Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 16:04:10 -0400 Subject: [PATCH 21/26] Apply to Other Print Statement as Well --- programs/fileio.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index d149c5a64..4073f030b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1592,15 +1592,15 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, if (g_display_prefs.displayLevel >= 2 && !fCtx->hasStdoutOutput && (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { + UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); + UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + DISPLAYLEVEL(2,"%-20s : (%.*f%s => %.*f%s, %s) \n", srcFileName, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, + hr_isize.precision, hr_isize.value, hr_isize.suffix, + hr_osize.precision, hr_osize.value, hr_osize.suffix, dstFileName); } else { - UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); - UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%.*f%s => %.*f%s, %s) \n", srcFileName, (double)compressedfilesize / (double)readsize * 100, From 9c340ce014e70d413e7a01f0d0b77376a24db54d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 9 Jun 2021 16:13:00 -0400 Subject: [PATCH 22/26] Require `-vv` to Enable Full Precision --- programs/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 838da8304..2622ae825 100644 --- a/programs/util.c +++ b/programs/util.c @@ -306,7 +306,7 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf) UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { UTIL_HumanReadableSize_t hrs; - if (g_utilDisplayLevel > 2) { + if (g_utilDisplayLevel > 3) { /* In verbose mode, do not scale sizes down, except in the case of * values that exceed the integral precision of a double. */ if (size >= (1ull << 53)) { From 2af3687c507a2800eb900ea49665a2f67896e83f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Jun 2021 12:06:51 -0400 Subject: [PATCH 23/26] Switch to Binary Size Prefixes (e.g., "MB" -> "MiB") Suggested by @aqrit, a little more verbose, but hopefully addresses a real ambiguity. --- programs/util.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/util.c b/programs/util.c index 2622ae825..96459b9d3 100644 --- a/programs/util.c +++ b/programs/util.c @@ -311,7 +311,7 @@ UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { * values that exceed the integral precision of a double. */ if (size >= (1ull << 53)) { hrs.value = (double)size / (1ull << 20); - hrs.suffix = " MB"; + hrs.suffix = " MiB"; /* At worst, a double representation of a maximal size will be * accurate to better than tens of kilobytes. */ hrs.precision = 2; @@ -324,22 +324,22 @@ UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) { /* In regular mode, scale sizes down and use suffixes. */ if (size >= (1ull << 60)) { hrs.value = (double)size / (1ull << 60); - hrs.suffix = " EB"; + hrs.suffix = " EiB"; } else if (size >= (1ull << 50)) { hrs.value = (double)size / (1ull << 50); - hrs.suffix = " PB"; + hrs.suffix = " PiB"; } else if (size >= (1ull << 40)) { hrs.value = (double)size / (1ull << 40); - hrs.suffix = " TB"; + hrs.suffix = " TiB"; } else if (size >= (1ull << 30)) { hrs.value = (double)size / (1ull << 30); - hrs.suffix = " GB"; + hrs.suffix = " GiB"; } else if (size >= (1ull << 20)) { hrs.value = (double)size / (1ull << 20); - hrs.suffix = " MB"; + hrs.suffix = " MiB"; } else if (size >= (1ull << 10)) { hrs.value = (double)size / (1ull << 10); - hrs.suffix = " KB"; + hrs.suffix = " KiB"; } else { hrs.value = (double)size; hrs.suffix = " B"; From 87e94e3e39434d6c7175a0c9d67c2620cf37d721 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Jun 2021 12:31:42 -0400 Subject: [PATCH 24/26] Convert Other Size Displays to Use Human-Readable Formatting --- programs/fileio.c | 80 +++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4073f030b..4da1740d9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1342,6 +1342,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, unsigned inputPresented = 0; unsigned inputBlocked = 0; unsigned lastJobID = 0; + UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize); DISPLAYLEVEL(6, "compression using zstd format \n"); @@ -1352,6 +1353,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, /* unknown source size; use the declared stream size */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); } + (void)srcFileName; /* Main compression loop */ @@ -1395,14 +1397,17 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, if (READY_FOR_UPDATE()) { ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; + UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed); + UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed); + UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced); /* display progress notifications */ if (g_display_prefs.displayLevel >= 3) { - DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", + DISPLAYUPDATE(3, "\r(L%i) Buffered :%6.*f%4s - Consumed :%6.*f%4s - Compressed :%6.*f%s => %.2f%% ", compressionLevel, - (unsigned)((zfp.ingested - zfp.consumed) >> 20), - (unsigned)(zfp.consumed >> 20), - (unsigned)(zfp.produced >> 20), + buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix, + consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix, + produced_hrs.precision, produced_hrs.value, produced_hrs.suffix, cShare ); } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { /* Require level 2 or forcibly displayed progress counter for summarized updates */ @@ -1419,10 +1424,10 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); } } - DISPLAYLEVEL(1, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); + DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix); if (fileSize != UTIL_FILESIZE_UNKNOWN) - DISPLAYLEVEL(2, "/ %2u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(1, "MB ==> %2.f%%", cShare); + DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix); + DISPLAYLEVEL(1, " ==> %2.f%%", cShare); DELAY_NEXT_UPDATE(); } @@ -2165,6 +2170,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); const int displayLevel = (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2; + UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize); if (ZSTD_isError(readSizeHint)) { DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", srcFileName, ZSTD_getErrorName(readSizeHint)); @@ -2179,15 +2185,15 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, size_t srcFileNameSize = strlen(srcFileName); if (srcFileNameSize > 18) { const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; - DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %u MB... ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix); } else { - DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %u MB... ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix); } } else { - DISPLAYUPDATE(displayLevel, "\r%-20.20s : %u MB... ", - srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ", + srcFileName, hrs.precision, hrs.value, hrs.suffix); } if (inBuff.pos > 0) { @@ -2411,9 +2417,11 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, /* Write Block */ if (decodedBytes) { + UTIL_HumanReadableSize_t hrs; storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); filesize += decodedBytes; - DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + hrs = UTIL_makeHumanReadableSize(filesize); + DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); } if (!nextToLoad) break; @@ -2981,25 +2989,24 @@ getFileInfo(fileInfo_t* info, const char* srcFileName) static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) { - unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); - const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; - double const windowSizeUnit = (double)info->windowSize / unit; - double const compressedSizeUnit = (double)info->compressedSize / unit; - double const decompressedSizeUnit = (double)info->decompressedSize / unit; + UTIL_HumanReadableSize_t const window_hrs = UTIL_makeHumanReadableSize(info->windowSize); + UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(info->compressedSize); + UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(info->decompressedSize); double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)info->compressedSize; const char* const checkString = (info->usesCheck ? "XXH64" : "None"); if (displayLevel <= 2) { if (!info->decompUnavailable) { - DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", + DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %s\n", info->numSkippableFrames + info->numActualFrames, info->numSkippableFrames, - compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, + decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, ratio, checkString, inFileName); } else { - DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n", + DISPLAYOUT("%6d %5d %6.*f%4s %5s %s\n", info->numSkippableFrames + info->numActualFrames, info->numSkippableFrames, - compressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, checkString, inFileName); } } else { @@ -3007,15 +3014,15 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); if (info->numSkippableFrames) DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); - DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", - windowSizeUnit, unitStr, + DISPLAYOUT("Window Size: %.*f%s (%llu B)\n", + window_hrs.precision, window_hrs.value, window_hrs.suffix, (unsigned long long)info->windowSize); - DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", - compressedSizeUnit, unitStr, + DISPLAYOUT("Compressed Size: %.*f%s (%llu B)\n", + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, (unsigned long long)info->compressedSize); if (!info->decompUnavailable) { - DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", - decompressedSizeUnit, unitStr, + DISPLAYOUT("Decompressed Size: %.*f%s (%llu B)\n", + decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, (unsigned long long)info->decompressedSize); DISPLAYOUT("Ratio: %.4f\n", ratio); } @@ -3103,24 +3110,23 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis error |= FIO_listFile(&total, filenameTable[u], displayLevel); } } if (numFiles > 1 && displayLevel <= 2) { /* display total */ - unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); - const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; - double const compressedSizeUnit = (double)total.compressedSize / unit; - double const decompressedSizeUnit = (double)total.decompressedSize / unit; + UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(total.compressedSize); + UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(total.decompressedSize); double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/(double)total.compressedSize; const char* const checkString = (total.usesCheck ? "XXH64" : ""); DISPLAYOUT("----------------------------------------------------------------- \n"); if (total.decompUnavailable) { - DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n", + DISPLAYOUT("%6d %5d %6.*f%4s %5s %u files\n", total.numSkippableFrames + total.numActualFrames, total.numSkippableFrames, - compressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, checkString, (unsigned)total.nbFiles); } else { - DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n", + DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %u files\n", total.numSkippableFrames + total.numActualFrames, total.numSkippableFrames, - compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, + compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, + decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, ratio, checkString, (unsigned)total.nbFiles); } } return error; From 94cf57bb134af9747a0f21b81c31aec67460c4db Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Jun 2021 13:14:18 -0400 Subject: [PATCH 25/26] Update Tests to Reflect New Formatting --- tests/playTests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 252939006..54f2033cf 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1445,7 +1445,7 @@ datagen -g0 > tmp5 zstd tmp5 zstd -l tmp5.zst zstd -l tmp5* && die "-l must fail on non-zstd file" -zstd -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header +zstd -lv tmp5.zst | grep "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header zstd -lv tmp5* && die "-l must fail on non-zstd file" println "\n===> zstd --list/-l test with no content size field " @@ -1616,7 +1616,7 @@ roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" if [ -n "$hasMT" ] then - println "\n===> zstdmt long round-trip tests " + println "\n===> zstdmt long round-trip tests " roundTripTest -g80000000 -P99 "19 -T2" " " roundTripTest -g5000000000 -P99 "1 -T2" " " roundTripTest -g500000000 -P97 "1 -T999" " " From 8c00807bbcee5fdeb1db126cec4529f09375dc2f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Jun 2021 13:28:38 -0400 Subject: [PATCH 26/26] Whitespace Fixes to Improve Cross-Line Alignment --- programs/fileio.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4da1740d9..da5412f27 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1403,7 +1403,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, /* display progress notifications */ if (g_display_prefs.displayLevel >= 3) { - DISPLAYUPDATE(3, "\r(L%i) Buffered :%6.*f%4s - Consumed :%6.*f%4s - Compressed :%6.*f%s => %.2f%% ", + DISPLAYUPDATE(3, "\r(L%i) Buffered :%6.*f%4s - Consumed :%6.*f%4s - Compressed :%6.*f%4s => %.2f%% ", compressionLevel, buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix, consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix, @@ -1600,13 +1600,13 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx, UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%.*f%s => %.*f%s, %s) \n", + DISPLAYLEVEL(2,"%-20s : (%6.*f%4s => %6.*f%4s, %s) \n", srcFileName, hr_isize.precision, hr_isize.value, hr_isize.suffix, hr_osize.precision, hr_osize.value, hr_osize.suffix, dstFileName); } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%.*f%s => %.*f%s, %s) \n", + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%4s => %6.*f%4s, %s) \n", srcFileName, (double)compressedfilesize / (double)readsize * 100, hr_isize.precision, hr_isize.value, hr_isize.suffix, @@ -1903,7 +1903,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "%3d files compressed : %.2f%% (%.*f%s => %.*f%s)\n", + DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n", fCtx->nbFilesProcessed, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, hr_isize.precision, hr_isize.value, hr_isize.suffix, @@ -2529,7 +2529,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx, fCtx->totalBytesOutput += (size_t)filesize; DISPLAYLEVEL(2, "\r%79s\r", ""); /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) || + if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) || g_display_prefs.displayLevel >= 3 || g_display_prefs.progressSetting == FIO_ps_always) { DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize);