diff --git a/appveyor.yml b/appveyor.yml index 27994853a..5887d52e6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -218,10 +218,10 @@ - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% - if [%HOST%]==[mingw] ( - SET "PATH_MINGW32=C:\MinGW\bin;C:\MinGW\usr\bin" && - SET "PATH_MINGW64=C:\msys64\mingw64\bin;C:\msys64\usr\bin" && - COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe && - COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe + SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && + SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe ) - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" diff --git a/programs/bench.c b/programs/bench.c index d25ff8f04..c3681eb05 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -146,8 +146,11 @@ typedef struct { } blockParam_t; -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +#undef MIN +#undef MAX +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, diff --git a/programs/dibio.c b/programs/dibio.c index b7ed280ea..aac36425c 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -89,7 +89,8 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } -#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#undef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) /* ******************************************************** diff --git a/programs/fileio.c b/programs/fileio.c index f1f5a5577..9bca20664 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -96,6 +96,7 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; static clock_t g_time = 0; +#undef MIN #define MIN(a,b) ((a) < (b) ? (a) : (b)) /* ************************************************************ diff --git a/programs/util.h b/programs/util.h index 9992d79da..b989e8232 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,6 +25,7 @@ extern "C" { #include /* malloc */ #include /* size_t, ptrdiff_t */ #include /* fprintf */ +#include /* strncmp */ #include /* stat, utime */ #include /* stat */ #if defined(_MSC_VER) @@ -488,6 +489,183 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu if (filenameTable) free((void*)filenameTable); } +/* count the number of physical cores */ +#if defined(_WIN32) || defined(WIN32) + +#include + +typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); + +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores = 0; + if (numPhysicalCores != 0) return numPhysicalCores; + + { LPFN_GLPI glpi; + BOOL done = FALSE; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; + DWORD returnLength = 0; + size_t byteOffset = 0; + + glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), + "GetLogicalProcessorInformation"); + + if (glpi == NULL) { + goto failed; + } + + while(!done) { + DWORD rc = glpi(buffer, &returnLength); + if (FALSE == rc) { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + if (buffer) + free(buffer); + buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); + + if (buffer == NULL) { + perror("zstd"); + exit(1); + } + } else { + /* some other error */ + goto failed; + } + } else { + done = TRUE; + } + } + + ptr = buffer; + + while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { + + if (ptr->Relationship == RelationProcessorCore) { + numPhysicalCores++; + } + + ptr++; + byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + } + + free(buffer); + + return numPhysicalCores; + } + +failed: + /* try to fall back on GetSystemInfo */ + { SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + numPhysicalCores = sysinfo.dwNumberOfProcessors; + if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */ + } + return numPhysicalCores; +} + +#elif defined(__APPLE__) + +#include + +/* Use apple-provided syscall + * see: man 3 sysctl */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static S32 numPhysicalCores = 0; /* apple specifies int32_t */ + if (numPhysicalCores != 0) return numPhysicalCores; + + { size_t size = sizeof(S32); + int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0); + if (ret != 0) { + if (errno == ENOENT) { + /* entry not present, fall back on 1 */ + numPhysicalCores = 1; + } else { + perror("zstd: can't get number of physical cpus"); + exit(1); + } + } + + return numPhysicalCores; + } +} + +#elif defined(__linux__) + +/* parse /proc/cpuinfo + * siblings / cpu cores should give hyperthreading ratio + * otherwise fall back on sysconf */ +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + static int numPhysicalCores = 0; + + if (numPhysicalCores != 0) return numPhysicalCores; + + numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (numPhysicalCores == -1) { + /* value not queryable, fall back on 1 */ + return numPhysicalCores = 1; + } + + /* try to determine if there's hyperthreading */ + { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); + size_t const BUF_SIZE = 80; + char buff[BUF_SIZE]; + + int siblings = 0; + int cpu_cores = 0; + int ratio = 1; + + if (cpuinfo == NULL) { + /* fall back on the sysconf value */ + return numPhysicalCores; + } + + /* assume the cpu cores/siblings values will be constant across all + * present processors */ + while (!feof(cpuinfo)) { + if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { + if (strncmp(buff, "siblings", 8) == 0) { + const char* const sep = strchr(buff, ':'); + if (*sep == '\0') { + /* formatting was broken? */ + goto failed; + } + + siblings = atoi(sep + 1); + } + if (strncmp(buff, "cpu cores", 9) == 0) { + const char* const sep = strchr(buff, ':'); + if (*sep == '\0') { + /* formatting was broken? */ + goto failed; + } + + cpu_cores = atoi(sep + 1); + } + } else if (ferror(cpuinfo)) { + /* fall back on the sysconf value */ + goto failed; + } + } + if (siblings && cpu_cores) { + ratio = siblings / cpu_cores; + } +failed: + fclose(cpuinfo); + return numPhysicalCores = numPhysicalCores / ratio; + } +} + +#else + +UTIL_STATIC int UTIL_countPhysicalCores(void) +{ + /* assume 1 */ + return 1; +} + +#endif #if defined (__cplusplus) } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 18e259c74..73b4eab9d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -602,6 +602,11 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); #endif + if (nbThreads == 0) { + /* try to guess */ + nbThreads = UTIL_countPhysicalCores(); + DISPLAYLEVEL(3, "Note: %d physical core(s) detected\n", nbThreads); + } g_utilDisplayLevel = g_displayLevel; if (!followLinks) { diff --git a/tests/playTests.sh b/tests/playTests.sh index 266bbd91b..3675bb168 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -11,6 +11,7 @@ roundTripTest() { local_p="$2" else local_c="$2" + local_p="" fi rm -f tmp1 tmp2 @@ -20,6 +21,23 @@ roundTripTest() { $DIFF -q tmp1 tmp2 } +fileRoundTripTest() { + if [ -n "$3" ]; then + local_c="$3" + local_p="$2" + else + local_c="$2" + local_p="" + fi + + rm -f tmp.zstd tmp.md5.1 tmp.md5.2 + $ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c | $ZSTD -d" + ./datagen $1 $local_p > tmp + cat tmp | $MD5SUM > tmp.md5.1 + $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2 + $DIFF -q tmp.md5.1 tmp.md5.2 +} + isTerminal=false if [ -t 0 ] && [ -t 1 ] then @@ -441,6 +459,8 @@ roundTripTest -g519K 6 # greedy, hash chain roundTripTest -g517K 16 # btlazy2 roundTripTest -g516K 19 # btopt +fileRoundTripTest -g500K + rm tmp* if [ "$1" != "--test-large-data" ]; then @@ -476,4 +496,6 @@ roundTripTest -g50000000 -P94 19 roundTripTest -g99000000 -P99 20 roundTripTest -g6000000000 -P99 1 +fileRoundTripTest -g4193M -P99 1 + rm tmp*