From 4e10ff15f59abfcfc569848b8a53edebfbb937ac Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 5 Mar 2021 18:04:45 -0500 Subject: [PATCH 1/8] Add Tests Checking File Permissions of Created Files --- tests/playTests.sh | 92 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index af80dfdab..358b6b512 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -124,6 +124,23 @@ case "$UNAME" in Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; esac +GET_PERMS="stat -c %a" +case "$UNAME" in + Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %a" ;; +esac + +assertFilePermissions() { + STAT1=$($GET_PERMS "$1") + STAT2=$2 + [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" +} + +assertSamePermissions() { + STAT1=$($GET_PERMS "$1") + STAT2=$($GET_PERMS "$2") + [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" +} + DIFF="diff" case "$UNAME" in SunOS) DIFF="gdiff" ;; @@ -445,6 +462,81 @@ if [ "$isWindows" = false ] ; then rm -rf tmp* fi +println "\n===> zstd created file permissions tests" +if [ "$isWindows" = false ] ; then + rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove + + ORIGINAL_UMASK=$(umask) + umask 0000 + + datagen > tmp1 + datagen > tmp2 + assertFilePermissions tmp1 666 + assertFilePermissions tmp2 666 + + println "test : copy 666 permissions in file -> file compression " + zstd -f tmp1 -o tmp1.zst + assertSamePermissions tmp1 tmp1.zst + println "test : copy 666 permissions in file -> file decompression " + zstd -f -d tmp1.zst -o tmp1.out + assertSamePermissions tmp1.zst tmp1.out + + rm -f tmp1.zst tmp1.out + + println "test : copy 400 permissions in file -> file compression (write to a read-only file) " + chmod 0400 tmp1 + assertFilePermissions tmp1 400 + zstd -f tmp1 -o tmp1.zst + assertSamePermissions tmp1 tmp1.zst + println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " + zstd -f -d tmp1.zst -o tmp1 + assertSamePermissions tmp1.zst tmp1 + + rm -f tmp1.zst tmp1.out + + println "test : check created permissions from stdin input in compression " + zstd -f -o tmp1.zst < tmp1 + assertFilePermissions tmp1.zst 666 + println "test : check created permissions from stdin input in decompression " + zstd -f -d -o tmp1.out < tmp1.zst + assertFilePermissions tmp1.out 666 + + rm -f tmp1.zst tmp1.out + + println "test : check created permissions from multiple inputs in compression " + zstd -f tmp1 tmp2 -o tmp1.zst + assertFilePermissions tmp1.zst 666 + println "test : check created permissions from multiple inputs in decompression " + cp tmp1.zst tmp2.zst + zstd -f -d tmp1.zst tmp2.zst -o tmp1.out + assertFilePermissions tmp1.out 666 + + rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out + + umask 0666 + chmod 0666 tmp1 tmp2 + + println "test : respect umask when copying permissions in file -> file compression " + zstd -f tmp1 -o tmp1.zst + assertFilePermissions tmp1.zst 0 + println "test : respect umask when copying permissions in file -> file decompression " + chmod 0666 tmp1.zst + zstd -f -d tmp1.zst -o tmp1.out + assertFilePermissions tmp1.out 0 + + rm -f tmp1.zst tmp1.out + + println "test : respect umask when compressing from stdin input " + zstd -f -o tmp1.zst < tmp1 + assertFilePermissions tmp1.zst 0 + println "test : respect umask when decompressing from stdin input " + chmod 0666 tmp1.zst + zstd -f -d -o tmp1.out < tmp1.zst + assertFilePermissions tmp1.out 0 + + rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out + umask $ORIGINAL_UMASK +fi if [ -n "$DEVNULLRIGHTS" ] ; then # these tests requires sudo rights, which is uncommon. From b87f97b3ea5a51eb884a62849cf752187bfb8004 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 8 Mar 2021 17:39:14 -0500 Subject: [PATCH 2/8] Create Files with Desired Permissions; Avoid chmod(); Remove UTIL_chmod() --- programs/fileio.c | 58 ++++++++++++++++++++++++----------------------- programs/util.c | 9 -------- programs/util.h | 7 +----- 3 files changed, 31 insertions(+), 43 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index eb06197f1..a16315306 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -25,9 +25,10 @@ ***************************************/ #include "platform.h" /* Large Files support, SET_BINARY_MODE */ #include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ -#include /* fprintf, fopen, fread, _fileno, stdin, stdout */ +#include /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ +#include /* O_WRONLY */ #include #include /* errno */ #include /* INT_MAX */ @@ -73,6 +74,9 @@ #define FNSPACE 30 +/* Default file permissions 0666 (modulated by umask) */ +#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) + /*-************************************* * Macros ***************************************/ @@ -637,7 +641,8 @@ static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFile * @result : FILE* to `dstFileName`, or NULL if it fails */ static FILE* FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, - const char* srcFileName, const char* dstFileName) + const char* srcFileName, const char* dstFileName, + const int mode) { if (prefs->testMode) return NULL; /* do not open file in test mode */ @@ -690,9 +695,11 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, FIO_removeFile(dstFileName); } } - { const int old_umask = UTIL_umask(0177); /* u-x,go-rwx */ - FILE* const f = fopen( dstFileName, "wb" ); - UTIL_umask(old_umask); + { const int fd = open(dstFileName, O_WRONLY|O_CREAT|O_TRUNC, mode); + FILE* f = NULL; + if (fd != -1) { + f = fdopen(fd, "wb"); + } if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); } @@ -1615,23 +1622,24 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, int closeDstFile = 0; int result; stat_t statbuf; - int transfer_permissions = 0; assert(ress.srcFile != NULL); if (ress.dstFile == NULL) { + int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; + if ( strcmp (srcFileName, stdinmark) + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) { + dstFilePermissions = statbuf.st_mode; + } + closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ /* Must only be added after FIO_openDstFile() succeeds. * Otherwise we may delete the destination file if it already exists, * and the user presses Ctrl-C when asked if they wish to overwrite. */ addHandler(dstFileName); - - if ( strcmp (srcFileName, stdinmark) - && UTIL_stat(srcFileName, &statbuf) - && UTIL_isRegularFileStat(&statbuf) ) - transfer_permissions = 1; } result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1651,11 +1659,6 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ ) { FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ - } else if (transfer_permissions) { - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: transferring permissions into dst: %s \n", dstFileName); - UTIL_setFileStat(dstFileName, &statbuf); - } else { - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: do not transfer permissions into dst: %s \n", dstFileName); } } @@ -1827,7 +1830,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, FIO_freeCResources(&ress); return 1; } - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { @@ -2517,13 +2520,19 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, { int result; stat_t statbuf; - int transfer_permissions = 0; int releaseDstFile = 0; if ((ress.dstFile == NULL) && (prefs->testMode==0)) { + int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; + if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) { + dstFilePermissions = statbuf.st_mode; + } + releaseDstFile = 1; - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); if (ress.dstFile==NULL) return 1; /* Must only be added after FIO_openDstFile() succeeds. @@ -2531,11 +2540,6 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, * and the user presses Ctrl-C when asked if they wish to overwrite. */ addHandler(dstFileName); - - if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ - && UTIL_stat(srcFileName, &statbuf) - && UTIL_isRegularFileStat(&statbuf) ) - transfer_permissions = 1; } result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); @@ -2553,8 +2557,6 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ ) { FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ - } else if ( transfer_permissions /* file permissions correctly extracted from src */ ) { - UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ } } @@ -2756,7 +2758,7 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, return 1; } if (!prefs->testMode) { - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { diff --git a/programs/util.c b/programs/util.c index 0d40a394e..ffda78665 100644 --- a/programs/util.c +++ b/programs/util.c @@ -159,15 +159,6 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) return chmod(filename, permissions); } -int UTIL_umask(int mode) { -#if PLATFORM_POSIX_VERSION > 0 - return umask(mode); -#else - /* do nothing, fake return value */ - return mode; -#endif -} - int UTIL_setFileStat(const char *filename, const stat_t *statbuf) { int res = 0; diff --git a/programs/util.h b/programs/util.h index 86743ce57..24cce4480 100644 --- a/programs/util.h +++ b/programs/util.h @@ -22,7 +22,7 @@ extern "C" { #include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */ #include /* size_t, ptrdiff_t */ #include /* stat, utime */ -#include /* stat, chmod, umask */ +#include /* stat, chmod */ #include "../lib/common/mem.h" /* U64 */ @@ -153,11 +153,6 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf); */ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions); -/** - * Wraps umask(). Does nothing when the platform doesn't have that concept. - */ -int UTIL_umask(int mode); - /* * In the absence of a pre-existing stat result on the file in question, these * functions will do a stat() call internally and then use that result to From 1fb10ba831bc01cc19427be5eb02021cfa8d2dca Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 8 Mar 2021 17:49:20 -0500 Subject: [PATCH 3/8] Don't Block Removing File on Being Able to Read It `open()`'s mode bits are only applied to files that are created by the call. If the output file already exists, but is not readable, the `fopen()` would fail, preventing us from removing it, which would mean that the file would not end up with the correct permission bits. It's not clear to me why the `fopen()` is there at all. `UTIL_isRegularFile()` should be sufficient, AFAICT. --- programs/fileio.c | 29 +++++++++++++---------------- tests/playTests.sh | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a16315306..790c10b87 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -669,7 +669,6 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, if (UTIL_isRegularFile(dstFileName)) { /* Check if destination file already exists */ - FILE* const fCheck = fopen( dstFileName, "rb" ); #if !defined(_WIN32) /* this test does not work on Windows : * `NUL` and `nul` are detected as regular files */ @@ -678,22 +677,20 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, dstFileName); } #endif - if (fCheck != NULL) { /* dst file exists, authorization prompt */ - fclose(fCheck); - if (!prefs->overwrite) { - if (g_display_prefs.displayLevel <= 1) { - /* No interaction possible */ - DISPLAY("zstd: %s already exists; not overwritten \n", - dstFileName); - return NULL; - } - DISPLAY("zstd: %s already exists; ", dstFileName); - if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) - return NULL; + if (!prefs->overwrite) { + if (g_display_prefs.displayLevel <= 1) { + /* No interaction possible */ + DISPLAY("zstd: %s already exists; not overwritten \n", + dstFileName); + return NULL; } - /* need to unlink */ - FIO_removeFile(dstFileName); - } } + DISPLAY("zstd: %s already exists; ", dstFileName); + if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) + return NULL; + } + /* need to unlink */ + FIO_removeFile(dstFileName); + } { const int fd = open(dstFileName, O_WRONLY|O_CREAT|O_TRUNC, mode); FILE* f = NULL; diff --git a/tests/playTests.sh b/tests/playTests.sh index 358b6b512..869b4445e 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -513,6 +513,21 @@ if [ "$isWindows" = false ] ; then rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out + println "test : check permissions on pre-existing output file in compression " + chmod 0600 tmp1 + touch tmp1.zst + chmod 0400 tmp1.zst + zstd -f tmp1 -o tmp1.zst + assertFilePermissions tmp1.zst 600 + println "test : check permissions on pre-existing output file in decompression " + chmod 0400 tmp1.zst + touch tmp1.out + chmod 0200 tmp1.out + zstd -f -d tmp1.zst -o tmp1.out + assertFilePermissions tmp1.out 400 + + rm -f tmp1.zst tmp1.out + umask 0666 chmod 0666 tmp1 tmp2 From 018ed6552a3ce4079e19085a0f8dc05f8b6d145a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 8 Mar 2021 18:03:55 -0500 Subject: [PATCH 4/8] Attempt to Fix `stat` Format for BSDs --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 869b4445e..5415656e0 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -126,7 +126,7 @@ esac GET_PERMS="stat -c %a" case "$UNAME" in - Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %a" ;; + Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; esac assertFilePermissions() { From 45c4918ccf3719fc7cacb1a5001ae7d49afd0a70 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 9 Mar 2021 01:24:11 -0500 Subject: [PATCH 5/8] Fix Build for Windows --- programs/fileio.c | 14 +++++++++++++- programs/platform.h | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 790c10b87..2b74feb85 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -74,8 +74,12 @@ #define FNSPACE 30 +#if !defined(_WIN32) /* Default file permissions 0666 (modulated by umask) */ #define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) +#else +#define DEFAULT_FILE_PERMISSIONS (0) +#endif /*-************************************* * Macros @@ -692,7 +696,15 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, FIO_removeFile(dstFileName); } - { const int fd = open(dstFileName, O_WRONLY|O_CREAT|O_TRUNC, mode); + { +#if defined(_WIN32) + /* Windows requires opening the file as a "binary" file to avoid + * mangling. This macro doesn't exist on unix. */ + const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY; +#else + const int openflags = O_WRONLY|O_CREAT|O_TRUNC; +#endif + const int fd = open(dstFileName, openflags, mode); FILE* f = NULL; if (fd != -1) { f = fdopen(fd, "wb"); diff --git a/programs/platform.h b/programs/platform.h index 3b8b505e1..b858e3b48 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -22,6 +22,7 @@ extern "C" { ****************************************/ #if defined(_MSC_VER) # define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_NONSTDC_NO_WARNINGS /* Disable C4996 complaining about posix function names */ # if (_MSC_VER <= 1800) /* 1800 == Visual Studio 2013 */ # define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ From bea1b2ba7057952cb5d4123595f74efb895801cd Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 6 Apr 2021 11:27:27 -0400 Subject: [PATCH 6/8] `rm -f` in playTests.sh --- tests/playTests.sh | 58 +++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 5415656e0..fa748c0cf 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -209,7 +209,7 @@ println "test : compress to stdout" zstd tmp -c > tmpCompressed zstd tmp --stdout > tmpCompressed # long command format println "test : compress to named file" -rm tmpCompressed +rm -f tmpCompressed zstd tmp -o tmpCompressed test -f tmpCompressed # file must be created println "test : force write, correct order" @@ -363,7 +363,7 @@ rm -f tmplog zstd tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" println "test : --rm on stdin" println a | zstd --rm > $INTOVOID # --rm should remain silent -rm tmp +rm -f tmp zstd -f tmp && die "tmp not present : should have failed" test ! -f tmp.zst # tmp.zst should not be created println "test : -d -f do not delete destination when source is not present" @@ -371,7 +371,7 @@ touch tmp # create destination file zstd -d -f tmp.zst && die "attempt to decompress a non existing file" test -f tmp # destination file should still be present println "test : -f do not delete destination when source is not present" -rm tmp # erase source file +rm -f tmp # erase source file touch tmp.zst # create destination file zstd -f tmp && die "attempt to compress a non existing file" test -f tmp.zst # destination file should still be present @@ -385,7 +385,7 @@ println "\n===> decompression only tests " dd bs=1048576 count=1 if=/dev/zero of=tmp zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" $DIFF -s tmp1 tmp -rm tmp* +rm -f tmp* println "\n===> compress multiple files" @@ -432,7 +432,7 @@ zstd -f tmp* test -f tmp1.zst test -f tmp2.zst test -f tmp3.zst -rm tmp1 tmp2 tmp3 +rm -f tmp1 tmp2 tmp3 println "decompress tmp* : " zstd -df ./*.zst test -f tmp1 @@ -447,7 +447,7 @@ zstd -dc tmpall* > tmpdec test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) println "compress multiple files including a missing one (notHere) : " zstd -f tmp1 notHere tmp2 && die "missing file not detected!" -rm tmp* +rm -f tmp* if [ "$isWindows" = false ] ; then @@ -774,16 +774,16 @@ $DIFF helloworld.tmp result.tmp ln -s helloworld.zst helloworld.link.zst $EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp $DIFF helloworld.tmp result.tmp -rm zstdcat -rm result.tmp +rm -f zstdcat +rm -f result.tmp println "testing zcat symlink" ln -sf "$ZSTD_BIN" zcat $EXE_PREFIX ./zcat helloworld.zst > result.tmp $DIFF helloworld.tmp result.tmp $EXE_PREFIX ./zcat helloworld.link.zst > result.tmp $DIFF helloworld.tmp result.tmp -rm zcat -rm ./*.tmp ./*.zstd +rm -f zcat +rm -f ./*.tmp ./*.zstd println "frame concatenation tests completed" @@ -845,7 +845,7 @@ zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated ls -ls tmpSparse* # look at file size and block size on disk $DIFF tmpSparse2M tmpSparseRegenerated -rm tmpSparse* +rm -f tmpSparse* println "\n===> stream-size mode" @@ -991,7 +991,7 @@ then println "- Create dictionary with multithreading enabled" zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict fi -rm tmp* dictionary +rm -f tmp* dictionary println "\n===> fastCover dictionary builder : advanced options " @@ -1033,7 +1033,7 @@ zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary -rm tmp* dictionary +rm -f tmp* dictionary println "\n===> legacy dictionary builder " @@ -1061,7 +1061,7 @@ zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c test -f tmpDict zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c test -f dictionary -rm tmp* dictionary +rm -f tmp* dictionary println "\n===> integrity tests " @@ -1133,7 +1133,7 @@ if [ $GZIPMODE -eq 1 ]; then gzip -t -v tmp.gz gzip -f tmp zstd -d -f -v tmp.gz - rm tmp* + rm -f tmp* else println "gzip binary not detected" fi @@ -1150,7 +1150,7 @@ if [ $GZIPMODE -eq 1 ]; then zstd -f tmp cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* + rm -f tmp* else println "gzip mode not supported" fi @@ -1181,7 +1181,7 @@ if [ $LZMAMODE -eq 1 ]; then lzma -Q -f -k --lzma1 tmp zstd -d -f -v tmp.xz zstd -d -f -v tmp.lzma - rm tmp* + rm -f tmp* println "Creating symlinks" ln -s "$ZSTD_BIN" ./xz ln -s "$ZSTD_BIN" ./unxz @@ -1198,8 +1198,8 @@ if [ $LZMAMODE -eq 1 ]; then ./xz -d tmp.xz lzma -Q tmp ./lzma -d tmp.lzma - rm xz unxz lzma unlzma - rm tmp* + rm -f xz unxz lzma unlzma + rm -f tmp* else println "xz binary not detected" fi @@ -1218,7 +1218,7 @@ if [ $LZMAMODE -eq 1 ]; then cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* + rm -f tmp* else println "xz mode not supported" fi @@ -1237,7 +1237,7 @@ if [ $LZ4MODE -eq 1 ]; then lz4 -t -v tmp.lz4 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout zstd -d -f -v tmp.lz4 - rm tmp* + rm -f tmp* else println "lz4 binary not detected" fi @@ -1253,7 +1253,7 @@ if [ $LZ4MODE -eq 1 ]; then zstd -f tmp cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" - rm tmp* + rm -f tmp* else println "\nlz4 mode not supported" fi @@ -1287,7 +1287,7 @@ rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd datagen > tmp tar cf tmp.tar tmp zstd tmp.tar -o tmp.tzst -rm tmp.tar +rm -f tmp.tar zstd -d tmp.tzst [ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" rm -f tmp.tar tmp.tzst @@ -1366,7 +1366,7 @@ then ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread - rm mt_tmp* + rm -f mt_tmp* println "\n===> ovLog tests " datagen -g2MB > tmp @@ -1390,7 +1390,7 @@ else println "\n===> no multithreading, skipping zstdmt tests " fi -rm tmp* +rm -f tmp* println "\n===> zstd --list/-l single frame tests " datagen > tmp1 @@ -1423,9 +1423,9 @@ zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" -rm $TEST_DATA_FILE -rm $FULL_COMPRESSED_FILE -rm $TRUNCATED_COMPRESSED_FILE +rm -f $TEST_DATA_FILE +rm -f $FULL_COMPRESSED_FILE +rm -f $TRUNCATED_COMPRESSED_FILE println "\n===> zstd --list/-l errors when presented with stdin / no files" zstd -l && die "-l must fail on empty list of files" @@ -1469,7 +1469,7 @@ zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 zstd -b1e10i0 --trace tmp.trace tmp1 zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 -rm tmp* +rm -f tmp* println "\n===> zstd long distance matching tests " From da61918c75bf2eb4519e3f12d21daebac5ba3311 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 6 Apr 2021 11:29:28 -0400 Subject: [PATCH 7/8] Also Pass Mode Bits in on Windows I think in some unix emulation environments on Windows, (cygwin?) mode bits are somehow respected. So we might as well pass them in. Can't hurt. --- programs/fileio.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 2b74feb85..5d7d40ddc 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -74,11 +74,12 @@ #define FNSPACE 30 -#if !defined(_WIN32) /* Default file permissions 0666 (modulated by umask) */ +#if !defined(_WIN32) +/* These macros aren't defined on windows. */ #define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) #else -#define DEFAULT_FILE_PERMISSIONS (0) +#define DEFAULT_FILE_PERMISSIONS (0666) #endif /*-************************************* From 4f9c6fdb7f3f2482cb2db52b1867d051f8bf1c0f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 5 May 2021 13:13:56 -0400 Subject: [PATCH 8/8] Attempt to Fix Windows Build Error --- programs/fileio.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5d7d40ddc..571b35489 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -702,14 +702,19 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, /* Windows requires opening the file as a "binary" file to avoid * mangling. This macro doesn't exist on unix. */ const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY; + const int fd = _open(dstFileName, openflags, mode); + FILE* f = NULL; + if (fd != -1) { + f = _fdopen(fd, "wb"); + } #else const int openflags = O_WRONLY|O_CREAT|O_TRUNC; -#endif const int fd = open(dstFileName, openflags, mode); FILE* f = NULL; if (fd != -1) { f = fdopen(fd, "wb"); } +#endif if (f == NULL) { DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); }