sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-04 00:04:23 -05:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3479 from felixhandte/faster-file-ops

Browse Source 
Use `f`-variants of `chmod()` and `chown()`
This commit is contained in:
Felix Handte 2023-02-16 13:07:34 -05:00 committed by GitHub
commit 1c42844668
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 152 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
programs/fileio.c
View File

@ -1681,6 +1681,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
int result;
int transferStat = 0;
FILE *dstFile;
int dstFd = -1;
assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
@ -1696,6 +1697,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
if (dstFile==NULL) return 1; /* could not open dstFileName */
dstFd = fileno(dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
@ -1709,14 +1711,20 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
if (closeDstFile) {
clearHandler();
if (transferStat) {
UTIL_setFDStat(dstFd, dstFileName, srcFileStat);
}
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName);
if (AIO_WritePool_closeFile(ress.writeCtx)) { /* error closing file */
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
}
if (transferStat) {
UTIL_setFileStat(dstFileName, srcFileStat);
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
@ -2540,6 +2548,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
int result;
int releaseDstFile = 0;
int transferStat = 0;
int dstFd = 0;
if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
FILE *dstFile;
@ -2555,6 +2564,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
if (dstFile==NULL) return 1;
dstFd = fileno(dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
/* Must only be added after FIO_openDstFile() succeeds.
@ -2568,13 +2578,18 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
if (releaseDstFile) {
clearHandler();
if (transferStat) {
UTIL_setFDStat(dstFd, dstFileName, srcFileStat);
}
if (AIO_WritePool_closeFile(ress.writeCtx)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result = 1;
}
if (transferStat) {
UTIL_setFileStat(dstFileName, srcFileStat);
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */

92
programs/util.c
View File

@ -102,6 +102,17 @@ UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
#define chmod _chmod
#endif
#ifndef ZSTD_HAVE_FCHMOD
#if PLATFORM_POSIX_VERSION >= 199309L
#define ZSTD_HAVE_FCHMOD
#endif
#endif
#ifndef ZSTD_HAVE_FCHOWN
#if PLATFORM_POSIX_VERSION >= 200809L
#define ZSTD_HAVE_FCHOWN
#endif
#endif
/*-****************************************
* Console log
@ -147,21 +158,38 @@ void UTIL_traceFileStat(void)
g_traceFileStat = 1;
}
int UTIL_stat(const char* filename, stat_t* statbuf)
int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_stat(%s)", filename);
UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename);
#if defined(_MSC_VER)
ret = !_stat64(filename, statbuf);
if (fd >= 0) {
ret = !_fstat64(fd, statbuf);
} else {
ret = !_stat64(filename, statbuf);
}
#elif defined(__MINGW32__) && defined (__MSVCRT__)
ret = !_stati64(filename, statbuf);
if (fd >= 0) {
ret = !_fstati64(fd, statbuf);
} else {
ret = !_stati64(filename, statbuf);
}
#else
ret = !stat(filename, statbuf);
if (fd >= 0) {
ret = !fstat(fd, statbuf);
} else {
ret = !stat(filename, statbuf);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_stat(const char* filename, stat_t* statbuf)
{
return UTIL_fstat(-1, filename, statbuf);
}
int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
@ -183,11 +211,16 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
/* like chmod, but avoid changing permission of /dev/null */
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
{
return UTIL_fchmod(-1, filename, statbuf, permissions);
}
int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions)
{
stat_t localStatBuf;
UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
if (statbuf == NULL) {
if (!UTIL_stat(filename, &localStatBuf)) {
if (!UTIL_fstat(fd, filename, &localStatBuf)) {
UTIL_TRACE_RET(0);
return 0;
}
@ -197,9 +230,20 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
UTIL_TRACE_RET(0);
return 0; /* pretend success, but don't change anything */
}
UTIL_TRACE_CALL("chmod");
#ifdef ZSTD_HAVE_FCHMOD
if (fd >= 0) {
int ret;
UTIL_TRACE_CALL("fchmod");
ret = fchmod(fd, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
} else
#endif
{
int const ret = chmod(filename, permissions);
int ret;
UTIL_TRACE_CALL("chmod");
ret = chmod(filename, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
@ -236,19 +280,21 @@ int UTIL_utime(const char* filename, const stat_t *statbuf)
}
int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
{
return UTIL_setFDStat(-1, filename, statbuf);
}
int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf)
{
int res = 0;
stat_t curStatBuf;
UTIL_TRACE_CALL("UTIL_setFileStat(%s)", filename);
UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename);
if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
UTIL_TRACE_RET(-1);
return -1;
}
/* set access and modification times */
res += UTIL_utime(filename, statbuf);
/* Mimic gzip's behavior:
*
* "Change the group first, then the permissions, then the owner.
@ -258,13 +304,27 @@ int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
* setgid bits." */
#if !defined(_WIN32)
res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
#ifdef ZSTD_HAVE_FCHOWN
if (fd >= 0) {
res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */
} else
#endif
{
res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
}
#endif
res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
#ifdef ZSTD_HAVE_FCHOWN
if (fd >= 0) {
res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */
} else
#endif
{
res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
}
#endif
errno = 0;

13
programs/util.h
View File

@ -126,15 +126,25 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const
/**
* Calls platform's equivalent of stat() on filename and writes info to statbuf.
* Returns success (1) or failure (0).
*
* UTIL_fstat() is like UTIL_stat() but takes an optional fd that refers to the
* file in question. It turns out that this can be meaningfully faster. If fd is
* -1, behaves just like UTIL_stat() (i.e., falls back to using the filename).
*/
int UTIL_stat(const char* filename, stat_t* statbuf);
int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf);
/**
* Instead of getting a file's stats, this updates them with the info in the
* provided stat_t. Currently sets owner, group, atime, and mtime. Will only
* update this info for regular files.
*
* UTIL_setFDStat() also takes an fd, and will preferentially use that to
* indicate which file to modify, If fd is -1, it will fall back to using the
* filename.
*/
int UTIL_setFileStat(const char* filename, const stat_t* statbuf);
int UTIL_setFDStat(const int fd, const char* filename, const stat_t* statbuf);
/**
* Set atime to now and mtime to the st_mtim in statbuf.
@ -159,8 +169,11 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf);
* Like chmod(), but only modifies regular files. Provided statbuf may be NULL,
* in which case this function will stat() the file internally, in order to
* check whether it should be modified.
*
* If fd is -1, fd is ignored and the filename is used.
*/
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions);
int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions);
/*
* In the absence of a pre-existing stat result on the file in question, these

26
tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
View File

@ -3,40 +3,40 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file, file.zst)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_setFileStat(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_setFileStat(4, file.zst)
Trace:FileStat: > UTIL_stat(4, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_chmod(file.zst, 0642)
Trace:FileStat: > chmod
Trace:FileStat: > fchmod
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: < 0

10
tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
View File

@ -5,20 +5,20 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537

10
tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
View File

@ -3,22 +3,22 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1

6
tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
View File

@ -5,14 +5,14 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1

22
tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
View File

@ -5,34 +5,34 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file.zst, file)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_setFileStat(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_setFileStat(4, file)
Trace:FileStat: > UTIL_stat(4, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_chmod(file, 0642)
Trace:FileStat: > chmod
Trace:FileStat: > fchmod
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: < 0

6
tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
View File

@ -5,14 +5,14 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0

8
tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
View File

@ -3,18 +3,18 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 1

4
tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
View File

@ -5,10 +5,10 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0