From 6653321ba34f593e04dca5f57b3a22f9bdc91675 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 18 Jun 2020 09:28:18 -0700 Subject: [PATCH 1/3] passing streamSrcSize when set --- programs/fileio.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e0f7fdef2..390e8def5 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -839,8 +839,10 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, /* need to update memLimit before calling createDictBuffer * because of memLimit check inside it */ - if (prefs->patchFromMode) - FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), maxSrcFileSize, cLevel); + if (prefs->patchFromMode) { + unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; + FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); + } ress.dstBuffer = malloc(ress.dstBufferSize); ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ if (!ress.srcBuffer || !ress.dstBuffer) From b06710889ff77ad10ced072232b1e4fa16468a20 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 18 Jun 2020 09:31:06 -0700 Subject: [PATCH 2/3] error when not using --stream-size on pipe --- programs/fileio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 390e8def5..5e9d622f9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -772,6 +772,8 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, { unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX); + if (maxSize == UTIL_FILESIZE_UNKNOWN) + EXM_THROW(42, "Using --patch-from with stdin requires --stream-size"); assert(maxSize != UTIL_FILESIZE_UNKNOWN); if (maxSize > maxWindowSize) EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB)); From de48f35306b72fd7482b30c457a608d8e9d0651d Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Thu, 18 Jun 2020 09:36:36 -0700 Subject: [PATCH 3/3] adding --patch-from --stream-size test --- tests/playTests.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index 7a2689cd3..c3168d53c 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -1304,6 +1304,13 @@ datagen -g5000000 > tmp_patch zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" rm -rf tmp* +println "\n===> patch-from --stream-size test" +datagen -g1000 -P50 > tmp_dict +datagen -g1000 -P10 > tmp_patch +cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die +cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff +rm -rf tmp* + println "\n===> large files tests " roundTripTest -g270000000 1