From 3d523c741be041f17c28e43b89ab6dfcaee281d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 5 Jun 2018 11:23:18 -0700 Subject: [PATCH 1/6] added workSpaceTooLarge and workSpaceWasteful also : slightly increased speed of test fuzzer.16 --- lib/compress/zstd_compress.c | 73 ++++++++++++++++++--------- lib/compress/zstd_compress_internal.h | 6 ++- tests/fuzzer.c | 12 +++-- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7a30d5525..b87ae3819 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -695,7 +695,8 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) /** ZSTD_clampCParams() : * make CParam values within valid range. * @return : valid CParams */ -static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) { # define CLAMP(val,min,max) { \ if (val (U32)ZSTD_btultra) cParams.strategy = ZSTD_btultra; + if ((U32)(cParams.targetLength) < ZSTD_TARGETLENGTH_MIN) + cParams.targetLength = ZSTD_TARGETLENGTH_MIN; + CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra); return cParams; } @@ -723,8 +725,11 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) optimize `cPar` for a given input (`srcSize` and `dictSize`). mostly downsizing to reduce memory consumption and initialization latency. Both `srcSize` and `dictSize` are optional (use 0 if unknown). - Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */ -ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) + Note : cPar is assumed validated. Use ZSTD_checkCParams() to ensure this condition. */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) { static const U64 minSrcSize = 513; /* (1<<9) + 1 */ static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); @@ -756,13 +761,18 @@ ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameter return cPar; } -ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) { cPar = ZSTD_clampCParams(cPar); return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); } -static size_t ZSTD_sizeof_matchState(ZSTD_compressionParameters const* cParams, const U32 forCCtx) +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const U32 forCCtx) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; @@ -848,12 +858,14 @@ size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); } -static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) { +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); return ZSTD_estimateCStreamSize_usingCParams(cParams); } -size_t ZSTD_estimateCStreamSize(int compressionLevel) { +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ int level; size_t memBudget = 0; for (level=1; level<=compressionLevel; level++) { @@ -1042,10 +1054,18 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, return ptr; } +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as ZSTD_WORKSPACETOOLARGE_FACTOR times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_MAX 128 /* when workspace is continuously too large + * at least that number of times, + * context's memory usage is actually wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ + /*! ZSTD_resetCCtx_internal() : note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, - ZSTD_CCtx_params params, U64 pledgedSrcSize, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { @@ -1057,8 +1077,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (ZSTD_equivalentParams(zc->appliedParams, params, zc->inBuffSize, zc->blockSize, zbuff, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%u)", - zc->appliedParams.cParams.windowLog, (U32)zc->blockSize); + DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)", + zc->appliedParams.cParams.windowLog, zc->blockSize); return ZSTD_continueCCtx(zc, params, pledgedSrcSize); } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); @@ -1069,8 +1089,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); assert(params.ldmParams.hashEveryLog < 32); - zc->ldmState.hashPower = - ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); + zc->ldmState.hashPower = ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); } { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); @@ -1082,7 +1101,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; + void* ptr; /* used to partition workSpace */ /* Check if workSpace is large enough, alloc a new one if needed */ { size_t const entropySpace = HUF_WORKSPACE_SIZE; @@ -1094,14 +1113,19 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const neededSpace = entropySpace + blockStateSpace + ldmSpace + ldmSeqSpace + matchStateSize + tokenSpace + bufferSpace; - DEBUGLOG(4, "Need %uKB workspace, including %uKB for match state, and %uKB for buffers", - (U32)(neededSpace>>10), (U32)(matchStateSize>>10), (U32)(bufferSpace>>10)); - DEBUGLOG(4, "windowSize: %u - blockSize: %u", (U32)windowSize, (U32)blockSize); - if (zc->workSpaceSize < neededSpace) { /* too small : resize */ - DEBUGLOG(4, "Need to update workSpaceSize from %uK to %uK", - (unsigned)(zc->workSpaceSize>>10), - (unsigned)(neededSpace>>10)); + int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; + int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; + int const workSpaceWasteful = 0 && workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX); + DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", + neededSpace>>10, matchStateSize>>10, bufferSpace>>10); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0; + if (workSpaceTooSmall || workSpaceWasteful) { + DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB", + zc->workSpaceSize >> 10, + neededSpace >> 10); /* static cctx : no resize, error out */ if (zc->staticSize) return ERROR(memory_allocation); @@ -1110,9 +1134,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; + zc->workSpaceTooLarge = 0; ptr = zc->workSpace; - /* Statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workSpace; diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index fd072f309..b2475aaef 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -27,6 +27,7 @@ extern "C" { #endif + /*-************************************* * Constants ***************************************/ @@ -37,7 +38,8 @@ extern "C" { It's not a big deal though : candidate will just be sorted again. Additionnally, candidate position 1 will be lost. But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. - The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy */ + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy + Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ /*-************************************* @@ -204,6 +206,8 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; U32 dictID; + + int workSpaceTooLarge; void* workSpace; size_t workSpaceSize; size_t blockSize; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b0c425e1e..8dabcddbc 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -450,14 +450,16 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3d : large window log smaller data : ", testNb++); + DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_parameters params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); - size_t const nbCompressions = (1U << 31) / CNBuffSize + 1; - size_t i; + ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0); + size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 1; /* ensure U32 overflow protection is triggered */ + size_t cnb; + assert(cctx != NULL); params.fParams.contentSizeFlag = 0; params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; - for (i = 0; i < nbCompressions; ++i) { + for (cnb = 0; cnb < nbCompressions; ++cnb) { + DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions); CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); } From f7392f3dc95e26922763bc5817ab98b0ee4fa696 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 5 Jun 2018 14:53:28 -0700 Subject: [PATCH 2/6] added test case --- doc/zstd_manual.html | 8 ++++---- lib/compress/zstd_compress.c | 6 ++++-- tests/fuzzer.c | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index aae6d0f37..3d107746b 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -82,7 +82,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);

`src` should point to the start of a ZSTD encoded frame. `srcSize` must be at least as large as the frame header. hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. - @return : - decompressed size of the frame in `src`, if known + @return : - decompressed size of `src` frame content, if known - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) note 1 : a 0 return value means the frame is valid but "empty". @@ -92,7 +92,8 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); Optionally, application can rely on some implicit limit, as ZSTD_decompress() only needs an upper bound of decompressed size. (For example, data could be necessarily cut into blocks <= 16 KB). - note 3 : decompressed size is always present when compression is done with ZSTD_compress() + note 3 : decompressed size is always present when compression is completed using single-pass functions, + such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). note 4 : decompressed size can be very large (64-bits value), potentially larger than what local system can handle as a single memory segment. In which case, it's necessary to use streaming mode to decompress data. @@ -107,8 +108,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); Both functions work the same way, but ZSTD_getDecompressedSize() blends "empty", "unknown" and "error" results to the same return value (0), while ZSTD_getFrameContentSize() gives them separate return values. - `src` is the start of a zstd compressed frame. - @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. + @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise.


Helper functions

#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b87ae3819..d9e3f8143 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1079,6 +1079,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                 zbuff, pledgedSrcSize)) {
             DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
                         zc->appliedParams.cParams.windowLog, zc->blockSize);
+            zc->workSpaceTooLarge += (zc->workSpaceTooLarge > 0);   /* if it was too large, it still is */
             return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
     DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
@@ -1116,12 +1117,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
             int const workSpaceTooSmall = zc->workSpaceSize < neededSpace;
             int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace;
-            int const workSpaceWasteful = 0 && workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
+            int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
+            zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
+
             DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",
                         neededSpace>>10, matchStateSize>>10, bufferSpace>>10);
             DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
 
-            zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
             if (workSpaceTooSmall || workSpaceWasteful) {
                 DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB",
                             zc->workSpaceSize >> 10,
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 8dabcddbc..6e60b74cb 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -450,6 +450,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    /* this test is really too long, and should be made faster */
     DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
         ZSTD_parameters params = ZSTD_getParams(-9, ZSTD_CONTENTSIZE_UNKNOWN, 0);
@@ -467,6 +468,40 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
+    {   ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
+        assert(largeCCtx != NULL);
+        CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) );   /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
+        CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
+        {   size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx);   /* size of context must be measured after compression */
+            {   ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
+                assert(smallCCtx != NULL);
+                CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
+                {   size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
+                    DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
+                                largeCCtxSize>>10, smallCCtxSize>>10);
+                    assert(largeCCtxSize > 32* smallCCtxSize);  /* note : "too large" definition is handled within zstd_compress.c .
+                                                                 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
+                }
+                ZSTD_freeCCtx(smallCCtx);
+            }
+            {   U32 const maxNbAttempts = 1100;   /* nb of usages before triggering size down is handled within zstd_compress.c.
+                                                   * currently defined as 128x, but could be adjusted in the future.
+                                                   * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
+                U32 u;
+                for (u=0; u
Date: Wed, 6 Jun 2018 15:00:27 -0700
Subject: [PATCH 3/6] changed member name to workSpaceOversizedDuration

---
 lib/compress/zstd_compress.c          | 8 ++++----
 lib/compress/zstd_compress_internal.h | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d9e3f8143..341636604 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1079,7 +1079,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                                 zbuff, pledgedSrcSize)) {
             DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
                         zc->appliedParams.cParams.windowLog, zc->blockSize);
-            zc->workSpaceTooLarge += (zc->workSpaceTooLarge > 0);   /* if it was too large, it still is */
+            zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0);   /* if it was too large, it still is */
             return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
     DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
@@ -1117,8 +1117,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
             int const workSpaceTooSmall = zc->workSpaceSize < neededSpace;
             int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace;
-            int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceTooLarge > ZSTD_WORKSPACETOOLARGE_MAX);
-            zc->workSpaceTooLarge = workSpaceTooLarge ? zc->workSpaceTooLarge+1 : 0;
+            int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAX);
+            zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0;
 
             DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",
                         neededSpace>>10, matchStateSize>>10, bufferSpace>>10);
@@ -1136,7 +1136,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
                 zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
                 if (zc->workSpace == NULL) return ERROR(memory_allocation);
                 zc->workSpaceSize = neededSpace;
-                zc->workSpaceTooLarge = 0;
+                zc->workSpaceOversizedDuration = 0;
                 ptr = zc->workSpace;
 
                 /* Statically sized space.
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index b2475aaef..726eb1393 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -207,7 +207,7 @@ struct ZSTD_CCtx_s {
     ZSTD_CCtx_params appliedParams;
     U32   dictID;
 
-    int workSpaceTooLarge;
+    int workSpaceOversizedDuration;
     void* workSpace;
     size_t workSpaceSize;
     size_t blockSize;

From f1ea383f4514d16937a30f600df1505d67f5548b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 6 Jun 2018 15:04:12 -0700
Subject: [PATCH 4/6] context can be sized down even with constant parameters

when parameters are "equivalent",
the context is re-used in continue mode,
hence needed workspace size is not recalculated.
This incidentally also evades the size-down check and action.

This patch intercepts the "continue mode"
so that the size-down check and action is actually triggered.
---
 lib/compress/zstd_compress.c | 13 +++++++------
 tests/fuzzer.c               |  3 +--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 341636604..e0c29ac44 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -1054,10 +1054,10 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
     return ptr;
 }
 
-#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as ZSTD_WORKSPACETOOLARGE_FACTOR times larger than needed */
-#define ZSTD_WORKSPACETOOLARGE_MAX 128  /* when workspace is continuously too large
-                                         * at least that number of times,
-                                         * context's memory usage is actually wasteful,
+#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */
+#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128  /* when workspace is continuously too large
+                                         * during at least this number of times,
+                                         * context's memory usage is considered wasteful,
                                          * because it's sized to handle a worst case scenario which rarely happens.
                                          * In which case, resize it down to free some memory */
 
@@ -1080,7 +1080,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
             DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
                         zc->appliedParams.cParams.windowLog, zc->blockSize);
             zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0);   /* if it was too large, it still is */
-            return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
+            if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION)
+                return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
     }   }
     DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
 
@@ -1117,7 +1118,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
             int const workSpaceTooSmall = zc->workSpaceSize < neededSpace;
             int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace;
-            int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAX);
+            int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION);
             zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0;
 
             DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers",
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 6e60b74cb..6660fb062 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -490,8 +490,7 @@ static int basicUnitTests(U32 seed, double compressibility)
                                                    * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
                 U32 u;
                 for (u=0; u
Date: Wed, 6 Jun 2018 18:15:57 -0700
Subject: [PATCH 5/6] added ld -v to aarch64fuzz

to better understand the travis CI linking bug
---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 5756e630c..4695ed9f5 100644
--- a/Makefile
+++ b/Makefile
@@ -183,6 +183,7 @@ armfuzz: clean
 	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 aarch64fuzz: clean
+	ld -v
 	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 ppcfuzz: clean

From 3cdb342bd5bb771b1e165f34daeb47e449d2cbdf Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 7 Jun 2018 12:18:47 -0700
Subject: [PATCH 6/6] disable aarch64 test on travis CI

there's a bug on travis' ld
> collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped

I cannot reproduce it.
Note that travis' ld version is 2.24
while the one in my Linux VM is 2.26.1.
ld is part of binutils, so it's pretty hard to change.

I would expect the bug to no longer be triggered after some random code change.
To be re-enabled later.
---
 .travis.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a9c1db525..a8426a8da 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,11 @@ matrix:
     - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
 
     - env: Cmd='make arminstall && make armfuzz'
-    - env: Cmd='make arminstall && make aarch64fuzz'
+# Following test is disabled, as there is a bug in Travis' ld
+# preventing aarch64 compilation to complete.
+# > collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped
+# to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug
+#    - env: Cmd='make arminstall && make aarch64fuzz'
     - env: Cmd='make ppcinstall && make ppcfuzz'
     - env: Cmd='make ppcinstall && make ppc64fuzz'
     - env: Cmd='make -j uasanregressiontest'