These functions make it possible to estimate memory usage of a future {D,C}Ctx, before its creation. - The objective is to guide decision before allocation. - ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large". - If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. + ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one. + It will also consider src size to be arbitrarily "large", which is worst case. + If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. Note : CCtx estimation is only correct for single-threaded compression
ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large". - If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. +
ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one. + It will also consider src size to be arbitrarily "large", which is worst case. + If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. Note : CStream estimation is only correct for single-threaded compression. ZSTD_DStream memory budget depends on window Size. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9ea866bf2..b396311d3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -4009,30 +4009,30 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { -{ /* "default" */ +{ /* "default" - guarantees a monotonically increasing memory budget */ /* W, C, H, S, L, TL, strat */ { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ - { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/ - { 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/ - { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ - { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ - { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */ + { 20, 17, 18, 1, 5, 16, ZSTD_dfast }, /* level 4 */ + { 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */ { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ - { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ - { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ - { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ + { 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ { 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */ { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ { 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */ - { 26, 26, 23, 7, 3,256, ZSTD_btultra }, /* level 21 */ + { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ }, { /* for srcSize <= 256 KB */ @@ -4115,6 +4115,43 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, }; +/* This function just controls + * the monotonic memory budget increase of ZSTD_defaultCParameters[0]. + * Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled + */ +MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void) +{ +# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c))) +# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field) +# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \ + assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \ + assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \ + } + + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20); + ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21); + assert(ZSTD_maxCLevel()==22); +} + /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. * Size values are optional, provide 0 if not known or unused */ @@ -4123,6 +4160,15 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l size_t const addedSize = srcSizeHint ? 0 : 500; U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) + static int g_monotonicTest = 1; + if (g_monotonicTest) { + ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(); + g_monotonicTest=0; + } +#endif + if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */ if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel]; diff --git a/lib/zstd.h b/lib/zstd.h index 700a22f7b..59a09be6f 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -497,9 +497,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); /*! ZSTD_estimate*() : * These functions make it possible to estimate memory usage * of a future {D,C}Ctx, before its creation. - * The objective is to guide decision before allocation. - * ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large". - * If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. + * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * Note : CCtx estimation is only correct for single-threaded compression */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); @@ -507,8 +507,9 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cPa ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); /*! ZSTD_estimate?StreamSize() : - * ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large". - * If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * Note : CStream estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size.