From ccfcc643da9e9f1dbf957731d2d4fc1f3150a98f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 11:28:02 -0700 Subject: [PATCH 01/10] Check if dict is empty before reading first byte --- lib/common/entropy_common.c | 4 +++- lib/legacy/zstd_v01.c | 5 ++++- lib/legacy/zstd_v02.c | 4 +++- lib/legacy/zstd_v03.c | 4 +++- lib/legacy/zstd_v04.c | 4 +++- lib/legacy/zstd_v05.c | 4 +++- lib/legacy/zstd_v06.c | 4 +++- lib/legacy/zstd_v07.c | 4 +++- 8 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index acd966999..6625a8057 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -168,9 +168,11 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, { U32 weightTotal; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index fe9c5ccdd..c115fd822 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -958,13 +958,16 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize) U32 weightTotal; U32 maxBits; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; U32 nextRankStart; void* ptr = DTable+1; HUF_DElt* const dt = (HUF_DElt*)ptr; + if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + iSize = ip[0]; + FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */ //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index de1592e18..cd1ce647e 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -1607,10 +1607,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index caad331d3..2de41f2bd 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -1604,10 +1604,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 05e40aac5..599778b87 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1896,10 +1896,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) /* special header */ diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 96ffceb97..dcaa2ac02 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1873,10 +1873,12 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 weightTotal; U32 tableLog; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; U32 n; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 96a84d3e8..cce629343 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1932,9 +1932,11 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta { U32 weightTotal; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 62285238a..a1f1911a4 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1382,9 +1382,11 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, { U32 weightTotal; const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; + size_t iSize; size_t oSize; + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ if (iSize >= 128) { /* special header */ From 4db751668fbf72cd23205fb1ecd582149a0db210 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 15:49:50 -0700 Subject: [PATCH 02/10] Fix buffer overrun in ZSTD_loadEntropy() The table log set by `FSE_readNCount()` was not checked in `ZSTD_loadEntropy()`. This caused `FSE_buildDTable(dctx->MLTable, ...)` to overwrite the beginning of `dctx->hufTable`. The benchmarks look good, there is no obvious performance regression: > ./zstds/zstd.opt.0 -i10 -b1 -e5 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 268.2 MB/s , 701.0 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 199.5 MB/s , 666.9 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 154.9 MB/s , 655.6 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 128.9 MB/s , 648.4 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 98.4 MB/s , 633.4 MB/s > ./zstds/zstd.opt.2 -i10 -b1 -e5 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 266.1 MB/s , 703.7 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 199.0 MB/s , 666.6 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 156.2 MB/s , 656.2 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 133.2 MB/s , 647.4 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 96.3 MB/s , 633.3 MB/s --- lib/decompress/zstd_decompress.c | 9 ++++++--- lib/legacy/zstd_v05.c | 9 ++++++--- lib/legacy/zstd_v06.c | 9 ++++++--- lib/legacy/zstd_v07.c | 9 ++++++--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 990a64910..83d01f3ad 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1315,25 +1315,28 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c } { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + U32 offcodeMaxValue=MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index dcaa2ac02..7446b8457 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3667,11 +3667,11 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d { size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize; short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSEv05Log; + U32 offcodeMaxValue=MaxOff, offcodeLog; short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSEv05Log; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSEv05Log; + unsigned litlengthMaxValue = MaxLL, litlengthLog; hSize = HUFv05_readDTableX4(dctx->hufTableX4, dict, dictSize); if (HUFv05_isError(hSize)) return ERROR(dictionary_corrupted); @@ -3680,6 +3680,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d offcodeHeaderSize = FSEv05_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); if (FSEv05_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSEv05Log) return ERROR(dictionary_corrupted); errorCode = FSEv05_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); dict = (const char*)dict + offcodeHeaderSize; @@ -3687,12 +3688,14 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d matchlengthHeaderSize = FSEv05_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); if (FSEv05_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSEv05Log) return ERROR(dictionary_corrupted); errorCode = FSEv05_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); dict = (const char*)dict + matchlengthHeaderSize; dictSize -= matchlengthHeaderSize; litlengthHeaderSize = FSEv05_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + if (litlengthLog > LLFSEv05Log) return ERROR(dictionary_corrupted); if (FSEv05_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); errorCode = FSEv05_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index cce629343..403a3a0b3 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3829,9 +3829,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d dictSize -= hSize; { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + U32 offcodeMaxValue=MaxOff, offcodeLog; offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } dict = (const char*)dict + offcodeHeaderSize; @@ -3839,9 +3840,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } dict = (const char*)dict + matchlengthHeaderSize; @@ -3849,9 +3851,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index a1f1911a4..95572b93b 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -4104,27 +4104,30 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si } { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + U32 offcodeMaxValue=MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); { size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } dictPtr += litlengthHeaderSize; From bfd943ace5ff23d12501a744d719cb0e306513b4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 16:55:52 -0700 Subject: [PATCH 03/10] Fix buffer overrun in ZSTD_loadDictEntropyStats() The table log set by `FSE_readNCount()` was not checked in `ZSTD_loadDictEntropyStats()`. This caused `FSE_buildCTable()` to stack/heap overflow in a few places. The benchmarks look good, there is no obvious compression performance regression: > ./zstds/zstd.opt.0 -i10 -b1 -e10 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 271.6 MB/s , 716.8 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 204.8 MB/s , 671.1 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 156.8 MB/s , 658.6 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 136.4 MB/s , 665.3 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 98.9 MB/s , 647.0 MB/s 6#silesia.tar : 211988480 -> 62979643 (3.366), 65.2 MB/s , 670.4 MB/s 7#silesia.tar : 211988480 -> 61974560 (3.421), 44.9 MB/s , 688.2 MB/s 8#silesia.tar : 211988480 -> 61028308 (3.474), 32.4 MB/s , 711.9 MB/s 9#silesia.tar : 211988480 -> 60416751 (3.509), 21.1 MB/s , 718.1 MB/s 10#silesia.tar : 211988480 -> 60174239 (3.523), 22.2 MB/s , 721.8 MB/s > ./compress_zstds/zstd.opt.1 -i10 -b1 -e10 ~/bench/silesia.tar 1#silesia.tar : 211988480 -> 73656930 (2.878), 273.8 MB/s , 722.0 MB/s 2#silesia.tar : 211988480 -> 70162842 (3.021), 203.2 MB/s , 666.6 MB/s 3#silesia.tar : 211988480 -> 66997986 (3.164), 157.4 MB/s , 666.5 MB/s 4#silesia.tar : 211988480 -> 66002591 (3.212), 132.1 MB/s , 661.9 MB/s 5#silesia.tar : 211988480 -> 65008480 (3.261), 96.8 MB/s , 641.6 MB/s 6#silesia.tar : 211988480 -> 62979643 (3.366), 63.1 MB/s , 677.0 MB/s 7#silesia.tar : 211988480 -> 61974560 (3.421), 44.3 MB/s , 678.2 MB/s 8#silesia.tar : 211988480 -> 61028308 (3.474), 33.1 MB/s , 708.9 MB/s 9#silesia.tar : 211988480 -> 60416751 (3.509), 21.5 MB/s , 710.1 MB/s 10#silesia.tar : 211988480 -> 60174239 (3.523), 21.9 MB/s , 723.9 MB/s --- lib/compress/zstd_compress.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e71873b03..c44ef3d41 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2471,25 +2471,28 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ } { short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; + unsigned offcodeMaxValue = MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } From fd9808704799f74a30246754ebf239ac1e8cafc9 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 17 Oct 2016 18:16:57 -0700 Subject: [PATCH 04/10] Fix stack buffer overflow in HUF_readCTable() If `w ==0` on line 153, then `CTable[n].nbBits == tableLog + 1`. Then `nbPerRank[CTable[n].nbBits]` and `valPerRank[CTable[n].nbBits]` are stack buffer overflows. --- lib/compress/huf_compress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index b7d3d77a2..41de7449a 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -155,8 +155,8 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si } } /* fill val */ - { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; { U32 n; for (n=0; n Date: Tue, 18 Oct 2016 14:22:49 -0700 Subject: [PATCH 05/10] Backport fix from commit 9e8b09a Fixes uninitialized memory reads. Full commit hash: 9e8b09a7bd42dd06ee62b33aff215fbb52708d7b --- lib/legacy/zstd_v05.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 7446b8457..a8045aea0 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2967,6 +2967,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, break; } if (litSize > BLOCKSIZE) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (HUFv05_isError(singleStream ? HUFv05_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) : From 7b06ad7a05eabf1e03a9eeff4d6fea7fe63eb0f1 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 18 Oct 2016 14:52:34 -0700 Subject: [PATCH 06/10] Backport fix from commit 125d817 This fixes a read of unitialized memory. Full commit hash: 125d81774fe87a2bc18023d999d8e510678c38fb. --- lib/legacy/zstd_v05.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index a8045aea0..06dba0667 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2944,6 +2944,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, { size_t litSize, litCSize, singleStream=0; U32 lhSize = ((istart[0]) >> 4) & 3; + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ switch(lhSize) { case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ From bb68062c590dbd46905907dd2a63a658040a79d4 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 18 Oct 2016 16:08:52 -0700 Subject: [PATCH 07/10] Unitialized memory read in ZSTD_decodeSeqHeaders() Caused by two things: 1. Not checking that `ip` is in range except for the first byte. 2. `ZSTDv0{5,6}_decodeLiteralsBlock()` could return a value larger than `srcSize`. --- lib/decompress/zstd_decompress.c | 7 +++++-- lib/legacy/zstd_v01.c | 1 + lib/legacy/zstd_v05.c | 9 ++++++++- lib/legacy/zstd_v06.c | 8 ++++++-- lib/legacy/zstd_v07.c | 7 +++++-- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 83d01f3ad..de5f46eef 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -710,10 +710,13 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, { int nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; return 1; } if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else + } else { + if (ip >= iend) return ERROR(srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } } *nbSeqPtr = nbSeq; } diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index c115fd822..217f34aa3 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1536,6 +1536,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx, { size_t rleSize = litbp.origSize; if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall); + if (!srcSize) return ERROR(srcSize_wrong); memset(oend - rleSize, *ip, rleSize); *litStart = oend - rleSize; *litSize = rleSize; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 06dba0667..0b46f0895 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2994,6 +2994,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, lhSize=3; litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litCSize = ((istart[1] & 3) << 8) + istart[2]; + if (litCSize + litSize > srcSize) return ERROR(corruption_detected); errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); if (HUFv05_isError(errorCode)) return ERROR(corruption_detected); @@ -3050,6 +3051,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx, break; case 3: litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ break; } if (litSize > BLOCKSIZE) return ERROR(corruption_detected); @@ -3083,17 +3085,22 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps /* SeqHead */ *nbSeq = *ip++; if (*nbSeq==0) return 1; - if (*nbSeq >= 128) + if (*nbSeq >= 128) { + if (ip >= iend) return ERROR(srcSize_wrong); *nbSeq = ((nbSeq[0]-128)<<8) + *ip++; + } + if (ip >= iend) return ERROR(srcSize_wrong); LLtype = *ip >> 6; Offtype = (*ip >> 4) & 3; MLtype = (*ip >> 2) & 3; if (*ip & 2) { + if (ip+3 > iend) return ERROR(srcSize_wrong); dumpsLength = ip[2]; dumpsLength += ip[1] << 8; ip += 3; } else { + if (ip+2 > iend) return ERROR(srcSize_wrong); dumpsLength = ip[1]; dumpsLength += (ip[0] & 1) << 8; ip += 2; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 403a3a0b3..659b0e8ea 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3185,6 +3185,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx, lhSize=3; litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litCSize = ((istart[1] & 3) << 8) + istart[2]; + if (litCSize + litSize > srcSize) return ERROR(corruption_detected); { size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); if (HUFv06_isError(errorCode)) return ERROR(corruption_detected); @@ -3304,10 +3305,13 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr, { int nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; return 1; } if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else + } else { + if (ip >= iend) return ERROR(srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } } *nbSeqPtr = nbSeq; } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 95572b93b..c8f223af7 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3531,10 +3531,13 @@ size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr, { int nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; return 1; } if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else + } else { + if (ip >= iend) return ERROR(srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } } *nbSeqPtr = nbSeq; } From d760529a058e38bb7a24b5be31060d4b2a020f6f Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 19 Oct 2016 11:19:54 -0700 Subject: [PATCH 08/10] Fix stack buffer overrun when weightTotal == 0 If `weightTotal == 0`, then `BIT_highbit32(weightTotal)` is undefined behavior in the case that it calls `__builtin_clz()`. If `tableLog == HUF_TABLELOG_ABSOLUTEMAX` then we will access one byte beyond the end of the buffer. --- lib/common/entropy_common.c | 1 + lib/legacy/zstd_v01.c | 1 + lib/legacy/zstd_v02.c | 1 + lib/legacy/zstd_v03.c | 1 + lib/legacy/zstd_v04.c | 1 + lib/legacy/zstd_v05.c | 1 + lib/legacy/zstd_v06.c | 1 + lib/legacy/zstd_v07.c | 1 + 8 files changed, 8 insertions(+) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index 6625a8057..18bba0e87 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -200,6 +200,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 217f34aa3..5c36c2108 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1008,6 +1008,7 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize) rankVal[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected; /* get last non-null symbol weight (implied, total must be 2^n) */ maxBits = FSE_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index cd1ce647e..24498fedf 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -1654,6 +1654,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 2de41f2bd..a3bd1da23 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -1651,6 +1651,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 599778b87..60479cb79 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1943,6 +1943,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BIT_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 0b46f0895..2fde052c2 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1912,6 +1912,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ tableLog = BITv05_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 659b0e8ea..3d6512846 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -1971,6 +1971,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BITv06_highbit32(weightTotal) + 1; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index c8f223af7..ace64ede4 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -1421,6 +1421,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, rankStats[huffWeight[n]]++; weightTotal += (1 << huffWeight[n]) >> 1; } } + if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ { U32 const tableLog = BITv07_highbit32(weightTotal) + 1; From f9c9af3c2e84bb61841ff41aa8e5afbd2ca83d7d Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 19 Oct 2016 17:22:08 -0700 Subject: [PATCH 09/10] Reject dictionaries with incomplete entropy tables If a dictionary specifies that a symbol has probability zero in its `matchLength`, `literalLength`, or `offset` FSE table, but the symbol appears when compressing input, the compressor fails. Ensure that dictionaries support all `matchLength`, and `literalLength` codes. They must also support all of the `offset` codes required to represent every possible offset that can appear in the first block. --- lib/compress/zstd_compress.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c44ef3d41..fb12898e8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2448,6 +2448,20 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t } +/* Dictionaries that assign zero probability to symbols that show up causes problems + when FSE encoding. Refuse dictionaries that assign zero probability to symbols + that we may encounter during compression. + NOTE: This behavior is not standard and could be improved in the future. */ +static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); + } + return 0; +} + + /* Dictionary format : Magic == ZSTD_DICT_MAGIC (4 bytes) HUF_writeCTable(256) @@ -2464,17 +2478,19 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; { size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize); if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); dictPtr += hufHeaderSize; } - { short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff, offcodeLog; + { unsigned offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -2484,6 +2500,8 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + /* Every match length code must have non-zero probability */ + CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -2493,6 +2511,8 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + /* Every literal length code must have non-zero probability */ + CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } @@ -2503,6 +2523,13 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dictPtr += 12; + { size_t const maxOffset = (dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */ + /* Calculate minimum offset code required to represent maxOffset */ + unsigned const offcodeMax = ZSTD_highbit32(maxOffset); + /* Every possible supported offset <= dictContentSize + 128 KB must be representable */ + CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + } + cctx->flagStaticTables = 1; return dictPtr - (const BYTE*)dict; } From b2c39a22b028cd4d7967e233e2bf8e1d0bdc4a44 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 24 Oct 2016 14:11:27 -0700 Subject: [PATCH 10/10] Fix compiler narrowing warning --- lib/compress/zstd_compress.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1d4450aa5..6922aab6c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2523,9 +2523,12 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dictPtr += 12; - { size_t const maxOffset = (dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */ - /* Calculate minimum offset code required to represent maxOffset */ - unsigned const offcodeMax = ZSTD_highbit32(maxOffset); + { U32 offcodeMax = MaxOff; + if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */ + /* Calculate minimum offset code required to represent maxOffset */ + offcodeMax = ZSTD_highbit32(maxOffset); + } /* Every possible supported offset <= dictContentSize + 128 KB must be representable */ CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); }