diff --git a/lib/Makefile b/lib/Makefile index 6bdf2f876..ce66ebbe0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -118,8 +118,8 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h - @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zdict.h + @install -m 644 zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @install -m 644 zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed uninstall: @@ -129,8 +129,8 @@ uninstall: @[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h - @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h - @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 4d737605a..65fea546e 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -80,6 +80,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->lit = ssPtr->litStart; ssPtr->litLength = ssPtr->litLengthStart; ssPtr->matchLength = ssPtr->matchLengthStart; + ssPtr->longLengthID = 0; } @@ -183,7 +184,7 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t /* resize params, to use less memory when necessary */ { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; - if (rSize < (1<windowLog > srcLog) params->windowLog = srcLog; } } @@ -224,7 +225,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (1<seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); - zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (1<seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1))); zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1)); zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; zc->seqStore.litLengthSum = 0; @@ -620,14 +621,16 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; - U16* const llTable = seqStorePtr->litLengthStart; + const U16* const llTable = seqStorePtr->litLengthStart; BYTE* const llCodeTable = seqStorePtr->llCodeStart; size_t u; for (u=0; ulongLength; llTable[u] = (U16)ll; } + U32 const ll = llTable[u]; llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll]; - } } + } + if (seqStorePtr->longLengthID==1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + } /* Offset codes */ { const U32* const offsetTable = seqStorePtr->offsetStart; @@ -646,14 +649,16 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; const BYTE ML_deltaCode = 36; - U16* const mlTable = seqStorePtr->matchLengthStart; + const U16* const mlTable = seqStorePtr->matchLengthStart; BYTE* const mlCodeTable = seqStorePtr->mlCodeStart; size_t u; for (u=0; ulongLength; mlTable[u] = (U16)ml; } + U32 const ml = mlTable[u]; mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml]; - } } + } + if (seqStorePtr->longLengthID==2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + } } @@ -738,8 +743,8 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, Offtype = FSE_ENCODING_RLE; } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { Offtype = FSE_ENCODING_STATIC; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (Offbits-1)))) { - FSE_buildCTable_raw(CTable_OffsetBits, Offbits); + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) { + FSE_buildCTable(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog); Offtype = FSE_ENCODING_RAW; } else { size_t nbSeq_1 = nbSeq; @@ -864,15 +869,15 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B seqStorePtr->lit += litLength; /* literal Length */ - if (litLength>=65535) { *(seqStorePtr->litLength++) = 65535; seqStorePtr->longLength = (U32)litLength; } - else *seqStorePtr->litLength++ = (U16)litLength; + if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->litLength - seqStorePtr->litLengthStart); } + *seqStorePtr->litLength++ = (U16)litLength; /* match offset */ *(seqStorePtr->offset++) = (U32)offsetCode + 1; /* match Length */ - if (matchCode>=65535) { *(seqStorePtr->matchLength++) = 65535; seqStorePtr->longLength = (U32)matchCode; } - else *seqStorePtr->matchLength++ = (U16)matchCode; + if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->matchLength - seqStorePtr->matchLengthStart); } + *seqStorePtr->matchLength++ = (U16)matchCode; } @@ -1508,13 +1513,13 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( - /* *********************** * Hash Chain *************************/ #define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] + /* Update chains up to ip (excluded) Assumption : always within prefix (ie. not within extDict) */ FORCE_INLINE @@ -2251,7 +2256,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); + U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; // printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 2dbe5ce89..60595b45c 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -500,34 +500,6 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, @return : nb bytes read from src, or an error code if it fails, testable with ZSTD_isError() */ -FORCE_INLINE size_t ZSTD_buildSeqTableOff(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog, - const void* src, size_t srcSize) -{ - U32 max = (1< max, data is corrupted */ - return 1; - case FSE_ENCODING_RAW : - FSE_buildDTable_raw(DTable, rawBits); - return 0; - case FSE_ENCODING_STATIC: - return 0; - default : /* impossible */ - case FSE_ENCODING_DYNAMIC : - { U32 tableLog; - S16 norm[MaxSeq+1]; - size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSE_isError(headerSize)) return ERROR(corruption_detected); - if (tableLog > maxLog) return ERROR(corruption_detected); - FSE_buildDTable(DTable, norm, max, tableLog); - return headerSize; - } } -} - - FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U32 maxLog, const void* src, size_t srcSize, const S16* defaultNorm, U32 defaultLog) @@ -594,7 +566,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); ip += bhSize; } - { size_t const bhSize = ZSTD_buildSeqTableOff(DTableOffb, Offtype, Offbits, OffFSELog, ip, iend-ip); + { size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog); if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); ip += bhSize; } @@ -650,7 +622,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, - 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 }; + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 }; /* sequence */ { size_t offset; @@ -660,7 +632,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); } - + if (offset < ZSTD_REP_NUM) { if (llCode == 0 && offset <= 1) offset = 1-offset; @@ -678,7 +650,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) } else { offset -= ZSTD_REP_MOVE; seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; } seq->offset = offset; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 34d8b5441..31a99cbde 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -104,11 +104,10 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define REPCODE_STARTVALUE 1 #define Litbits 8 -#define Offbits 5 #define MaxLit ((1<litSum = (2<litLengthSum = MaxLL+1; ssPtr->matchLengthSum = MaxML+1; - ssPtr->offCodeSum = (1<offCodeSum = (MaxOff+1); ssPtr->matchSum = (2<= ZSTD_REP_NUM) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; @@ -860,7 +860,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (opt[cur].off >= ZSTD_REP_NUM) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 8d0e8d280..e804a5083 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -57,7 +57,7 @@ extern "C" { /*-************************************* * Types ***************************************/ -#define ZSTD_WINDOWLOG_MAX (MEM_32bits() ? 25 : 27) +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) #define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CHAINLOG_MIN 4