sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-04 00:02:33 -04:00
Code Issues Projects Releases Wiki Activity

Revert "pass dictionary loading method as parameter"

Browse Source 
This reverts commit 821fc567f93a415e9fbe856271ccd452ee7acf07.
This commit is contained in:
Yann Collet 2025-02-05 18:47:26 -08:00
parent c7cd7dc04b
commit 23e5f80390
6 changed files with 48 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/compress/zstd_compress.c
View File

@ -1354,12 +1354,10 @@ size_t ZSTD_CCtx_refPrefix_advanced(
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't ref a prefix when ctx not in init stage.");
ZSTD_clearAllDicts(cctx);
if (prefixSize > 0) {
RETURN_ERROR_IF(prefix == NULL, dictionary_wrong, "Invalid prefix pointer");
if (prefix != NULL && prefixSize > 0) {
cctx->prefixDict.dict = prefix;
cctx->prefixDict.dictSize = prefixSize;
cctx->prefixDict.dictContentType = dictContentType;
cctx->prefixDict.loadMethod = ZSTD_dlm_byRef;
}
return 0;
}
@ -3068,7 +3066,7 @@ ZSTD_entropyCompressSeqStore(
/* ZSTD_selectBlockCompressor() :
* Not static, but internal use only (used by long distance matcher)
* assumption : strat is a valid strategy */
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_DictMode_e dictMode)
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
{
static const ZSTD_BlockCompressor_f blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
{ ZSTD_compressBlock_fast /* default for 0 */,
@ -3300,7 +3298,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
}
/* select and store sequences */
{ ZSTD_DictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
{ ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
size_t lastLLSize;
{ int i;
for (i = 0; i < ZSTD_REP_NUM; ++i)
@ -6353,7 +6351,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
size_t inSize)
{
ZSTD_CCtx_params params = cctx->requestedParams;
ZSTD_PrefixDict const prefixDict = cctx->prefixDict;
ZSTD_prefixDict const prefixDict = cctx->prefixDict;
FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
@ -6409,7 +6407,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers);
FORWARD_IF_ERROR( ZSTDMT_initCStream_internal(
cctx->mtctx,
prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, prefixDict.loadMethod,
prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0;
cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize;

27
lib/compress/zstd_compress_internal.h
View File

@ -12,8 +12,8 @@
* that shall **only** be used by modules within lib/compress.
*/
#ifndef ZSTD_COMPRESS_INTERNAL_H
#define ZSTD_COMPRESS_INTERNAL_H
#ifndef ZSTD_COMPRESS_H
#define ZSTD_COMPRESS_H
/*-*************************************
* Dependencies
@ -21,10 +21,10 @@
#include "../common/zstd_internal.h"
#include "zstd_cwksp.h"
#ifdef ZSTD_MULTITHREAD
# include "zstdmt_compress.h" /* ZSTDMT_CCtx */
# include "zstdmt_compress.h"
#endif
#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */
#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */
/*-*************************************
* Constants
@ -46,12 +46,11 @@
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
typedef struct {
typedef struct ZSTD_prefixDict_s {
const void* dict;
size_t dictSize;
ZSTD_dictContentType_e dictContentType;
ZSTD_dictLoadMethod_e loadMethod;
} ZSTD_PrefixDict;
} ZSTD_prefixDict;
typedef struct {
void* dictBuffer;
@ -468,7 +467,7 @@ typedef struct {
U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
ZSTD_entropyCTablesMetadata_t entropyMetadata;
} ZSTD_BlockSplitCtx;
} ZSTD_blockSplitCtx;
struct ZSTD_CCtx_s {
ZSTD_compressionStage_e stage;
@ -526,7 +525,7 @@ struct ZSTD_CCtx_s {
/* Dictionary */
ZSTD_localDict localDict;
const ZSTD_CDict* cdict;
ZSTD_PrefixDict prefixDict; /* single-usage dictionary */
ZSTD_prefixDict prefixDict; /* single-usage dictionary */
/* Multi-threading */
#ifdef ZSTD_MULTITHREAD
@ -539,7 +538,7 @@ struct ZSTD_CCtx_s {
#endif
/* Workspace for block splitter */
ZSTD_BlockSplitCtx blockSplitCtx;
ZSTD_blockSplitCtx blockSplitCtx;
/* Buffer for output from external sequence producer */
ZSTD_Sequence* extSeqBuf;
@ -554,7 +553,7 @@ typedef enum {
ZSTD_extDict = 1,
ZSTD_dictMatchState = 2,
ZSTD_dedicatedDictSearch = 3
} ZSTD_DictMode_e;
} ZSTD_dictMode_e;
typedef enum {
ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict.
@ -579,7 +578,7 @@ typedef enum {
typedef size_t (*ZSTD_BlockCompressor_f) (
ZSTD_MatchState_t* bs, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_DictMode_e dictMode);
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@ -1069,7 +1068,7 @@ MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
* Inspects the provided matchState and figures out what dictMode should be
* passed to the compressor.
*/
MEM_STATIC ZSTD_DictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms)
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms)
{
return ZSTD_window_hasExtDict(ms->window) ?
ZSTD_extDict :
@ -1634,4 +1633,4 @@ size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
#endif /* ZSTD_COMPRESS_INTERNAL_H */
#endif /* ZSTD_COMPRESS_H */

16
lib/compress/zstd_lazy.c
View File

@ -74,7 +74,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms,
U32 curr, const BYTE* inputEnd,
U32 nbCompares, U32 btLow,
const ZSTD_DictMode_e dictMode)
const ZSTD_dictMode_e dictMode)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const bt = ms->chainTable;
@ -168,7 +168,7 @@ size_t ZSTD_DUBT_findBetterDictMatch (
size_t bestLength,
U32 nbCompares,
U32 const mls,
const ZSTD_DictMode_e dictMode)
const ZSTD_dictMode_e dictMode)
{
const ZSTD_MatchState_t * const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
@ -244,7 +244,7 @@ size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offBasePtr,
U32 const mls,
const ZSTD_DictMode_e dictMode)
const ZSTD_dictMode_e dictMode)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
@ -396,7 +396,7 @@ size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offBasePtr,
const U32 mls /* template */,
const ZSTD_DictMode_e dictMode)
const ZSTD_dictMode_e dictMode)
{
DEBUGLOG(7, "ZSTD_BtFindBestMatch");
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
@ -668,7 +668,7 @@ size_t ZSTD_HcFindBestMatch(
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_DictMode_e dictMode)
const U32 mls, const ZSTD_dictMode_e dictMode)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const chainTable = ms->chainTable;
@ -1142,7 +1142,7 @@ size_t ZSTD_RowFindBestMatch(
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_DictMode_e dictMode,
const U32 mls, const ZSTD_dictMode_e dictMode,
const U32 rowLog)
{
U32* const hashTable = ms->hashTable;
@ -1492,7 +1492,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
U32 const mls,
U32 const rowLog,
searchMethod_e const searchMethod,
ZSTD_DictMode_e const dictMode)
ZSTD_dictMode_e const dictMode)
{
if (dictMode == ZSTD_noDict) {
ZSTD_SWITCH_SEARCH_METHOD(noDict)
@ -1518,7 +1518,7 @@ size_t ZSTD_compressBlock_lazy_generic(
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth,
ZSTD_DictMode_e const dictMode)
ZSTD_dictMode_e const dictMode)
{
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;

14
lib/compress/zstd_opt.c
View File

@ -562,7 +562,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_updateTree_internal(
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
const U32 mls, const ZSTD_DictMode_e dictMode)
const U32 mls, const ZSTD_dictMode_e dictMode)
{
const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base);
@ -592,7 +592,7 @@ ZSTD_insertBtAndGetAllMatches (
ZSTD_MatchState_t* ms,
U32* nextToUpdate3,
const BYTE* const ip, const BYTE* const iLimit,
const ZSTD_DictMode_e dictMode,
const ZSTD_dictMode_e dictMode,
const U32 rep[ZSTD_REP_NUM],
const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
const U32 lengthToBeat,
@ -838,7 +838,7 @@ U32 ZSTD_btGetAllMatches_internal(
const U32 rep[ZSTD_REP_NUM],
U32 const ll0,
U32 const lengthToBeat,
const ZSTD_DictMode_e dictMode,
const ZSTD_dictMode_e dictMode,
const U32 mls)
{
assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls);
@ -886,7 +886,7 @@ GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
}
static ZSTD_getAllMatchesFn
ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_DictMode_e const dictMode)
ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode)
{
ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
@ -1079,7 +1079,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const int optLevel,
const ZSTD_DictMode_e dictMode)
const ZSTD_dictMode_e dictMode)
{
optState_t* const optStatePtr = &ms->opt;
const BYTE* const istart = (const BYTE*)src;
@ -1445,7 +1445,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
static size_t ZSTD_compressBlock_opt0(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, const ZSTD_DictMode_e dictMode)
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
}
@ -1454,7 +1454,7 @@ static size_t ZSTD_compressBlock_opt0(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
static size_t ZSTD_compressBlock_opt2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, const ZSTD_DictMode_e dictMode)
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
}

16
lib/compress/zstdmt_compress.c
View File

@ -1248,7 +1248,7 @@ static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params)
size_t ZSTDMT_initCStream_internal(
ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictLoadMethod_e dictLoadMethod,
const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{
@ -1274,6 +1274,18 @@ size_t ZSTDMT_initCStream_internal(
mtctx->params = params;
mtctx->frameContentSize = pledgedSrcSize;
ZSTD_freeCDict(mtctx->cdictLocal);
if (dict) {
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
mtctx->cdictLocal = NULL;
mtctx->cdict = cdict;
}
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(&params);
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetSectionSize = params.jobSize;
@ -1347,7 +1359,7 @@ size_t ZSTDMT_initCStream_internal(
} else {
/* note : a loadPrefix becomes an internal CDict */
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
dictLoadMethod, dictContentType,
ZSTD_dlm_byRef, dictContentType,
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);

2
lib/compress/zstdmt_compress.h
View File

@ -64,7 +64,7 @@ size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
* even if they are not needed for the current compression.
* @return : 0, or an error code */
size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictLoadMethod_e dictLoadMethod,
const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType,
const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize);