Disable tests and refactor

This commit is contained in:
Stella Lau 2017-08-21 01:59:08 -07:00
parent 023b24e6d4
commit f181f33bdf
13 changed files with 102 additions and 129 deletions

View File

@ -221,6 +221,7 @@ typedef struct seqDef_s {
U16 matchLength; U16 matchLength;
} seqDef; } seqDef;
typedef struct { typedef struct {
seqDef* sequencesStart; seqDef* sequencesStart;
seqDef* sequences; seqDef* sequences;
@ -240,7 +241,6 @@ struct ZSTD_CCtx_params_s {
ZSTD_frameParameters fParams; ZSTD_frameParameters fParams;
int compressionLevel; int compressionLevel;
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
/* Dictionary */ /* Dictionary */
@ -251,9 +251,10 @@ struct ZSTD_CCtx_params_s {
U32 nbThreads; U32 nbThreads;
unsigned jobSize; unsigned jobSize;
unsigned overlapSizeLog; unsigned overlapSizeLog;
#if 0
/* Test parameter */ /* Test parameter */
U32 testParam; U32 testParam;
#endif
}; };
typedef struct { typedef struct {
@ -349,16 +350,17 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
/*! ZSTD_initCStream_internal() : /*! ZSTD_initCStream_internal_opaque() :
* Private use only. Init streaming operation. * Private use only. Init streaming operation.
* expects params to be valid. * expects params to be valid.
* must receive dict, or cdict, or none, but not both. * must receive dict, or cdict, or none, but not both.
* @return : 0, or an error code */ * @return : 0, or an error code */
size_t ZSTD_initCStream_internal_opaque(ZSTD_CStream* zcs, size_t ZSTD_initCStream_internal_opaque(
const void* dict, size_t dictSize, ZSTD_CStream* zcs,
const ZSTD_CDict* cdict, const void* dict, size_t dictSize,
ZSTD_CCtx_params params, const ZSTD_CDict* cdict,
unsigned long long pledgedSrcSize); ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize);
/*! ZSTD_compressStream_generic() : /*! ZSTD_compressStream_generic() :
* Private use only. To be called from zstdmt_compress.c in single-thread mode. */ * Private use only. To be called from zstdmt_compress.c in single-thread mode. */

View File

@ -70,7 +70,6 @@ struct ZSTD_CDict_s {
ZSTD_CCtx* refContext; ZSTD_CCtx* refContext;
}; /* typedef'd to ZSTD_CDict within "zstd.h" */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */
struct ZSTD_CCtx_s { struct ZSTD_CCtx_s {
const BYTE* nextSrc; /* next block here to continue on current prefix */ const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */ const BYTE* base; /* All regular indexes relative to this position */
@ -81,10 +80,14 @@ struct ZSTD_CCtx_s {
U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */ U32 loadedDictEnd; /* index of end of dictionary */
// U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */ #if 0
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
#endif
ZSTD_compressionStage_e stage; ZSTD_compressionStage_e stage;
U32 dictID; U32 dictID;
// int compressionLevel; #if 0
int compressionLevel;
#endif
ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params requestedParams;
ZSTD_CCtx_params appliedParams; ZSTD_CCtx_params appliedParams;
void* workSpace; void* workSpace;
@ -117,15 +120,19 @@ struct ZSTD_CCtx_s {
U32 frameEnded; U32 frameEnded;
/* Dictionary */ /* Dictionary */
// ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */ #if 0
// U32 dictContentByRef; ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
U32 dictContentByRef;
#endif
ZSTD_CDict* cdictLocal; ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict; const ZSTD_CDict* cdict;
const void* prefix; const void* prefix;
size_t prefixSize; size_t prefixSize;
/* Multi-threading */ /* Multi-threading */
// U32 nbThreads; #if 0
U32 nbThreads;
#endif
ZSTDMT_CCtx* mtctx; ZSTDMT_CCtx* mtctx;
}; };
@ -203,29 +210,8 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
/* private API call, for dictBuilder only */ /* private API call, for dictBuilder only */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
#if 0
// TODO: get rid of this function
static ZSTD_parameters ZSTD_getParamsFromCCtxParams(const ZSTD_CCtx_params cctxParams)
{
ZSTD_parameters params;
params.cParams = cctxParams.cParams;
params.fParams = cctxParams.fParams;
return params;
}
#endif
#if 0 /* TODO: get rid of this function if possible*/
// TODO: get rid of this function too
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromParams(ZSTD_parameters params) {
ZSTD_CCtx_params cctxParams;
memset(&cctxParams, 0, sizeof(ZSTD_CCtx_params));
cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams;
return cctxParams;
}
#endif
// TODO: get rid of this function too
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
@ -234,12 +220,6 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
cctxParams.cParams = cParams; cctxParams.cParams = cParams;
return cctxParams; return cctxParams;
} }
#if 0
// TODO: get rid of this function
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) {
return ZSTD_getParamsFromCCtxParams(cctx->appliedParams);
}
#endif
/* older variant; will be deprecated */ /* older variant; will be deprecated */
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value) size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
@ -261,7 +241,6 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
#define ZSTD_CLEVEL_CUSTOM 999 #define ZSTD_CLEVEL_CUSTOM 999
#if 0
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{ {
if (cctx->requestedParams.compressionLevel==ZSTD_CLEVEL_CUSTOM) return; if (cctx->requestedParams.compressionLevel==ZSTD_CLEVEL_CUSTOM) return;
@ -270,7 +249,6 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
cctx->pledgedSrcSizePlusOne-1, 0); cctx->pledgedSrcSizePlusOne-1, 0);
cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
} }
#endif
ZSTD_CCtx_params* ZSTD_createCCtxParams(void) ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
{ {
@ -290,27 +268,15 @@ size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params)
return 0; return 0;
} }
size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
ZSTD_compressionParameters cParams)
{ {
memset(params, 0, sizeof(ZSTD_CCtx_params)); memset(cctxParams, 0, sizeof(ZSTD_CCtx_params));
params->cParams = cParams; cctxParams->cParams = params.cParams;
params->compressionLevel = ZSTD_CLEVEL_CUSTOM; cctxParams->fParams = params.fParams;
cctxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
return 0; return 0;
} }
ZSTD_CCtx_params* ZSTD_createAndInitCCtxParams(
int compressionLevel, unsigned long long estimatedSrcSize,
size_t dictSize)
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
if (params == NULL) { return NULL; }
ZSTD_initCCtxParams(params, ZSTD_getCParams(
compressionLevel, estimatedSrcSize, dictSize));
params->compressionLevel = ZSTD_CLEVEL_CUSTOM;
return params;
}
size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params)
{ {
if (params == NULL) { return 0; } if (params == NULL) { return 0; }
@ -318,8 +284,6 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params)
return 0; return 0;
} }
static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params) static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params)
{ {
if (params->compressionLevel == ZSTD_CLEVEL_CUSTOM) return; if (params->compressionLevel == ZSTD_CLEVEL_CUSTOM) return;
@ -341,6 +305,10 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
switch(param) switch(param)
{ {
case ZSTD_p_compressionLevel: case ZSTD_p_compressionLevel:
if (value == 0) return 0;
if (cctx->cdict) return ERROR(stage_wrong);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_windowLog: case ZSTD_p_windowLog:
case ZSTD_p_hashLog: case ZSTD_p_hashLog:
case ZSTD_p_chainLog: case ZSTD_p_chainLog:
@ -350,6 +318,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
case ZSTD_p_compressionStrategy: case ZSTD_p_compressionStrategy:
if (value == 0) return 0; /* special value : 0 means "don't change anything" */ if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
ZSTD_cLevelToCParams(cctx);
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_p_contentSizeFlag: case ZSTD_p_contentSizeFlag:
@ -396,11 +365,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
if (cctx->requestedParams.nbThreads <= 1) return ERROR(parameter_unsupported); if (cctx->requestedParams.nbThreads <= 1) return ERROR(parameter_unsupported);
assert(cctx->mtctx != NULL); assert(cctx->mtctx != NULL);
return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value); return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
#if 0
case ZSTD_p_test : case ZSTD_p_test :
DEBUGLOG(2, "Setting test parameter = %u", value); DEBUGLOG(2, "Setting test parameter = %u", value);
cctx->requestedParams.testParam = (value > 0); cctx->requestedParams.testParam = (value > 0);
return 0; return 0;
#endif
default: return ERROR(parameter_unsupported); default: return ERROR(parameter_unsupported);
} }
@ -477,12 +447,13 @@ size_t ZSTD_CCtxParam_setParameter(
params->fParams.checksumFlag = value > 0; params->fParams.checksumFlag = value > 0;
return 0; return 0;
case ZSTD_p_dictIDFlag : case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
params->fParams.noDictIDFlag = (value == 0); params->fParams.noDictIDFlag = (value == 0);
return 0; return 0;
case ZSTD_p_dictMode : case ZSTD_p_dictMode :
/* restrict dictionary mode to "rawContent" or "fullDict" only */
ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent); ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
if (value > (unsigned)ZSTD_dm_fullDict) { if (value > (unsigned)ZSTD_dm_fullDict) {
return ERROR(parameter_outOfBound); return ERROR(parameter_outOfBound);
@ -491,6 +462,7 @@ size_t ZSTD_CCtxParam_setParameter(
return 0; return 0;
case ZSTD_p_refDictContent : case ZSTD_p_refDictContent :
/* dictionary content will be referenced, instead of copied */
params->dictContentByRef = value > 0; params->dictContentByRef = value > 0;
return 0; return 0;
@ -516,16 +488,18 @@ size_t ZSTD_CCtxParam_setParameter(
if (params->nbThreads <= 1) { return ERROR(parameter_unsupported); } if (params->nbThreads <= 1) { return ERROR(parameter_unsupported); }
params->overlapSizeLog = value; params->overlapSizeLog = value;
return 0; return 0;
#if 0
case ZSTD_p_test : case ZSTD_p_test :
DEBUGLOG(2, "setting opaque: ZSTD_p_test: %u", value); DEBUGLOG(2, "setting opaque: ZSTD_p_test: %u", value);
params->testParam = (value > 0); params->testParam = (value > 0);
return 0; return 0;
#endif
default: return ERROR(parameter_unsupported); default: return ERROR(parameter_unsupported);
} }
} }
#if 0
static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params) static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params)
{ {
DEBUGLOG(2, "======CCtxParams======"); DEBUGLOG(2, "======CCtxParams======");
@ -551,19 +525,24 @@ static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params)
params->nbThreads, params->nbThreads,
params->jobSize, params->jobSize,
params->overlapSizeLog); params->overlapSizeLog);
#if 0
DEBUGLOG(2, "testParam: %u", DEBUGLOG(2, "testParam: %u",
params->testParam); params->testParam);
#endif
} }
#endif
// This function should probably be updated whenever ZSTD_CCtx_params is updated. /**
ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params) * This function should be updated whenever ZSTD_CCtx_params is updated.
* Parameters are copied manually before the dictionary is loaded.
* The multithreading parameters jobSize and overlapSizeLog are set only if
* nbThreads >= 1.
*/
size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params)
{ {
if (params == NULL) { return ERROR(GENERIC); } if (params == NULL) { return ERROR(GENERIC); }
if (cctx->cdict) { return ERROR(stage_wrong); } if (cctx->cdict) { return ERROR(stage_wrong); }
DEBUGLOG(2, "Applying cctx params\n");
ZSTD_debugPrintCCtxParams(params);
/* Assume the compression and frame parameters are validated */ /* Assume the compression and frame parameters are validated */
cctx->requestedParams.cParams = params->cParams; cctx->requestedParams.cParams = params->cParams;
cctx->requestedParams.fParams = params->fParams; cctx->requestedParams.fParams = params->fParams;
@ -585,8 +564,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params*
cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) ); cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) );
} }
/* Copy test parameter */ #if 0
/* Copy test parameter */
cctx->requestedParams.testParam = params->testParam; cctx->requestedParams.testParam = params->testParam;
#endif
return 0; return 0;
} }
@ -3247,7 +3228,6 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
{ {
// TODO: Applied params compression level okay? Gets overwritten
U32 const cLevel = cctx->appliedParams.compressionLevel; U32 const cLevel = cctx->appliedParams.compressionLevel;
ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ? ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ?
cctx->appliedParams.cParams : cctx->appliedParams.cParams :
@ -3710,7 +3690,8 @@ static size_t ZSTD_initCDict_internal_opaque(ZSTD_CDict* cdict,
} }
cdict->dictContentSize = dictSize; cdict->dictContentSize = dictSize;
/* Frame parameters should be zero? */ /* TODO: do the frame parameters need to be zero?
* does nbThreads need to be zero? */
CHECK_F( ZSTD_compressBegin_internal(cdict->refContext, CHECK_F( ZSTD_compressBegin_internal(cdict->refContext,
cdict->dictContent, dictSize, cdict->dictContent, dictSize,
NULL, NULL,
@ -3748,9 +3729,6 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_opaque(
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
/* Initialize to 0 to preserve semantics */
ZSTD_frameParameters const fParams = { 0, 0, 0 };
params.fParams = fParams;
if (!cdict || !cctx) { if (!cdict || !cctx) {
ZSTD_free(cdict, customMem); ZSTD_free(cdict, customMem);
@ -3759,7 +3737,6 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_opaque(
} }
cdict->refContext = cctx; cdict->refContext = cctx;
if (ZSTD_isError( ZSTD_initCDict_internal_opaque( if (ZSTD_isError( ZSTD_initCDict_internal_opaque(
cdict, cdict,
dictBuffer, dictSize, dictBuffer, dictSize,
@ -3777,8 +3754,6 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem) ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
{ {
ZSTD_CCtx_params cctxParams = ZSTD_makeCCtxParamsFromCParams(cParams); ZSTD_CCtx_params cctxParams = ZSTD_makeCCtxParamsFromCParams(cParams);
ZSTD_frameParameters const fParams = { 0, 0, 0 };
cctxParams.fParams = fParams;
cctxParams.dictMode = dictMode; cctxParams.dictMode = dictMode;
cctxParams.dictContentByRef = byReference; cctxParams.dictContentByRef = byReference;
return ZSTD_createCDict_advanced_opaque(dictBuffer, dictSize, cctxParams, customMem); return ZSTD_createCDict_advanced_opaque(dictBuffer, dictSize, cctxParams, customMem);
@ -3837,6 +3812,7 @@ ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque(
cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize); cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
params->dictContentByRef = 1; params->dictContentByRef = 1;
/* What if nbThreads > 1? */
if (ZSTD_isError( ZSTD_initCDict_internal_opaque(cdict, dict, dictSize, *params) )) if (ZSTD_isError( ZSTD_initCDict_internal_opaque(cdict, dict, dictSize, *params) ))
return NULL; return NULL;
@ -3970,11 +3946,10 @@ size_t ZSTD_CStreamOutSize(void)
static size_t ZSTD_resetCStream_internal_opaque( static size_t ZSTD_resetCStream_internal_opaque(
ZSTD_CStream* zcs, ZSTD_CStream* zcs,
const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, const void* dict, size_t dictSize,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
{ {
params.dictMode = dictMode;
DEBUGLOG(4, "ZSTD_resetCStream_internal"); DEBUGLOG(4, "ZSTD_resetCStream_internal");
/* params are supposed to be fully validated at this point */ /* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
@ -4004,8 +3979,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
ZSTD_CCtx_params cctxParams = zcs->requestedParams; ZSTD_CCtx_params cctxParams = zcs->requestedParams;
cctxParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
cctxParams.dictMode = dictMode; return ZSTD_resetCStream_internal_opaque(zcs, dict, dictSize,
return ZSTD_resetCStream_internal_opaque(zcs, dict, dictSize, dictMode,
cdict, cctxParams, pledgedSrcSize); cdict, cctxParams, pledgedSrcSize);
} }
#endif #endif
@ -4018,14 +3992,15 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
if (params.compressionLevel != ZSTD_CLEVEL_CUSTOM) { if (params.compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(params.compressionLevel, pledgedSrcSize, 0 /* dictSize */); params.cParams = ZSTD_getCParams(params.compressionLevel, pledgedSrcSize, 0 /* dictSize */);
} }
return ZSTD_resetCStream_internal_opaque(zcs, NULL, 0, params.dictMode, zcs->cdict, params, pledgedSrcSize); return ZSTD_resetCStream_internal_opaque(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
} }
size_t ZSTD_initCStream_internal_opaque(ZSTD_CStream* zcs, size_t ZSTD_initCStream_internal_opaque(
const void* dict, size_t dictSize, ZSTD_CStream* zcs,
const ZSTD_CDict* cdict, const void* dict, size_t dictSize,
ZSTD_CCtx_params params, const ZSTD_CDict* cdict,
unsigned long long pledgedSrcSize) ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{ {
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@ -4038,8 +4013,8 @@ size_t ZSTD_initCStream_internal_opaque(ZSTD_CStream* zcs,
} }
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
zcs->requestedParams.dictContentByRef, params.dictContentByRef,
zcs->requestedParams.dictMode, params.dictMode,
params.cParams, zcs->customMem); params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@ -4055,8 +4030,7 @@ size_t ZSTD_initCStream_internal_opaque(ZSTD_CStream* zcs,
zcs->requestedParams = params; zcs->requestedParams = params;
return ZSTD_resetCStream_internal_opaque( return ZSTD_resetCStream_internal_opaque(
zcs, NULL, 0, zcs->requestedParams.dictMode, zcs->cdict, zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
params, pledgedSrcSize);
} }
#if 0 #if 0
@ -4339,7 +4313,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
} else } else
#endif #endif
{ {
CHECK_F( ZSTD_resetCStream_internal_opaque(cctx, prefix, prefixSize, params.dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); CHECK_F( ZSTD_resetCStream_internal_opaque(cctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
} } } }
/* compression stage */ /* compression stage */

View File

@ -186,6 +186,10 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
ZSTD_free(buf.start, bufPool->cMem); ZSTD_free(buf.start, bufPool->cMem);
} }
/**
* TODO
* Resets parameters to zero for jobs?
*/
static void ZSTDMT_zeroCCtxParams(ZSTD_CCtx_params* params) static void ZSTDMT_zeroCCtxParams(ZSTD_CCtx_params* params)
{ {
params->forceWindow = 0; params->forceWindow = 0;
@ -778,7 +782,6 @@ size_t ZSTDMT_initCStream_internal_opaque(
} }
/** ZSTDMT_initCStream_internal() : /** ZSTDMT_initCStream_internal() :
* internal usage only */ * internal usage only */
size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,

View File

@ -69,15 +69,6 @@ ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
const ZSTD_CDict* cdict, const ZSTD_CDict* cdict,
ZSTD_parameters const params, ZSTD_parameters const params,
unsigned overlapLog); unsigned overlapLog);
#if 0
ZSTDLIB_API size_t ZSTDMT_compress_advanced_opaque(
ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_CCtx_params* const params,
unsigned overlapLog);
#endif
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */

View File

@ -510,7 +510,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* It will also consider src size to be arbitrarily "large", which is worst case. * It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* TODO: ZSTD_estimateCStreamSize_advanced_opaque * TODO: ZSTD_estimateCStreamSize_advanced_opaque()
* Note : CStream estimation is only correct for single-threaded compression. * Note : CStream estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size. * ZSTD_DStream memory budget depends on window Size.
* This information can be passed manually, using ZSTD_estimateDStreamSize, * This information can be passed manually, using ZSTD_estimateDStreamSize,
@ -527,7 +527,7 @@ ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t sr
/*! ZSTD_estimate?DictSize() : /*! ZSTD_estimate?DictSize() :
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
* ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced(). * ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
* TODO: ZSTD_estimateCDictSize_advanced_opaque(), can set by reference * TODO: ZSTD_estimateCDictSize_advanced_opaque()
* Note : dictionary created "byReference" are smaller */ * Note : dictionary created "byReference" are smaller */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference); ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
@ -615,11 +615,9 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque(
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createAndInitCCtxParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
* `estimatedSrcSize` value is optional, select 0 if not known */ * `estimatedSrcSize` value is optional, select 0 if not known */
@ -647,7 +645,6 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params); ZSTD_parameters params);
/*! ZSTD_compress_usingCDict_advanced() : /*! ZSTD_compress_usingCDict_advanced() :
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
@ -1003,7 +1000,9 @@ typedef enum {
/* advanced parameters - may not remain available after API update */ /* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */ * even when referencing into Dictionary content (default:0) */
#if 0
ZSTD_p_test, ZSTD_p_test,
#endif
} ZSTD_cParameter; } ZSTD_cParameter;
@ -1014,8 +1013,9 @@ typedef enum {
* @result : 0, or an error code (which can be tested with ZSTD_isError()). */ * @result : 0, or an error code (which can be tested with ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
/* TODO */
ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value); ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value);
/* TODO */
ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params);
/*! ZSTD_CCtx_setPledgedSrcSize() : /*! ZSTD_CCtx_setPledgedSrcSize() :

View File

@ -47,8 +47,7 @@ DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wredundant-decls
ZSTD_DEBUG_FLAGS = -g -DZSTD_DEBUG=2 CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) $(ZSTD_DEBUG_FLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)

View File

@ -213,9 +213,10 @@ void FIO_setOverlapLog(unsigned overlapLog){
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
g_overlapLog = overlapLog; g_overlapLog = overlapLog;
} }
#if 0
static U32 g_testParamFlag = 0; static U32 g_testParamFlag = 0;
void FIO_setTestParamFlag(unsigned testParamFlag) { g_testParamFlag = testParamFlag; } void FIO_setTestParamFlag(unsigned testParamFlag) { g_testParamFlag = testParamFlag; }
#endif
/*-************************************* /*-*************************************
* Functions * Functions
@ -413,9 +414,10 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
#if 0
/* Test */ /* Test */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_test, g_testParamFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_test, g_testParamFlag) );
#endif
} }
#elif defined(ZSTD_MULTITHREAD) #elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);

View File

@ -56,6 +56,9 @@ void FIO_setMemLimit(unsigned memLimit);
void FIO_setNbThreads(unsigned nbThreads); void FIO_setNbThreads(unsigned nbThreads);
void FIO_setBlockSize(unsigned blockSize); void FIO_setBlockSize(unsigned blockSize);
void FIO_setOverlapLog(unsigned overlapLog); void FIO_setOverlapLog(unsigned overlapLog);
#if 0
void FIO_setTestParamFlag(unsigned testParamFlag);
#endif
/*-************************************* /*-*************************************

View File

@ -430,7 +430,9 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
#if 0
if (!strcmp(argument, "--testParam")) { FIO_setTestParamFlag(1); continue; } if (!strcmp(argument, "--testParam")) { FIO_setTestParamFlag(1); continue; }
#endif
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
#endif #endif

View File

@ -25,7 +25,7 @@ PRGDIR = ../programs
PYTHON ?= python3 PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest TESTARTEFACT := versionsTest namespaceTest
DEBUGLEVEL= 2 DEBUGLEVEL= 1
DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL) DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL)
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
@ -169,10 +169,9 @@ datagen : $(PRGDIR)/datagen.c datagencli.c
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
OPAQUEFILES := $(ZSTD_FILES) $(ZDICT_FILES) roundTripCrashOpaque.c cctxParamRoundTrip : LDFLAGS += $(MULTITHREAD_CPP)
roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_CPP) cctxParamRoundTrip : LDFLAGS += $(MULTITHREAD_LD)
roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_LD) cctxParamRoundTrip : $(ZSTD_FILES) cctxParamRoundTrip.c
roundTripCrashOpaque : $(OPAQUEFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
longmatch : $(ZSTD_FILES) longmatch.c longmatch : $(ZSTD_FILES) longmatch.c

View File

@ -38,13 +38,16 @@
fprintf(stderr, \ fprintf(stderr, \
"Error=> %s: %s", \ "Error=> %s: %s", \
#f, ZSTD_getErrorName(err)); \ #f, ZSTD_getErrorName(err)); \
exit(1); \ crash(1); \
} } } }
/** roundTripTest() : /** roundTripTest() :
* Compresses `srcBuff` into `compressedBuff`, * Compresses `srcBuff` into `compressedBuff`,
* then decompresses `compressedBuff` into `resultBuff`. * then decompresses `compressedBuff` into `resultBuff`.
*
* Parameters are currently set manually.
*
* @return : result of decompression, which should be == `srcSize` * @return : result of decompression, which should be == `srcSize`
* or an error code if either compression or decompression fails. * or an error code if either compression or decompression fails.
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` * Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`

View File

@ -136,10 +136,8 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con
buffIn.src = src; buffIn.src = src;
buffIn.size = srcSize; buffIn.size = srcSize;
buffIn.pos = 0; buffIn.pos = 0;
ZSTD_compressStream(g_cstream, &buffOut, &buffIn); ZSTD_compressStream(g_cstream, &buffOut, &buffIn);
ZSTD_endStream(g_cstream, &buffOut); ZSTD_endStream(g_cstream, &buffOut);
return buffOut.pos; return buffOut.pos;
} }
@ -385,13 +383,11 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
const BYTE* ip = dstBuff; const BYTE* ip = dstBuff;
const BYTE* iend; const BYTE* iend;
size_t frameHeaderSize, cBlockSize; size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */ ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) { if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
@ -399,7 +395,6 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
} }
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
ip += ZSTD_blockHeaderSize; /* skip block header */ ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc); ZSTD_decompressBegin(g_zdc);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip; g_cSize = iend-ip;

View File

@ -1655,9 +1655,9 @@ static int fuzzerTests_newAPI_opaque(U32 seed, U32 nbTests, unsigned startTest,
} }
if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1) ); if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1) );
#if 0
if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, FUZ_rand(&lseed) & 1) ); if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, FUZ_rand(&lseed) & 1) );
#endif
/* Apply parameters */ /* Apply parameters */