sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-12-01 00:02:43 -05:00
Code Issues Projects Releases Wiki Activity

Split samples into train and test sets

Browse Source
This commit is contained in:
Jennifer Liu 2018-06-29 12:33:34 -07:00
parent 1ab57a7ce1
commit 0ef06f2e8a
2 changed files with 55 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
lib/dictBuilder/cover.c
View File

@ -39,6 +39,7 @@
* Constants * Constants
***************************************/ ***************************************/
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
#define DEFAULT_SPLITPOINT 0.8
/*-************************************* /*-*************************************
* Console display * Console display
@ -203,6 +204,8 @@ typedef struct {
size_t *offsets; size_t *offsets;
const size_t *samplesSizes; const size_t *samplesSizes;
size_t nbSamples; size_t nbSamples;
size_t nbTrainSamples;
size_t nbTestSamples;
U32 *suffix; U32 *suffix;
size_t suffixSize; size_t suffixSize;
U32 *freqs; U32 *freqs;
@ -220,10 +223,10 @@ static COVER_ctx_t *g_ctx = NULL;
/** /**
* Returns the sum of the sample sizes. * Returns the sum of the sample sizes.
*/ */
static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { static size_t COVER_sum(const size_t *samplesSizes, unsigned firstSample, unsigned lastSample) {
size_t sum = 0; size_t sum = 0;
size_t i; size_t i;
for (i = 0; i < nbSamples; ++i) { for (i = firstSample; i < lastSample; ++i) {
sum += samplesSizes[i]; sum += samplesSizes[i];
} }
return sum; return sum;
@ -494,6 +497,10 @@ static int COVER_checkParameters(ZDICT_cover_params_t parameters,
if (parameters.d > parameters.k) { if (parameters.d > parameters.k) {
return 0; return 0;
} }
/* 0 < splitPoint < 1 */
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1){
return 0;
}
return 1; return 1;
} }
@ -531,9 +538,10 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
*/ */
static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples, const size_t *samplesSizes, unsigned nbSamples,
unsigned d) { unsigned d, double splitPoint) {
const BYTE *const samples = (const BYTE *)samplesBuffer; const BYTE *const samples = (const BYTE *)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); const unsigned first = 0;
const size_t totalSamplesSize = COVER_sum(samplesSizes, first, nbSamples);
/* Checks */ /* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) || if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
@ -541,15 +549,38 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
(U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20)); (U32)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
return 0; return 0;
} }
/* Split samples into testing and training sets */
const unsigned nbTrainSamples = nbSamples * splitPoint;
const unsigned nbTestSamples = nbSamples - nbTrainSamples;
/* Check if there's training sample */
if (nbTrainSamples < 1) {
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100));
DISPLAYLEVEL(1, "nbSamples is %u", nbSamples);
return 0;
}
/* Check if there's testing sample when splitPoint is nonzero */
if (nbTestSamples < 1 && splitPoint != 1.0) {
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
DISPLAYLEVEL(1, "splitPoint is %i", (int)(splitPoint*100));
DISPLAYLEVEL(1, "nbSamples is %u", nbSamples);
return 0;
}
const size_t trainingSamplesSize = COVER_sum(samplesSizes, first, nbTrainSamples);
const size_t testSamplesSize = COVER_sum(samplesSizes, nbTrainSamples, nbSamples);
/* Zero the context */ /* Zero the context */
memset(ctx, 0, sizeof(*ctx)); memset(ctx, 0, sizeof(*ctx));
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples, DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
(U32)totalSamplesSize); (U32)trainingSamplesSize);
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
(U32)testSamplesSize);
ctx->samples = samples; ctx->samples = samples;
ctx->samplesSizes = samplesSizes; ctx->samplesSizes = samplesSizes;
ctx->nbSamples = nbSamples; ctx->nbSamples = nbSamples;
ctx->nbTrainSamples = nbTrainSamples;
ctx->nbTestSamples = nbTestSamples;
/* Partial suffix array */ /* Partial suffix array */
ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1; ctx->suffixSize = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
/* Maps index to the dmerID */ /* Maps index to the dmerID */
ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
@ -563,7 +594,7 @@ static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
ctx->freqs = NULL; ctx->freqs = NULL;
ctx->d = d; ctx->d = d;
/* Fill offsets from the samlesSizes */ /* Fill offsets from the samplesSizes */
{ {
U32 i; U32 i;
ctx->offsets[0] = 0; ctx->offsets[0] = 0;
@ -665,7 +696,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
BYTE* const dict = (BYTE*)dictBuffer; BYTE* const dict = (BYTE*)dictBuffer;
COVER_ctx_t ctx; COVER_ctx_t ctx;
COVER_map_t activeDmers; COVER_map_t activeDmers;
parameters.splitPoint = 1.0;
/* Initialize global data */ /* Initialize global data */
g_displayLevel = parameters.zParams.notificationLevel; g_displayLevel = parameters.zParams.notificationLevel;
/* Checks */ /* Checks */
@ -683,8 +714,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
return ERROR(dstSize_tooSmall); return ERROR(dstSize_tooSmall);
} }
/* Initialize context and activeDmers */ /* Initialize context and activeDmers */
const double all = 1.0;
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
parameters.d)) { parameters.d, all)) {
return ERROR(GENERIC); return ERROR(GENERIC);
} }
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
@ -839,7 +871,7 @@ typedef struct COVER_tryParameters_data_s {
} COVER_tryParameters_data_t; } COVER_tryParameters_data_t;
/** /**
* Tries a set of parameters and upates the COVER_best_t with the results. * Tries a set of parameters and updates the COVER_best_t with the results.
* This function is thread safe if zstd is compiled with multithreaded support. * This function is thread safe if zstd is compiled with multithreaded support.
* It takes its parameters as an *OWNING* opaque pointer to support threading. * It takes its parameters as an *OWNING* opaque pointer to support threading.
*/ */
@ -870,7 +902,7 @@ static void COVER_tryParameters(void *opaque) {
dictBufferCapacity, parameters); dictBufferCapacity, parameters);
dictBufferCapacity = ZDICT_finalizeDictionary( dictBufferCapacity = ZDICT_finalizeDictionary(
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
parameters.zParams); parameters.zParams);
if (ZDICT_isError(dictBufferCapacity)) { if (ZDICT_isError(dictBufferCapacity)) {
DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
@ -889,7 +921,7 @@ static void COVER_tryParameters(void *opaque) {
/* Allocate dst with enough space to compress the maximum sized sample */ /* Allocate dst with enough space to compress the maximum sized sample */
{ {
size_t maxSampleSize = 0; size_t maxSampleSize = 0;
for (i = 0; i < ctx->nbSamples; ++i) { for (i = ctx->nbTrainSamples; i < ctx->nbSamples; ++i) {
maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
} }
dstCapacity = ZSTD_compressBound(maxSampleSize); dstCapacity = ZSTD_compressBound(maxSampleSize);
@ -904,7 +936,7 @@ static void COVER_tryParameters(void *opaque) {
} }
/* Compress each sample and sum their sizes (or error) */ /* Compress each sample and sum their sizes (or error) */
totalCompressedSize = dictBufferCapacity; totalCompressedSize = dictBufferCapacity;
for (i = 0; i < ctx->nbSamples; ++i) { for (i = ctx->nbTrainSamples; i < ctx->nbSamples; ++i) {
const size_t size = ZSTD_compress_usingCDict( const size_t size = ZSTD_compress_usingCDict(
cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
ctx->samplesSizes[i], cdict); ctx->samplesSizes[i], cdict);
@ -941,6 +973,8 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
ZDICT_cover_params_t *parameters) { ZDICT_cover_params_t *parameters) {
/* constants */ /* constants */
const unsigned nbThreads = parameters->nbThreads; const unsigned nbThreads = parameters->nbThreads;
const double splitPoint =
parameters->splitPoint == 0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
@ -958,6 +992,10 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
POOL_ctx *pool = NULL; POOL_ctx *pool = NULL;
/* Checks */ /* Checks */
if (splitPoint <= 0 || splitPoint >= 1) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
return ERROR(GENERIC);
}
if (kMinK < kMaxD || kMaxK < kMinK) { if (kMinK < kMaxD || kMaxK < kMinK) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
return ERROR(GENERIC); return ERROR(GENERIC);
@ -988,7 +1026,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
/* Initialize the context for this value of d */ /* Initialize the context for this value of d */
COVER_ctx_t ctx; COVER_ctx_t ctx;
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) { if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint)) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
COVER_best_destroy(&best); COVER_best_destroy(&best);
POOL_free(pool); POOL_free(pool);
@ -1013,6 +1051,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
data->parameters = *parameters; data->parameters = *parameters;
data->parameters.k = k; data->parameters.k = k;
data->parameters.d = d; data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps; data->parameters.steps = kSteps;
data->parameters.zParams.notificationLevel = g_displayLevel; data->parameters.zParams.notificationLevel = g_displayLevel;
/* Check the parameters */ /* Check the parameters */

1
lib/dictBuilder/zdict.h
View File

@ -86,6 +86,7 @@ typedef struct {
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training */
ZDICT_params_t zParams; ZDICT_params_t zParams;
} ZDICT_cover_params_t; } ZDICT_cover_params_t;