mirror of
https://github.com/facebook/zstd.git
synced 2025-10-15 00:02:02 -04:00
e859862341
Dictionaries are prebuilt and saved as part of the data object. The config decides whether or not to use the dictionary if it is available. Configs that require dictionaries are only run with data that have dictionaries. The method will skip configs that are irrelevant, so for example ZSTD_compress() will skip configs with dictionaries. I've also trimmed the silesia source to 1MB per file (12 MB total), and added 500 samples from the github data set with a dictionary. I've intentionally added an extra line to the `results.csv` to make the nightly build fail, so that we can see how CircleCI reports it. Full list of changes: * Add pre-built dictionaries to the data. * Add `use_dictionary` and `no_pledged_src_size` flags to the config. * Add a config using a dictionary for every level. * Add a config that specifies no pledged source size. * Support dictionaries and streaming in the `zstdcli` method. * Add a context-reuse method using `ZSTD_compressCCtx()`. * Clean up the formatting of the `results.csv` file to align columns. * Add `--data`, `--config`, and `--method` flags to constrain each to a particular value. This is useful for debugging a failure or debugging a particular config/method/data.
90 lines
3.4 KiB
C
90 lines
3.4 KiB
C
/*
|
|
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
|
* All rights reserved.
|
|
*
|
|
* This source code is licensed under both the BSD-style license (found in the
|
|
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
* in the COPYING file in the root directory of this source tree).
|
|
* You may select, at your option, one of the above-listed licenses.
|
|
*/
|
|
|
|
#include "config.h"
|
|
|
|
/* Define a config for each fast level we want to test with. */
|
|
#define FAST_LEVEL(x) \
|
|
param_value_t const level_fast##x##_param_values[] = { \
|
|
{.param = ZSTD_p_compressionLevel, .value = (unsigned)-x}, \
|
|
}; \
|
|
config_t const level_fast##x = { \
|
|
.name = "level -" #x, \
|
|
.cli_args = "--fast=" #x, \
|
|
.param_values = PARAM_VALUES(level_fast##x##_param_values), \
|
|
}; \
|
|
config_t const level_fast##x##_dict = { \
|
|
.name = "level -" #x " with dict", \
|
|
.cli_args = "--fast=" #x, \
|
|
.param_values = PARAM_VALUES(level_fast##x##_param_values), \
|
|
.use_dictionary = 1, \
|
|
};
|
|
|
|
/* Define a config for each level we want to test with. */
|
|
#define LEVEL(x) \
|
|
param_value_t const level_##x##_param_values[] = { \
|
|
{.param = ZSTD_p_compressionLevel, .value = (unsigned)x}, \
|
|
}; \
|
|
config_t const level_##x = { \
|
|
.name = "level " #x, \
|
|
.cli_args = "-" #x, \
|
|
.param_values = PARAM_VALUES(level_##x##_param_values), \
|
|
}; \
|
|
config_t const level_##x##_dict = { \
|
|
.name = "level " #x " with dict", \
|
|
.cli_args = "-" #x, \
|
|
.param_values = PARAM_VALUES(level_##x##_param_values), \
|
|
.use_dictionary = 1, \
|
|
};
|
|
|
|
|
|
#define PARAM_VALUES(pv) \
|
|
{ .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) }
|
|
|
|
#include "levels.h"
|
|
|
|
#undef LEVEL
|
|
#undef FAST_LEVEL
|
|
|
|
static config_t no_pledged_src_size = {
|
|
.name = "no source size",
|
|
.cli_args = "",
|
|
.param_values = {.data = NULL, .size = 0},
|
|
.no_pledged_src_size = 1,
|
|
};
|
|
|
|
static config_t const* g_configs[] = {
|
|
|
|
#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
|
|
#define LEVEL(x) &level_##x, &level_##x##_dict,
|
|
#include "levels.h"
|
|
#undef LEVEL
|
|
#undef FAST_LEVEL
|
|
|
|
&no_pledged_src_size,
|
|
NULL,
|
|
};
|
|
|
|
config_t const* const* configs = g_configs;
|
|
|
|
int config_skip_data(config_t const* config, data_t const* data) {
|
|
return config->use_dictionary && !data_has_dict(data);
|
|
}
|
|
|
|
int config_get_level(config_t const* config) {
|
|
param_values_t const params = config->param_values;
|
|
size_t i;
|
|
for (size_t i = 0; i < params.size; ++i) {
|
|
if (params.data[i].param == ZSTD_p_compressionLevel)
|
|
return params.data[i].value;
|
|
}
|
|
return CONFIG_NO_LEVEL;
|
|
}
|