sharpetronics/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-10-09 00:05:28 -04:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3544 from facebook/seek_faster

Browse Source 
Improved seekable format ingestion speed for small frame size
This commit is contained in:
Yann Collet 2023-03-10 12:33:33 -08:00 committed by GitHub
commit 134d332b10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 18 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/seekable_format/examples/parallel_compression.c
View File

@ -25,7 +25,7 @@
#include "pool.h" // use zstd thread pool for demo #include "pool.h" // use zstd thread pool for demo
#include "zstd_seekable.h" #include "../zstd_seekable.h"
static void* malloc_orDie(size_t size) static void* malloc_orDie(size_t size)
{ {

2
contrib/seekable_format/examples/parallel_processing.c
View File

@ -29,7 +29,7 @@
#include "pool.h" // use zstd thread pool for demo #include "pool.h" // use zstd thread pool for demo
#include "zstd_seekable.h" #include "../zstd_seekable.h"
#define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b))

17
contrib/seekable_format/examples/seekable_compression.c
View File

@ -13,7 +13,7 @@
#define ZSTD_STATIC_LINKING_ONLY #define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include "zstd_seekable.h" #include "../zstd_seekable.h"
static void* malloc_orDie(size_t size) static void* malloc_orDie(size_t size)
{ {
@ -112,20 +112,23 @@ static char* createOutFilename_orDie(const char* filename)
return (char*)outSpace; return (char*)outSpace;
} }
int main(int argc, const char** argv) { #define CLEVEL_DEFAULT 5
int main(int argc, const char** argv)
{
const char* const exeName = argv[0]; const char* const exeName = argv[0];
if (argc!=3) { if (argc<3 || argc>4) {
printf("wrong arguments\n"); printf("wrong arguments \n");
printf("usage:\n"); printf("usage: \n");
printf("%s FILE FRAME_SIZE\n", exeName); printf("%s FILE FRAME_SIZE [LEVEL] \n", exeName);
return 1; return 1;
} }
{ const char* const inFileName = argv[1]; { const char* const inFileName = argv[1];
unsigned const frameSize = (unsigned)atoi(argv[2]); unsigned const frameSize = (unsigned)atoi(argv[2]);
int const cLevel = (argc==4) ? atoi(argv[3]) : CLEVEL_DEFAULT;
char* const outFileName = createOutFilename_orDie(inFileName); char* const outFileName = createOutFilename_orDie(inFileName);
compressFile_orDie(inFileName, outFileName, 5, frameSize); compressFile_orDie(inFileName, outFileName, cLevel, frameSize);
free(outFileName); free(outFileName);
} }

2
contrib/seekable_format/examples/seekable_decompression.c
View File

@ -16,7 +16,7 @@
#include <zstd.h> // presumes zstd library is installed #include <zstd.h> // presumes zstd library is installed
#include <zstd_errors.h> #include <zstd_errors.h>
#include "zstd_seekable.h" #include "../zstd_seekable.h"
#define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b))

2
contrib/seekable_format/tests/seekable_tests.c
View File

@ -4,7 +4,7 @@
#include <stdio.h> #include <stdio.h>
#include <assert.h> #include <assert.h>
#include "zstd_seekable.h" #include "../zstd_seekable.h"
/* Basic unit tests for zstd seekable format */ /* Basic unit tests for zstd seekable format */
int main(int argc, const char** argv) int main(int argc, const char** argv)

4
contrib/seekable_format/zstd_seekable.h
View File

@ -15,8 +15,8 @@ extern "C" {
#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U #define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U
/* Limit the maximum size to avoid any potential issues storing the compressed size */ /* Limit maximum size to avoid potential issues storing the compressed size */
#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x80000000U #define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x40000000U
/*-**************************************************************************** /*-****************************************************************************
* Seekable Format * Seekable Format

2
contrib/seekable_format/zstdseek_compress.c
View File

@ -230,6 +230,8 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
const BYTE* const inBase = (const BYTE*) input->src + input->pos; const BYTE* const inBase = (const BYTE*) input->src + input->pos;
size_t inLen = input->size - input->pos; size_t inLen = input->size - input->pos;
assert(zcs->maxFrameSize < INT_MAX);
ZSTD_CCtx_setParameter(zcs->cstream, ZSTD_c_srcSizeHint, (int)zcs->maxFrameSize);
inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize)); inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize));
/* if we haven't finished flushing the last frame, don't start writing a new one */ /* if we haven't finished flushing the last frame, don't start writing a new one */