From 198e6aac4414da8931aba0429f243db4ea4ec710 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 20 Jul 2016 20:12:24 +0200 Subject: [PATCH] Literals header fields use little endian convention --- lib/compress/zstd_compress.c | 55 +++++++---------- lib/decompress/zstd_decompress.c | 101 +++++++++++++++++-------------- zstd_compression_format.md | 28 +++++---- 3 files changed, 95 insertions(+), 89 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 0c7046ffb..4aaae39fc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -572,17 +572,14 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void switch(flSize) { case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((lbt_raw<<6) + (0<<5) + srcSize); + ostart[0] = (BYTE)((U32)lbt_raw + (srcSize<<3)); break; case 2: /* 2 - 2 - 12 */ - ostart[0] = (BYTE)((lbt_raw<<6) + (2<<4) + (srcSize >> 8)); - ostart[1] = (BYTE)srcSize; + MEM_writeLE16(ostart, (U32)lbt_raw + (1<<2) + (srcSize<<4)); break; default: /*note : should not be necessary : flSize is within {1,2,3} */ case 3: /* 2 - 2 - 20 */ - ostart[0] = (BYTE)((lbt_raw<<6) + (3<<4) + (srcSize >> 16)); - ostart[1] = (BYTE)(srcSize>>8); - ostart[2] = (BYTE)srcSize; + MEM_writeLE32(ostart, (U32)lbt_raw + (3<<2) + (srcSize<<4)); break; } @@ -595,22 +592,19 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons BYTE* const ostart = (BYTE* const)dst; U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - (void)dstCapacity; /* dstCapacity guaranteed to be >=4, hence large enough */ + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ switch(flSize) { case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((lbt_rle<<6) + (0<<5) + srcSize); + ostart[0] = (BYTE)((U32)lbt_rle + (srcSize<<3)); break; case 2: /* 2 - 2 - 12 */ - ostart[0] = (BYTE)((lbt_rle<<6) + (2<<4) + (srcSize >> 8)); - ostart[1] = (BYTE)srcSize; + MEM_writeLE16(ostart, (U32)lbt_rle + (1<<2) + (srcSize<<4)); break; default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ case 3: /* 2 - 2 - 20 */ - ostart[0] = (BYTE)((lbt_rle<<6) + (3<<4) + (srcSize >> 16)); - ostart[1] = (BYTE)(srcSize>>8); - ostart[2] = (BYTE)srcSize; + MEM_writeLE32(ostart, (U32)lbt_rle + (3<<2) + (srcSize<<4)); break; } @@ -658,24 +652,22 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, switch(lhSize) { case 3: /* 2 - 2 - 10 - 10 */ - ostart[0] = (BYTE)((srcSize>>6) + (singleStream << 4) + (hType<<6)); - ostart[1] = (BYTE)((srcSize<<2) + (cLitSize>>8)); - ostart[2] = (BYTE)(cLitSize); - break; + { U32 const lhc = hType + (singleStream << 2) + (srcSize<<4) + (cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } case 4: /* 2 - 2 - 14 - 14 */ - ostart[0] = (BYTE)((srcSize>>10) + (2<<4) + (hType<<6)); - ostart[1] = (BYTE)(srcSize>> 2); - ostart[2] = (BYTE)((srcSize<<6) + (cLitSize>>8)); - ostart[3] = (BYTE)(cLitSize); - break; + { U32 const lhc = hType + (2 << 2) + (srcSize<<4) + (cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } default: /* should not be necessary, lhSize is only {3,4,5} */ case 5: /* 2 - 2 - 18 - 18 */ - ostart[0] = (BYTE)((srcSize>>14) + (3<<4) + (hType<<6)); - ostart[1] = (BYTE)(srcSize>>6); - ostart[2] = (BYTE)((srcSize<<2) + (cLitSize>>16)); - ostart[3] = (BYTE)(cLitSize>>8); - ostart[4] = (BYTE)(cLitSize); - break; + { U32 const lhc = hType + (3 << 2) + (srcSize<<4) + (cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } } return lhSize+cLitSize; } @@ -2735,8 +2727,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) BYTE* op = (BYTE*)dst; size_t fhSize = 0; - /* not even init ! */ - if (cctx->stage==0) return ERROR(stage_wrong); + if (cctx->stage==0) return ERROR(stage_wrong); /*< not even init ! */ /* special case : empty frame */ if (cctx->stage==1) { @@ -2748,7 +2739,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } /* frame epilogue */ - if (dstCapacity < 3) return ERROR(dstSize_tooSmall); + if (dstCapacity < ZSTD_blockHeaderSize) return ERROR(dstSize_tooSmall); { U32 const checksum = cctx->params.fParams.checksumFlag ? (U32)(XXH64_digest(&cctx->xxhState) >> 11) : 0; @@ -2756,7 +2747,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } cctx->stage = 0; /* return to "created but not init" status */ - return 3+fhSize; + return ZSTD_blockHeaderSize+fhSize; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 1f57f4804..4fbd1092f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -397,9 +397,9 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t * compatible with legacy mode * @return : decompressed size if known, 0 otherwise note : 0 can mean any of the following : - - decompressed size is not provided within frame header + - decompressed size is not present within frame header - frame header unknown / not supported - - frame header not completely provided (`srcSize` too small) */ + - frame header not complete (`srcSize` too small) */ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) @@ -464,33 +464,42 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); - switch((litBlockType_t)(istart[0]>> 6)) + switch((litBlockType_t)(istart[0] & 3)) { case lbt_huffman: - { size_t litSize, litCSize, singleStream=0; - U32 lhSize = (istart[0] >> 4) & 3; + { size_t lhSize, litSize, litCSize, singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */ - switch(lhSize) + switch(lhlCode) { - case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ + case 1: + singleStream = 1; + /* fall through */ + case 0: default: /* note : default is impossible, since lhlCode into [0..3] */ /* 2 - 2 - 10 - 10 */ - lhSize=3; - singleStream = istart[0] & 16; - litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); - litCSize = ((istart[1] & 3) << 8) + istart[2]; - break; + { U32 const lhc = MEM_readLE24(istart) >> 4; + lhSize = 3; + litSize = lhc & 0x3FF; + litCSize = lhc >> 10; + break; + } + case 2: /* 2 - 2 - 14 - 14 */ - lhSize=4; - litSize = ((istart[0] & 15) << 10) + (istart[1] << 2) + (istart[2] >> 6); - litCSize = ((istart[2] & 63) << 8) + istart[3]; - break; + { U32 const lhc = MEM_readLE32(istart) >> 4; + lhSize = 4; + litSize = lhc & 0x3FFF; + litCSize = lhc >> 14; + break; + } case 3: /* 2 - 2 - 18 - 18 */ - lhSize=5; - litSize = ((istart[0] & 15) << 14) + (istart[1] << 6) + (istart[2] >> 2); - litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4]; - break; + { U64 const lhc = (MEM_readLE32(istart) + (((U64)istart[4]) << 32)) >> 4; + lhSize = 5; + litSize = lhc & 0x3FFFF; + litCSize = lhc >> 18; + break; + } } if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); @@ -501,23 +510,23 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, return ERROR(corruption_detected); dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+8; + dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; dctx->litEntropy = 1; return litCSize + lhSize; } case lbt_repeat: - { size_t litSize, litCSize; - U32 lhSize = ((istart[0]) >> 4) & 3; - if (lhSize != 1) /* only case supported for now : small litSize, single stream */ + { size_t litSize, litCSize, lhSize; + U32 const lhc = MEM_readLE24(istart) >> 4; + if ((((istart[0]) >> 2) & 3) != 1) /* only case supported for now : small litSize, single stream */ return ERROR(corruption_detected); if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); /* 2 - 2 - 10 - 10 */ - lhSize=3; - litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); - litCSize = ((istart[1] & 3) << 8) + istart[2]; + lhSize = 3; + litSize = lhc & 0x3FF; + litCSize = lhc >> 10; if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); { size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable); @@ -529,19 +538,21 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, return litCSize + lhSize; } case lbt_raw: - { size_t litSize; - U32 lhSize = ((istart[0]) >> 4) & 3; - switch(lhSize) + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) { - case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ - lhSize=1; - litSize = istart[0] & 31; + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; break; - case 2: - litSize = ((istart[0] & 15) << 8) + istart[1]; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; break; case 3: - litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; break; } @@ -560,19 +571,21 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, return lhSize+litSize; } case lbt_rle: - { size_t litSize; - U32 lhSize = ((istart[0]) >> 4) & 3; - switch(lhSize) + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) { - case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ lhSize = 1; - litSize = istart[0] & 31; + litSize = istart[0] >> 3; break; - case 2: - litSize = ((istart[0] & 15) << 8) + istart[1]; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; break; case 3: - litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ break; } diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 199a172b1..66cfe29ec 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -435,15 +435,17 @@ followed by 1 or 4 streams. Header is in charge of describing how literals are packed. It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes, -using big-endian convention. +using little-endian convention. -| BlockType | sizes format | [compressed size] | regenerated size | -| --------- | ------------ | ----------------- | ---------------- | -| 2 bits | 1 - 2 bits | 0 - 18 bits | 5 - 20 bits | +| BlockType | sizes format | regenerated size | [compressed size] | +| --------- | ------------ | ---------------- | ----------------- | +| 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits | + +In this representation, bits on the left are smallest bits. __Block Type__ : -This is a 2-bits field, describing 4 different block types : +This field uses 2 lowest bits of first byte, describing 4 different block types : | Value | 0 | 1 | 2 | 3 | | ---------- | ---------- | ------ | --- | ------- | @@ -466,19 +468,19 @@ Sizes format are divided into 2 families : and the decompressed size. It will also decode the number of streams. - For Raw or RLE blocks, it's enough to decode the size to regenerate. -For values spanning several bytes, convention is Big-endian. +For values spanning several bytes, convention is Little-endian. -__Sizes format for Raw or RLE literals block__ : +__Sizes format for Raw and RLE literals block__ : -- Value : 0x : Regenerated size uses 5 bits (0-31). +- Value : x0 : Regenerated size uses 5 bits (0-31). Total literal header size is 1 byte. - `size = h[0] & 31;` -- Value : 10 : Regenerated size uses 12 bits (0-4095). + `size = h[0]>>3;` +- Value : 01 : Regenerated size uses 12 bits (0-4095). Total literal header size is 2 bytes. - `size = ((h[0] & 15) << 8) + h[1];` + `size = (h[0]>>4) + (h[1]<<4);` - Value : 11 : Regenerated size uses 20 bits (0-1048575). Total literal header size is 3 bytes. - `size = ((h[0] & 15) << 16) + (h[1]<<8) + h[2];` + `size = (h[0]>>4) + (h[1]<<4) + (h[2]<<12);` Note : it's allowed to represent a short value (ex : `13`) using a long format, accepting the reduced compacity. @@ -499,7 +501,7 @@ Note : also applicable to "repeat-stats" blocks. Compressed and regenerated sizes use 18 bits (0-262143). Total literal header size is 5 bytes. -Compressed and regenerated size fields follow big endian convention. +Compressed and regenerated size fields follow little endian convention. #### Huffman Tree description