From fe56eb9408004c5e9aacdbde59acb499b95831e2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 2 Apr 2016 00:21:55 +0200 Subject: [PATCH 1/4] removed unwanted files --- programs/result | 519 ---------------------------------------------- programs/tmp.mgz | Bin 5549 -> 0 bytes programs/tmp.zstd | Bin 6317 -> 0 bytes 3 files changed, 519 deletions(-) delete mode 100644 programs/result delete mode 100644 programs/tmp.mgz delete mode 100644 programs/tmp.zstd diff --git a/programs/result b/programs/result deleted file mode 100644 index 0d18ea646..000000000 --- a/programs/result +++ /dev/null @@ -1,519 +0,0 @@ -/* - bench.c - Demo module to benchmark open-source compression algorithms - Copyright (C) Yann Collet 2012-2015 - - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -/* ************************************** -* Compiler Options -****************************************/ -/* Disable some Visual warning messages */ -#ifdef _MSC_VER -# define _CRT_SECURE_NO_WARNINGS /* fopen */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - -/* Unix Large Files support (>4GB) */ -#define _FILE_OFFSET_BITS 64 -#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE -#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _LARGEFILE64_SOURCE -#endif - - -/* ************************************* -* Includes -***************************************/ -#include /* malloc, free */ -#include /* memset */ -#include /* fprintf, fopen, ftello64 */ -#include /* stat64 */ -#include /* stat64 */ -#include /* clock_t, clock, CLOCKS_PER_SEC */ - -/* sleep : posix - windows - others */ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__))) -# include /* sleep */ -# include /* setpriority */ -# define BMK_sleep(s) sleep(s) -# define HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) -#elif defined(_WIN32) -# include -# define BMK_sleep(s) Sleep(1000*s) -# define HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS); -#else -# define BMK_sleep(s) /* disabled */ -# define HIGH_PRIORITY -#endif - -#include "mem.h" -#include "zstd_static.h" -#include "xxhash.h" -#include "datagen.h" /* RDG_genBuffer */ - - -/* ************************************* -* Compiler specifics -***************************************/ -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - -#ifdef _MSC_VER -#define snprintf sprintf_s -#endif - - -/* ************************************* -* Constants -***************************************/ -#define NBLOOPS 3 -#define TIMELOOP_S 1 -#define ACTIVEPERIOD_S 70 -#define COOLPERIOD_S 10 - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); -#define DEFAULT_CHUNKSIZE (4 MB) - -static U32 g_compressibilityDefault = 50; - - -/* ************************************* -* console display -***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ - - -/* ************************************* -* Exceptions -***************************************/ -#ifndef DEBUG -# define DEBUG 0 -#endif -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -/* ************************************* -* Benchmark Parameters -***************************************/ -static U32 g_nbIterations = NBLOOPS; -static size_t g_blockSize = 0; - -void BMK_SetNbIterations(unsigned nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", g_nbIterations); -} - -void BMK_SetBlockSize(size_t blockSize) -{ - g_blockSize = blockSize; - DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10)); -} - - -/* ******************************************************** -* Private functions -**********************************************************/ -static clock_t BMK_clockSpan( clock_t clockStart ) -{ - return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */ -} - - -static U64 BMK_getFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - - -/* ******************************************************** -* Bench functions -**********************************************************/ -typedef struct -{ - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -static int BMK_benchMem(const void* srcBuffer, size_t srcSize, - const char* displayName, int cLevel, - const size_t* fileSizes, U32 nbFiles, - const void* dictBuffer, size_t dictBufferSize) -{ - size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; - blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); - size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* const compressedBuffer = malloc(maxCompressedSize); - void* const resultBuffer = malloc(srcSize); - ZSTD_CCtx* refCtx = ZSTD_createCCtx(); - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* refDCtx = ZSTD_createDCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - U32 nbBlocks; - - /* checks */ - if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx) - EXM_THROW(31, "not enough memory"); - - /* init */ - if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ - HIGH_PRIORITY; - - /* Init blockTable data */ - { const char* srcPtr = (const char*)srcBuffer; - char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb ACTIVEPERIOD_S * CLOCKS_PER_SEC) { - DISPLAY("\rcooling down ... \r"); - BMK_sleep(COOLPERIOD_S); - coolTime = clock(); - } - - /* Compression */ - DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize); - memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ - - clockStart = clock(); - while (clock() == clockStart); - clockStart = clock(); - { U32 nbLoops; - for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { - U32 blockNb; - ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel); - for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s\r", - testNb, displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) ); - - (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ -#if 1 - /* Decompression */ - memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ - - clockStart = clock(); - while (clock() == clockStart); - clockStart = clock(); - - { U32 nbLoops; - for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) { - U32 blockNb; - ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize); - for (blockNb=0; blockNb%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", - testNb, displayName, (U32)srcSize, (U32)cSize, ratio, - (double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC), - (double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) ); - - /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); - if (crcOrig!=crcCheck) { - size_t u; - DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); - for (u=0; u u) break; - bacc += blockTable[segNb].srcSize; - } - pos = (U32)(u - bacc); - bNb = pos / (128 KB); - DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); - break; - } - if (u==srcSize-1) { /* should never happen */ - DISPLAY("no difference detected\n"); - } } - break; - } } /* CRC Checking */ -#endif - } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ - DISPLAY("%2i#\n", cLevel); - } /* Bench */ - - /* clean up */ - free(compressedBuffer); - free(resultBuffer); - ZSTD_freeCCtx(refCtx); - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(refDCtx); - ZSTD_freeDCtx(dctx); - return 0; -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t const step = 64 MB; - BYTE* testmem = NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += step; - if (requiredMem > maxMemory) requiredMem = maxMemory; - - do { - testmem = (BYTE*)malloc((size_t)requiredMem); - requiredMem -= step; - } while (!testmem); - - free(testmem); - return (size_t)(requiredMem); -} - -static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, - const char* displayName, int cLevel, - const size_t* fileSizes, unsigned nbFiles, - const void* dictBuffer, size_t dictBufferSize) -{ - if (cLevel < 0) { /* range mode : test all levels from 1 to l */ - int l; - for (l=1; l <= -cLevel; l++) { - BMK_benchMem(srcBuffer, benchedSize, - displayName, l, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); - } - return; - } - BMK_benchMem(srcBuffer, benchedSize, - displayName, cLevel, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); -} - -static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) -{ - U64 total = 0; - unsigned n; - for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - fclose(f); - } -} - -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) -{ - void* srcBuffer; - size_t benchedSize; - void* dictBuffer = NULL; - size_t dictBufferSize = 0; - size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles); - char mfName[20] = {0}; - const char* displayName = NULL; - - if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); - - /* Load dictionary */ - if (dictFileName != NULL) { - U64 dictFileSize = BMK_getFileSize(dictFileName); - if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName); - dictBufferSize = (size_t)dictFileSize; - dictBuffer = malloc(dictBufferSize); - if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); - BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1); - } - - /* Memory allocation & restrictions */ - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; - if (benchedSize < totalSizeToLoad) - DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - srcBuffer = malloc(benchedSize); - if (!srcBuffer) EXM_THROW(12, "not enough memory"); - - /* Load input buffer */ - BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); - - /* Bench */ - snprintf (mfName, sizeof(mfName), " %u files", nbFiles); - if (nbFiles > 1) displayName = mfName; - else displayName = fileNamesTable[0]; - - BMK_benchCLevel(srcBuffer, benchedSize, - displayName, cLevel, - fileSizes, nbFiles, - dictBuffer, dictBufferSize); - - /* clean up */ - free(srcBuffer); - free(dictBuffer); - free(fileSizes); -} - - -static void BMK_syntheticTest(int cLevel, double compressibility) -{ - char name[20] = {0}; - size_t benchedSize = 10000000; - void* srcBuffer = malloc(benchedSize); - - /* Memory allocation */ - if (!srcBuffer) EXM_THROW(21, "not enough memory"); - - /* Fill input buffer */ - RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); - - /* Bench */ - snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0); - - /* clean up */ - free(srcBuffer); -} - - -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel) -{ - double const compressibility = (double)g_compressibilityDefault / 100; - - if (nbFiles == 0) - BMK_syntheticTest(cLevel, compressibility); - else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel); - return 0; -} - diff --git a/programs/tmp.mgz b/programs/tmp.mgz deleted file mode 100644 index 165a506f973944c12a4a4dbf42d17b733097f1d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5549 zcmV;e6;kRSiwFqlQvOx|19WY00M$HeQ{%>#-}w~{DW)VaU>j!UUW`eBZ5UU~qp(Rb zH<@x|xoxW=OI|$~$Xxcf?>VQtrPjkXPExyjT{U33PoF;TN2@hQds_@XBktL=Lz^A2 zHlO=!?vG=av&a_{b1V3r`3vqHgnk^@oZ0?-5%4f{e2-b~)DN6!HV+9?%U`?%&U6+r zqh+$smgm8c>+*=ztF`(8{CK}bfY zBfI7v4=k76$0OIV6$#9Oy>TR*@i7|-M7ZP%spv~8*p&~Mtq53eFb;gfCQm#dQPkBY z5@Ut=0bwz$2#F6sh`7)MieH#(Md{kZasi}#j+x^T?inZoz5qsG@~h*zpd%KCd=k4A zq02a8KXnJ6?j8s1;`TH9>Ehwx;&$-40npLRhgtr_1ukd4a2-Gmj0Kh#y#W1$;->S^ z`UGn(F1x+%;4`qnuDXNUPQTBt?jG0$yT5oCbX$+TiwAc9_;7#M?;JAL=SUbA+FdT@ z38@MsIp&e&xS?eIGblF%^4u|-Sx+34Y;)%ch_k@l7B6qnpX+;5!i9_ibSldRb0*C5 zqYC>9wn~=0O0yN#_3T3-&i6F{v%KG3kbggd1%TwrnZO2DuI~pGcIk%^F1fj2)q1T~ zJ*XX5YwWRqp)rP5Yx`bg*%57Q#h{6Rff7#%=79YXMq?)3E8q)1bRs`^VW(^sMT_wC z=x7Q&#iK)Ty`$EP<(>TVzN3CDANY2e3;@Nnw;QGOr-f(hS|QHNV^Te=4u!9Hy#v_>v^_nfmIoiGM{f)+f- zo3c5C7Hi5w2B6oM8amr3=^^a!BVhM4|S_pB9Y8*SrFc8e2lQ0Oj1~Xv< zZbUnAfSQm*A3f)p^{ikD76$SPSr{)CegFaT{N(!5BxuFbRkzm}-d$bwJA>h6chF}a zPEZ2O5CCIiI1FQNI5gS2cg)yMM|<}lPGHzH1=0N&jMaCo0LVJ7AB>zR1grY5*a>j| z1-G3E*a3($WNfp-j(Le`lDqZU!goL(VjobfkirC{ zKvCes2?NTOZ4Ky?&4n=EY@cYJ;K6P@=5IDjG&Qk=oq=1q&ggJ95m+&ZhS@2M-g>_(JQ5h7D%MF%5 zMb4b3Bp{RxJpDd|fJR>`tkt_~{m>uYcOKBOm<$P_%lQI41cTBugqp98H}=1V@Xg2c zr`(jK=J-?h_P8!=G&PtQ`~VH+-~VRWK^oEkFYfQb@Pv)0qc<0=Ph{a}ED2Ar9z?5g z#S(&rS!HAd)=v>O6()~B4OqXtWX~wDfs&FKKDHRu~%BJZi;3z&#lE zZ3tI}Sz!;Ii{7An(@EA1TfK{Z-)tbIP>wAjL63;zti&=@Brcm+-wt@i;cQ16z~nVV zJ9X^r#Pjpa3TN3d$jgusJQ!3;erR70VdOHNOdtoLscsY=lI*3T;YNW%Gxq+l+kfa> z>k5%W23WRX;LmqBbagW@S+hxB#ezgexfCbSgq{cyKqCDfhJP*+wq#z~ghNTf?Pc%o z?q18O$H^304^9r%Y%Q6(XbrkQb|8v$@7g&1v6@V^?(TZ&1T3m3F^0P0>3wgl~Em+t+;JUFhI=p~YpcIWEiu{RjDK0V(4(C_}MgVjFa zOXzw$uCwVdX_-cj3$EGb6DxKjpzeLOv7tVZQ6Ew$_~F8}UN+ZiyW79-U3@kU4-eD) zE%T89UdaO}Ylp)hFNTnd`w(>VD|?+EJ3Yg#Fv5WK$#f{G=|OYho;O{SeM|PyQ8n0W zRxtWH5J$Zs+`9_;@O;H14%SW?5atIV{!xeNeJE){6yiQaQRt-QVfM(8b+94va+5 zG-#HKHd)#E`DQry^lg-Lw07FH6|X5)=?<#1(>QwlI4Rr#*;#fmc+qQ?f=fl6 z%mOSSH{D+6Ao-r#Jr1!2qFFkPLTqd|#FT`|soGoWeHBM_CwXI8L>0Uw4v zFlt2-cBesu3c92=mt~kq0MOmVcEpCVpTb|Gcp@|#fU@O;k)p1!cM>WorZq%3Rm6e= z%TZ8LAZj802iPoas|r(s9fr|x?vFXN|5C_RDN*Rbr@lXi_>Z#Ev|xa+Ol^5c2KzS{ z8^w>m$<(m(#e^w@Cv~Q4Gz{$fC=h~_-U&mg^9d+c0J0x^f1VvFEP{F=zYK;1*sJhk zijc#ok=C#`-CM(|jIi*x%rgVNe9zz;7MCMat7lTZx_UI4?H-o&5!QLCu+19$%g7UFrM>*2d zAw94v-OyS+#u|Fa{XP?bPn8%jE#wdc?YvI}oWxIg0T@E`g zAk_hd9|(aq#MwWdvCTLM_$UeC+XazfNY6+D2Q*s|^TaxSwdHR`I(|pe%*6ZB2H+UW zP|uGT_xyM|!^27FilHh^2*e{`ojeG(kcJE`KA7jVkER|l*god00@y+Aqck9V-r;-h zi(15LA1UlWizW{YaktEoQA$u33DUU7vmdqOTWJD7XVON^Bs4Y>i3sonn85OS$U}<^ zkizRKN?}p$7!7Yn=@`06|y z`qId(NTZnA8OXpVinx=CW`>bsTp7-m;UN6EMvwin9kU2XBCL@Bo+&)(P*Dbj zD2LDVF10*>={Dh~vXpxKqz3hi+VIO^Vm^8yv38YQv%!kQlJuhDE!0UqGkm|5O003p zvp@!2iL9hp_SbH#z?KtGTq9+%9(ii02c@vD>R%E$zFNWDnNL}~ka13hjm%-Ok@x#b zF?&{6^{`gOpYP#kO$(QzzSY9f*pD$vO{_5DVblVN;}Jc54l8Q7of+1G2Q(Ejas~EX z;GmoT{PQR1DD?79)^)6^=_+ZoeBT{7bF>oKS!F{L?v9};8B7||UxVF`{|tW(p!eL_ zF>}F7EJLar5(Rc$My}J|XWRMqSwb%jXm(NAQi6m6j{pv{)5Vr@$-Iyw{)OE?V9HVZ zy?$uDmw(!I3JTK1B-M>g=nOJF9wA2DR)>W;6ei7V7p!z*vh)05cz)Zmkf&(~F`);H z{jVOL;^8!Cti)&2=QR2mdkNqwgTc>hK?W`Dx=)Vxgw)--bFd2qz+vrU$WC`_)tDWe zlWhrmsAL-s%0<~SPkqGW(-J4GK6l=ixGEN{()ZEvEl5qE@sO+2;Zgd8s9-hXfMSUt zzRoaj8tS~F+0@pUB__BM8Ab|GAZjO5`bU-wG>c9ExiV)|>E`TAGxLVRS)Mwbl#~addRU6teLsXuXOxu?2*DA7kEU~tPHMu{QK=SKZQY@>H^WuV|awpbG5%p0lSM#wzcgW4$Lmoslmy5WO z6QfG2iV;;+9;K`mHL^Oc(=TK9{o(P%tn7X`tWDU>SYEqj}arGd24Cnv6AIUp8LIp7XJT&+0HNG474?p`$=sz^&hxZ(Yo&8K#Oh zqLQ@Y>;Ge-ZWj`D`!6QyKb7v-e@5#23)tFkNTk>wKD1a1kGZi^&DkrZ3;mFT0FzUW zYzizr=+KQq=I8CELbNPa$T%C9+gGCQ?d@$QFOD<(zFRv1wX?4GWVy~*Fcs2W`rT)? z-Nc_Fa^&=>E+(h~A~z&RU6EK~Oeyi1dUna;)meo6L!NkFejaqR6L0gE_zO~+oz}&S z5bHuL=Gl->(QrrjW8s%1`Vx7MEZZ*DyGzB7Mba3JhVHrGJs{W!L$taM5wD4=Bs?ON zb`>;3ph&YajjWcY`k+*afqgwV5`y$B_MFA$x}d8*0$Fo8w$&(>$8P+fq|}){1S#5V zjkHE2CR_+EUaS9u{cyQvMWQJ~WlwO@ctpR2s!ZVyIb&Hz$e&GGsG_*pl;S(6;VnbD z$1PqQ^7s=}$TMpp-}iVEYjn#H9fNx|y_Cf5h>z*!Rw;t={I`THbw*thCAEtkEQR{p z4qr9)_3ZW?sVj3_BX>>?w$(|u?Q%$>@j{`(cOMGJauNs4WulDs@DwNMWI>$6lup^j z@wQyjE?JXQ#Bw6J_M+bhl0gZJkBQ@rZ|F`iUVKsS!i?eX(z^zb{qX#lZU&1DD2|eN zp?Sm0Y>&O3%!C>;^hkWdmJXbsv-$^o+zz&Sc81>@g+=r|Z)O%ZR?Kt|-N zB&`W{y*%I4ufJ+$j20*@ck9nHm0@P7?YxPTl^nd5-C-Q~k>!?MWE4h;M1@pU6fIMk z6Jm=HOFZPQ}cx5l&s zH0I*P*B*?7>^E_?^_zHc7hja+V7>l?_zaorgn022CmoOdmfiNZLqc>Bfq#S%-L!Bu zjDQG2k4sO2^I<0?M|kNd7je?(8!H^W7$;T$|0yL(azw$W9#Otojhb_#H7GLK+trC5 zyo#^xY)$Yg3cj>hTv1^=!ALKhFrH1CP0W^=#}HM3bmqWP=#0ewG(hj@(yO8L&|wuZ znQQiSz=T$WRn=}NACWS8O1c^gk*1_^u4(%K3s_xH*{Q_Io1Q$oktef}@3RoIEL7uR ztiRix|!2ihr8PlTkxBN+KdQK_c63 z7T_~Vz=uJ$D4cyk!+P0^#$&&kkgPv$NG~XPUs!Q-EgRXcAA-fH3Up~vfPy9}Hj962 zPxi7!f=y14uM(91+t}kw^h7 zf!FUR5)UXceCIyB(nD{>YnnrA7hxtF@D$9Fpr@nA1&7SabM=~FE{JJ_$>2qCz{NM+ zcPeZN`ho>0HR^kofv0z5msUZFbTL^HdrNPGlKeGhjNO=xUZB=A)mhBa4YR3q)hz+7 zP^zml-lL%o5ZYmNADdz35xv z{$u90P^!v-r6_gqvjVnaSALtpGp z61kn87_Rlrf;T$eBGw;B7Oh}Og{@n_^7y0n%3QTe-i9_rRH~=|DE>5oa1<9+j63;g zh8(NNd5*1kHf{+N0uVQ2sbY-kO)y5pC74O*pG$~Im}t|_Kj?INX~an~CD^0=a3%`f$e z`r146SNQ)hm3c+>;^{K4$VmmSomRy|kTtI0>5kw@H|Q%Ru-$sRySt-5j4*Or?SU*+ z?0BO&Gf%f!_AWCAlEQqGC4Q~dp($nFbrM13ua|tAlzEAT>Tk73F=>w%m%hqRPtdgL vN5z{a)OTsrFM5d*WZpH>aFkh`D~p#z(Mpcp%zJ?0!{z`S^2=d diff --git a/programs/tmp.zstd b/programs/tmp.zstd deleted file mode 100644 index 6f0296f66bdaf65851b97f0f417482e182131ac2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6317 zcmV;e7*gjZwJ-gJ0!;uIpd_9RBr}j8m@%sYqR2~R?=8W)$Yxt-l!?9lziC69E|*hI z7Z~SHgDd*VmCSav8IcG8fXIjhLMj7T13m+m(%3yhX&6nkNx6&}J`E&Fb06L6=8s4T z3R8Q60-aJ+XRgzFx1L8Rj>}2gK+<47p$II~a5fJbh>ruO+L`Az<=ooC$9?A~0DX`1 z)Fc&z<9Q>s*z1QqLis;UQGNR-HNiz3F|T~uei(c(0IcnSr|xD~);tD93czw^8nr?Rb= z^OpQko)-TviK?X-p4}eLb;azh@4F(7&e6|~@IQ+H(;0Jf?`ykXD|?jRjr|i179sHo zN&Mr$31K#%kAuV={)u=vhG{&Y&BL!f0?>xDxj2A44hAqEP3Qk5X^)X>yf<2$>q)(C zc11sbc9f^ZEeJp!htJ1>fFtlY92AN$0NN1n{899ijlZ{cN~gv1e2%01_U!0A%CGa> zBu9FT8b4J#vuA%li}4)c|4Y}Ns=V5XZ&&>GD9^s7+LlLmr6tC|}y+?sk5U@^5qXd!GLTbmB;g4ydrzO<2eE~uDwmpD;4KRK>WWX1z=SbNyVxv&rPYtyw-@| z69ztyMDt>J7W4lS?nZfDE28n$PK^6{o!#Es{VbN$_UtKc>}-D1kF(uv*O^PcO-5U7 z?f<0+CDLRdS?+V4-No8;$~qspakM?-YB$&!6Kgsja*MJ9oy!vxLXxy0Q6jHhg)jiW z7;f{q&a<2JHG24zg{AU#g}t1Z;#!M*W5y((V7*`N`+Q@@at3SkFpAC6Ej!S;e5;>) z=U6eX#2=BS%4%qUQw1ReF9-fX-Wd~L+GA0j_`CkghGdzjN1c#-enKvJK+At=fiz*y zT8>uFbrnrqKj_JgWd}N!bJJExlBN3f&X^d*W~r8mZn=)DojKml8?D~X-NhRvE*sKB zo)`V76OV~{eBl#v%CbyZrtxezS#~PRG@Q*tmYsUki7$LY-gVMb81fv?{n~pqkN?ud z0%_vfdm9oE@i73(It6GZh|ooofuN8CC=>w^x@f9Hgf5y4MChW)TsBwbG>|C0U1+ME zId<2|I)CtmC;Yb@eBlWHE%@No&b-Cjxy~Ov(+~Ik!53cnZ^75z+h4H$TWtNpD@*0q zH*Q(*iCzt~E1Hvu;N|=@nM_Vm$k7a-;ba*G@o~r;2FS8g_coU?l>ev6WC9U5a3rt| zMZ$>G!;^bci>uu%RRF)9=@(^ZF!H2R%S5jrgf5*Y!2cQpHrN4A$N(k-aY7Ccnv7+d zECk5}01+ZodrfncuCJb}nU@}c-1L8PnKBp##ac}lYZd?X+O(B>1d`AB32AT|a*Vd!;JoE_*~ zj*;iJ^Ln}QBb2@rjDgQ(JnQ!xNYgZBnE{MLN5hejaL~~Bq@nRC%Pwa9vJ7$+1kZ+m z%Xxd0?Ip(+maEg^l3z@0y)WGBruY~QmVGo58Fl~@!NHuO*eu;n<&0vpbn9;B95c=_ z<3}jCJ&a7j7zqlN($4Y}C3BtElXf=lT2$wywwk9-{220ue~?!@Q5h~%ThtjzA#+ik z$~;1`E@!o~xz`?9oNN5m@0QHOO7$9_r7;GTBZOr7gcWD zbvIQILXe^!ccMY^ud*Nu#|5lO0ZRgr%nIZY%J0PgPbQOB3(H+pC$)CF6A#$%30}_5 z_Il5;N4G0%Wqlzcz<=GdmP4^w`s;S)cqhjX;4g+-tY3$}Rz4o59(MuoKk`@g_CRRu7qPF2>V zIz>V6yQrJpjTRe{>)^ENr$bUtYH6@<_!EFbBL=4{V( zp6V`oDC)M9-(|b_5pLRwt~;ZxwwCK`D8QjJY48i5BD?^fG{`VT2tR35gq}@DvgE9b zH%e^azPN8^?PvRT@y!z3#j?5fF!E@ted&yerv!x%q(nZMOgL*rx1MU6nzLP&@x*Zr zjbC^{ol#IsQdv^GLX>z~;uXB{A#ktOS(&MI6UfSo{x!Uc31CF_9IsY#$e1f>? zeBh#`*@4dGjtE7PHYAIJn&9Bzpw?Tei^gp86BI&CwFtu9MGuYHf+%{6&`{&aCOS;K<83%ne8HDFx^*X84zXw?7p)5O7DA(O}C9loiLacNQCm&39j?j zxw-dN`?i#myRBXv{dPsy1)nU*b$#%1Qja?EiTWVdK78tf{Bqy`P_f`C$JYCG&7*Bw z)Gql4tv7vdxTx{Lw=HV$ax&af{krB6HHN=devf~UW9vP~i|XX;)|qXK8n)gk&7+;t zJfg-5z68@R#}#}DCJhL|1;Weu-NNwQQe7YXwQ~9ed> z9FE`NaD0~oUQUXwN0r^89~PaoW6XQhWFh>P7Y(E;JvkhJuRjy^H?Htfvs%2T=t7;;F>bq(Db8E740!vN}?%xtzZTcbE zC$gh>R5qYCOcRoA$USPMWsM|uExuMUAq^R9{*Gu zb;^@dGVj)2&&5zstb@9V40E?vw65$@AyaF$3xKHf!U&v^`L;2h`!4F%cM*0cfJ4Om zf1=y=t3tLn8l%EV!U^@XB*uq1g94K2F8|u=h1f2LUs%Y>kj1Z0ZS8(5hg=djRyqN=@HZDQ&kik7s zra2Ve9w>~uIf04b_?Y18IXo;}|2)ajyJIs8&|^IdhW$QR^=f>ma9H|t(PO>#iC^KZ z;hRxjKc#d`R&dnF>vmw(cMkLDh`QJwjA|FQCi_`KJJB;kPWJ{Sna@<9T2yZBUzcq; zJ*Fo;7Lzm)0G8&>-Ng?aS=EwcoPO`4sGO`06JFh z{|5a92W-#D<^_FpKcVNg^8V#Ek@fsq^>#rK~tsY5i4jk!UU zw`lo~%i{w~K08||zqHXA)URTWZA6+YJf_2w9c+tsr@77VmsdwDM7;WFAA2Ag$2D(cx`n>t#DHv&u#QN|M0J{g z&a>%y2RhdZTRKi_2&M<-+rklA0ArOglJ0OxRC%;6kNtadj0DxIk7Zn=nG z%L}}H@0~rSxT=S!R)|6HSGr;0YMawcIF4zHP$s74F!2~?aR3j=YQt`nWXI_Wqw)-H z~~~V4);b69QEf5axDZin3#2QnGO0Bzxr^Ynw-U zAu~yrBC4{2T#w`hwgkW_O0v+}DOGMjB4NsytrTFTUwXRN&986P!bFCLM0= z5Hz;4VV^s0X|{=-E7?Lcmgnx*1HqyM@^XrfY9cBuJXU*isK3C+CI*Oga6P>saroPS zZOds3k@GPgHa6HBPy=cNfp87i>0HFc$C++*ly+&9T!<0TyXU+d&tTE_#OU zlGK$eRy6qNteaw5bc*EExmsT$?Oh%16;8DvlV|EMBg$=jSR;C%5Xo9>G(D-bVmJjk zdX@ol(+@RkUHf1kC&PZBf+xe*1E@9Z@?{5SRzua7eN8{Bs^55`7>;o}T%wX#9$a~k ze~mdtIFGSVut8MMUSM3YZe^XPKDa92Yy| zVBvHQZ82;5KNY8{=D@=eba>n>Pgp>)UkLuz^!DV)lW#%XmazR&%>+npsP;KxfAP|to-q|iV)Q)dm3)J4P&Xu-$TCvIG8 z!A|9u7dS^R0)A42x75%dHv~vZTN;s{WGGqLn1sY)p?qX)H3-2x<{1}O_aLHP0d|pA z9uH17a^T5Y)W<^7OB2Z*pr*p?1Do)nvXMoUKc+@(Whn$kCtfDzkjV9kd)$D#eR%O- zo~ut$JxWkgC$#Bf`iNYGOX7Dl>%%AFrnqeRRE5kt}S zmA#N8(&!OsK?nW_AgX$YlF5A9Kju8v%j#k2QReAaQ8ug4(V-k?t43{ONGN0HEX87w z$jLoMx+yp^`VhmjjfBftAbq>2D|_nsBsCf)aZP!!x@*)MXC^JIC#-0}Tw|jCp(v+d z_tDAI1cUy7n>vsu+v;&K&@<|TzHw1yYne%@gWpkH;vVf($Pb~`c|8Z9mqQDoHObON z|L)VTr~B8)xd@!|L|WUkf$AP?!i8!b06}Ze)=z}PsEEYAP+dtM_|GIHU7r+1jLj5!( From 40358d01aa0ddd94e92b2cb7d9a2bd04f0f51b73 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 2 Apr 2016 00:40:09 +0200 Subject: [PATCH 2/4] Fix : rare bugs when hashLog < min in bt mode --- lib/zstd_compress.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 31d81ea2a..65d7258d9 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -193,6 +193,7 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t if (params->contentLog > maxContentLog) params->contentLog = maxContentLog; } /* <= ZSTD_CONTENTLOG_MAX */ if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ } From ef36390e4a2cb51b28f9b4a1d33b47a6dcaf745a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 2 Apr 2016 00:46:40 +0200 Subject: [PATCH 3/4] refined exception for hashlog < min --- lib/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 65d7258d9..5fe078fcd 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -164,8 +164,8 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz if (srcSize > (1ULL << ZSTD_WINDOWLOG_MIN)) return ZSTD_checkCParams(cParams); if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported); if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */ - if (srcSize <= (1ULL << cParams.hashLog)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */ if (srcSize <= (1ULL << cParams.contentLog)) cParams.contentLog = ZSTD_CONTENTLOG_MIN; /* fake value - temporary work around */ + if ((srcSize <= (1ULL << cParams.hashLog)) && ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */ return ZSTD_checkCParams(cParams); } From 5a88a70f970780f93f57fbc29b98838ee530560f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 3 Apr 2016 20:24:25 +0200 Subject: [PATCH 4/4] fixed buffer sync error in zbuff --- lib/zbuff.c | 36 +++++++++++++++++++----------------- lib/zstd_decompress.c | 14 +++++++------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/lib/zbuff.c b/lib/zbuff.c index 06a7ec4e8..721fad227 100644 --- a/lib/zbuff.c +++ b/lib/zbuff.c @@ -136,7 +136,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, zbc->inBuff = (char*)malloc(neededInBuffSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); } - zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize); + zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); } if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; @@ -222,7 +222,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, /* prepare next block */ zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; if (zbc->inBuffTarget > zbc->inBuffSize) - { zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; } /* note : inBuffSize >= blockSize */ + zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */ zbc->inToCompress = zbc->inBuffPos; if (cDst == op) { op += cSize; break; } /* no need to flush */ zbc->outBuffContentSize = cSize; @@ -326,10 +326,11 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader, struct ZBUFF_DCtx_s { ZSTD_DCtx* zc; ZSTD_frameParams fParams; - char* inBuff; + size_t blockSize; + char* inBuff; size_t inBuffSize; size_t inPos; - char* outBuff; + char* outBuff; size_t outBuffSize; size_t outStart; size_t outEnd; @@ -405,20 +406,21 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, } } /* Frame header instruct buffer sizes */ - { size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */ - if (zbc->inBuffSize < neededInSize) { + { size_t const blockSize = MIN(1 << zbc->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); + zbc->blockSize = blockSize; + if (zbc->inBuffSize < blockSize) { free(zbc->inBuff); - zbc->inBuffSize = neededInSize; - zbc->inBuff = (char*)malloc(neededInSize); + zbc->inBuffSize = blockSize; + zbc->inBuff = (char*)malloc(blockSize); if (zbc->inBuff == NULL) return ERROR(memory_allocation); - } } - { size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog; - if (zbc->outBuffSize < neededOutSize) { - free(zbc->outBuff); - zbc->outBuffSize = neededOutSize; - zbc->outBuff = (char*)malloc(neededOutSize); - if (zbc->outBuff == NULL) return ERROR(memory_allocation); - } } + } + { size_t const neededOutSize = ((size_t)1 << zbc->fParams.windowLog) + blockSize; + if (zbc->outBuffSize < neededOutSize) { + free(zbc->outBuff); + zbc->outBuffSize = neededOutSize; + zbc->outBuff = (char*)malloc(neededOutSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } } } zbc->stage = ZBUFFds_read; case ZBUFFds_read: @@ -471,7 +473,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, zbc->outStart += flushedSize; if (flushedSize == toFlushSize) { zbc->stage = ZBUFFds_read; - if (zbc->outStart + ZSTD_BLOCKSIZE_MAX > zbc->outBuffSize) + if (zbc->outStart + zbc->blockSize > zbc->outBuffSize) zbc->outStart = zbc->outEnd = 0; break; } diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 68bcaa8e1..fd9b40233 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -686,18 +686,18 @@ FORCE_INLINE size_t ZSTD_execSequence(BYTE* op, size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ BYTE* const oend_8 = oend-8; - const BYTE* const litEnd = *litPtr + sequence.litLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; /* check */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */ /* copy Literals */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ op = oLitEnd; - *litPtr = litEnd; /* update for next sequence */ + *litPtr = iLitEnd; /* update for next sequence */ /* copy Match */ if (sequence.offset > (size_t)(oLitEnd - base)) { @@ -761,7 +761,6 @@ static size_t ZSTD_decompressSequences( const BYTE* litPtr = dctx->litPtr; const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8; const BYTE* const litEnd = litPtr + dctx->litSize; - int nbSeq; U32* DTableLL = dctx->LLTable; U32* DTableML = dctx->MLTable; U32* DTableOffb = dctx->OffTable; @@ -769,13 +768,14 @@ static size_t ZSTD_decompressSequences( const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const U32 mls = dctx->fParams.mml; + int nbSeq; /* Build Decoding Tables */ - { size_t const errorCode = ZSTD_decodeSeqHeaders(&nbSeq, + { size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, ip, seqSize); - if (ZSTD_isError(errorCode)) return errorCode; - ip += errorCode; + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; } /* Regen sequences */