# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ################################################################
-RELEASE=r109
+RELEASE=r110
DESTDIR=
PREFIX=${DESTDIR}/usr
BINDIR=$(PREFIX)/bin
MANDIR=$(PREFIX)/share/man/man1
DISTRIBNAME=lz4-$(RELEASE).tar.gz
CC=gcc
-CFLAGS=-I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"v1.0.9\"
+CFLAGS=-I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\"
# Define *.exe as extension for Windows systems
# ifeq ($(OS),Windows_NT)
clean:
@rm -f core *.o lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \
- fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT)
+ fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) \
+ $(DISTRIBNAME)
@echo Cleaning completed
-r109 :\r
-lz4.c : corrected issue 98 (LZ4_compress_limitedOutput())\r
-Makefile : can specify version number from makefile\r
-\r
-r108 :\r
-lz4.c : corrected compression efficiency issue 97 in 64-bits chained mode (-BD) for streams > 4 GB (thanks Roman Strashkin for reporting)\r
-\r
-r107 :\r
-Makefile : support DESTDIR for staged installs. Thanks Jorge Aparicio.\r
-Makefile : make install installs both lz4 and lz4c (Jorge Aparicio)\r
-Makefile : removed -Wno-implicit-declaration compilation switch\r
-lz4cli.c : include <stduni.h> for isatty() (Luca Barbato)\r
-lz4.h : introduced LZ4_MAX_INPUT_SIZE constant (Shay Green)\r
-lz4.h : LZ4_compressBound() : unified macro and inline definitions (Shay Green)\r
-lz4.h : LZ4_decompressSafe_partial() : clarify comments (Shay Green)\r
-lz4.c : LZ4_compress() verify input size condition (Shay Green)\r
-bench.c : corrected a bug in free memory size evaluation\r
-cmake : install into bin/ directory (Richard Yao)\r
-cmake : check for just C compiler (Elan Ruusamae)\r
-\r
-r106 :\r
-Makefile : make dist modify text files in the package to respect Unix EoL convention\r
-lz4cli.c : corrected small display bug in HC mode\r
-\r
-r105 :\r
-Makefile : New install script and man page, contributed by Prasad Pandit\r
-lz4cli.c : Minor modifications, for easier extensibility\r
-COPYING : added license file\r
-LZ4_Streaming_Format.odt : modified file name to remove white space characters\r
-Makefile : .exe suffix now properly added only for Windows target
\ No newline at end of file
+r110 :
+lz4 & lz4hc : added capability to allocate state & stream state with custom allocator (issue 99)
+fuzzer & fullbench : updated to test new functions
+man : documented -l command (Legacy format, for Linux kernel compression) (issue 102)
+cmake : improved version by Mika Attila, building programs and libraries (issue 100)
+xxHash : updated to r33
+Makefile : clean also delete local package .tar.gz
+
+r109 :
+lz4.c : corrected issue 98 (LZ4_compress_limitedOutput())
+Makefile : can specify version number from makefile
+
+r108 :
+lz4.c : corrected compression efficiency issue 97 in 64-bits chained mode (-BD) for streams > 4 GB (thanks Roman Strashkin for reporting)
+
+r107 :
+Makefile : support DESTDIR for staged installs. Thanks Jorge Aparicio.
+Makefile : make install installs both lz4 and lz4c (Jorge Aparicio)
+Makefile : removed -Wno-implicit-declaration compilation switch
+lz4cli.c : include <stduni.h> for isatty() (Luca Barbato)
+lz4.h : introduced LZ4_MAX_INPUT_SIZE constant (Shay Green)
+lz4.h : LZ4_compressBound() : unified macro and inline definitions (Shay Green)
+lz4.h : LZ4_decompressSafe_partial() : clarify comments (Shay Green)
+lz4.c : LZ4_compress() verify input size condition (Shay Green)
+bench.c : corrected a bug in free memory size evaluation
+cmake : install into bin/ directory (Richard Yao)
+cmake : check for just C compiler (Elan Ruusamae)
+
+r106 :
+Makefile : make dist modify text files in the package to respect Unix EoL convention
+lz4cli.c : corrected small display bug in HC mode
+
+r105 :
+Makefile : New install script and man page, contributed by Prasad Pandit
+lz4cli.c : Minor modifications, for easier extensibility
+COPYING : added license file
+LZ4_Streaming_Format.odt : modified file name to remove white space characters
+Makefile : .exe suffix now properly added only for Windows target
-/*\r
- bench.c - Demo program to benchmark open-source compression algorithm\r
- Copyright (C) Yann Collet 2012-2013\r
- GPL v2 License\r
-\r
- This program is free software; you can redistribute it and/or modify\r
- it under the terms of the GNU General Public License as published by\r
- the Free Software Foundation; either version 2 of the License, or\r
- (at your option) any later version.\r
-\r
- This program is distributed in the hope that it will be useful,\r
- but WITHOUT ANY WARRANTY; without even the implied warranty of\r
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
- GNU General Public License for more details.\r
-\r
- You should have received a copy of the GNU General Public License along\r
- with this program; if not, write to the Free Software Foundation, Inc.,\r
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-// Disable some Visual warning messages\r
-#define _CRT_SECURE_NO_WARNINGS\r
-#define _CRT_SECURE_NO_DEPRECATE // VS2005\r
-\r
-// Unix Large Files support (>4GB)\r
-#define _FILE_OFFSET_BITS 64\r
-#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions\r
-# define _LARGEFILE_SOURCE \r
-#elif ! defined(__LP64__) // No point defining Large file for 64 bit\r
-# define _LARGEFILE64_SOURCE\r
-#endif\r
-\r
-// S_ISREG & gettimeofday() are not supported by MSVC\r
-#if defined(_MSC_VER) || defined(_WIN32)\r
-# define BMK_LEGACY_TIMER 1\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Includes\r
-//**************************************\r
-#include <stdlib.h> // malloc\r
-#include <stdio.h> // fprintf, fopen, ftello64\r
-#include <sys/types.h> // stat64\r
-#include <sys/stat.h> // stat64\r
-\r
-// Use ftime() if gettimeofday() is not available on your target\r
-#if defined(BMK_LEGACY_TIMER)\r
-# include <sys/timeb.h> // timeb, ftime\r
-#else\r
-# include <sys/time.h> // gettimeofday\r
-#endif\r
-\r
-#include "lz4.h"\r
-#define COMPRESSOR0 LZ4_compress\r
-#include "lz4hc.h"\r
-#define COMPRESSOR1 LZ4_compressHC\r
-#define DEFAULTCOMPRESSOR COMPRESSOR0\r
-\r
-#include "xxhash.h"\r
-\r
-\r
-//**************************************\r
-// Compiler specifics\r
-//**************************************\r
-#if !defined(S_ISREG)\r
-# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)\r
-#endif\r
-\r
-// GCC does not support _rotl outside of Windows\r
-#if !defined(_WIN32)\r
-# define _rotl(x,r) ((x << r) | (x >> (32 - r)))\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Basic Types\r
-//**************************************\r
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99\r
-# include <stdint.h>\r
- typedef uint8_t BYTE;\r
- typedef uint16_t U16;\r
- typedef uint32_t U32;\r
- typedef int32_t S32;\r
- typedef uint64_t U64;\r
-#else\r
- typedef unsigned char BYTE;\r
- typedef unsigned short U16;\r
- typedef unsigned int U32;\r
- typedef signed int S32;\r
- typedef unsigned long long U64;\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Constants\r
-//**************************************\r
-#define NBLOOPS 3\r
-#define TIMELOOP 2000\r
-\r
-#define KB *(1U<<10)\r
-#define MB *(1U<<20)\r
-#define GB *(1U<<30)\r
-\r
-#define KNUTH 2654435761U\r
-#define MAX_MEM (2 GB - 64 MB)\r
-#define DEFAULT_CHUNKSIZE (4 MB)\r
-\r
-\r
-//**************************************\r
-// Local structures\r
-//**************************************\r
-struct chunkParameters\r
-{\r
- U32 id;\r
- char* origBuffer;\r
- char* compressedBuffer;\r
- int origSize;\r
- int compressedSize;\r
-};\r
-\r
-struct compressionParameters\r
-{\r
- int (*compressionFunction)(const char*, char*, int);\r
- int (*decompressionFunction)(const char*, char*, int);\r
-};\r
-\r
-\r
-//**************************************\r
-// MACRO\r
-//**************************************\r
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)\r
-\r
-\r
-\r
-//**************************************\r
-// Benchmark Parameters\r
-//**************************************\r
-static int chunkSize = DEFAULT_CHUNKSIZE;\r
-static int nbIterations = NBLOOPS;\r
-static int BMK_pause = 0;\r
-\r
-void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }\r
-\r
-void BMK_SetNbIterations(int nbLoops)\r
-{\r
- nbIterations = nbLoops;\r
- DISPLAY("- %i iterations -\n", nbIterations);\r
-}\r
-\r
-void BMK_SetPause() { BMK_pause = 1; }\r
-\r
-\r
-//*********************************************************\r
-// Private functions\r
-//*********************************************************\r
-\r
-#if defined(BMK_LEGACY_TIMER)\r
-\r
-static int BMK_GetMilliStart()\r
-{\r
- // Based on Legacy ftime()\r
- // Rolls over every ~ 12.1 days (0x100000/24/60/60)\r
- // Use GetMilliSpan to correct for rollover\r
- struct timeb tb;\r
- int nCount;\r
- ftime( &tb );\r
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);\r
- return nCount;\r
-}\r
-\r
-#else\r
-\r
-static int BMK_GetMilliStart()\r
-{\r
- // Based on newer gettimeofday()\r
- // Use GetMilliSpan to correct for rollover\r
- struct timeval tv;\r
- int nCount;\r
- gettimeofday(&tv, NULL);\r
- nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);\r
- return nCount;\r
-}\r
-\r
-#endif\r
-\r
-\r
-static int BMK_GetMilliSpan( int nTimeStart )\r
-{\r
- int nSpan = BMK_GetMilliStart() - nTimeStart;\r
- if ( nSpan < 0 )\r
- nSpan += 0x100000 * 1000;\r
- return nSpan;\r
-}\r
-\r
-\r
-static size_t BMK_findMaxMem(U64 requiredMem)\r
-{\r
- size_t step = (64 MB);\r
- BYTE* testmem=NULL;\r
-\r
- requiredMem = (((requiredMem >> 26) + 1) << 26);\r
- requiredMem += 2*step;\r
- if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;\r
-\r
- while (!testmem)\r
- {\r
- requiredMem -= step;\r
- testmem = (BYTE*) malloc ((size_t)requiredMem);\r
- }\r
-\r
- free (testmem);\r
- return (size_t) (requiredMem - step);\r
-}\r
-\r
-\r
-static U64 BMK_GetFileSize(char* infilename)\r
-{\r
- int r;\r
-#if defined(_MSC_VER)\r
- struct _stat64 statbuf;\r
- r = _stat64(infilename, &statbuf);\r
-#else\r
- struct stat statbuf;\r
- r = stat(infilename, &statbuf);\r
-#endif\r
- if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...\r
- return (U64)statbuf.st_size;\r
-}\r
-\r
-\r
-//*********************************************************\r
-// Public function\r
-//*********************************************************\r
-\r
-int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)\r
-{\r
- int fileIdx=0;\r
- char* orig_buff;\r
- struct compressionParameters compP;\r
- int cfunctionId;\r
-\r
- U64 totals = 0;\r
- U64 totalz = 0;\r
- double totalc = 0.;\r
- double totald = 0.;\r
-\r
-\r
- // Init\r
- if (cLevel <3) cfunctionId = 0; else cfunctionId = 1;\r
- switch (cfunctionId)\r
- {\r
-#ifdef COMPRESSOR0\r
- case 0 : compP.compressionFunction = COMPRESSOR0; break;\r
-#endif\r
-#ifdef COMPRESSOR1\r
- case 1 : compP.compressionFunction = COMPRESSOR1; break;\r
-#endif\r
- default : compP.compressionFunction = DEFAULTCOMPRESSOR;\r
- }\r
- compP.decompressionFunction = LZ4_decompress_fast;\r
-\r
- // Loop for each file\r
- while (fileIdx<nbFiles)\r
- {\r
- FILE* inFile;\r
- char* inFileName;\r
- U64 inFileSize;\r
- size_t benchedSize;\r
- int nbChunks;\r
- int maxCompressedChunkSize;\r
- size_t readSize;\r
- char* compressedBuffer; int compressedBuffSize;\r
- struct chunkParameters* chunkP;\r
- U32 crcOrig;\r
-\r
- // Check file existence\r
- inFileName = fileNamesTable[fileIdx++];\r
- inFile = fopen( inFileName, "rb" );\r
- if (inFile==NULL)\r
- {\r
- DISPLAY( "Pb opening %s\n", inFileName);\r
- return 11;\r
- }\r
-\r
- // Memory allocation & restrictions\r
- inFileSize = BMK_GetFileSize(inFileName);\r
- benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;\r
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;\r
- if (benchedSize < inFileSize)\r
- {\r
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));\r
- }\r
-\r
- // Alloc\r
- chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));\r
- orig_buff = (char*)malloc((size_t )benchedSize);\r
- nbChunks = (int) (benchedSize / chunkSize) + 1;\r
- maxCompressedChunkSize = LZ4_compressBound(chunkSize);\r
- compressedBuffSize = nbChunks * maxCompressedChunkSize;\r
- compressedBuffer = (char*)malloc((size_t )compressedBuffSize);\r
-\r
-\r
- if (!orig_buff || !compressedBuffer)\r
- {\r
- DISPLAY("\nError: not enough memory!\n");\r
- free(orig_buff);\r
- free(compressedBuffer);\r
- free(chunkP);\r
- fclose(inFile);\r
- return 12;\r
- }\r
-\r
- // Init chunks data\r
- {\r
- int i;\r
- size_t remaining = benchedSize;\r
- char* in = orig_buff;\r
- char* out = compressedBuffer;\r
- for (i=0; i<nbChunks; i++)\r
- {\r
- chunkP[i].id = i;\r
- chunkP[i].origBuffer = in; in += chunkSize;\r
- if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }\r
- chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;\r
- chunkP[i].compressedSize = 0;\r
- }\r
- }\r
-\r
- // Fill input buffer\r
- DISPLAY("Loading %s... \r", inFileName);\r
- readSize = fread(orig_buff, 1, benchedSize, inFile);\r
- fclose(inFile);\r
-\r
- if (readSize != benchedSize)\r
- {\r
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);\r
- free(orig_buff);\r
- free(compressedBuffer);\r
- free(chunkP);\r
- return 13;\r
- }\r
-\r
- // Calculating input Checksum\r
- crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0);\r
-\r
-\r
- // Bench\r
- {\r
- int loopNb, chunkNb;\r
- size_t cSize=0;\r
- double fastestC = 100000000., fastestD = 100000000.;\r
- double ratio=0.;\r
- U32 crcCheck=0;\r
-\r
- DISPLAY("\r%79s\r", "");\r
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)\r
- {\r
- int nbLoops;\r
- int milliTime;\r
-\r
- // Compression\r
- DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize);\r
- { size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } // warmimg up memory\r
-\r
- nbLoops = 0;\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliStart() == milliTime);\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)\r
- {\r
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)\r
- chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);\r
- nbLoops++;\r
- }\r
- milliTime = BMK_GetMilliSpan(milliTime);\r
-\r
- if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;\r
- cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;\r
- ratio = (double)cSize/(double)benchedSize*100.;\r
-\r
- DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.);\r
-\r
- // Decompression\r
- { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing area, for CRC checking\r
-\r
- nbLoops = 0;\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliStart() == milliTime);\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)\r
- {\r
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)\r
- chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);\r
- nbLoops++;\r
- }\r
- milliTime = BMK_GetMilliSpan(milliTime);\r
-\r
- if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;\r
- DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);\r
-\r
- // CRC Checking\r
- crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);\r
- if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }\r
- }\r
-\r
- if (crcOrig==crcCheck)\r
- {\r
- if (ratio<100.)\r
- DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);\r
- else\r
- DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);\r
- }\r
- totals += benchedSize;\r
- totalz += cSize;\r
- totalc += fastestC;\r
- totald += fastestD;\r
- }\r
-\r
- free(orig_buff);\r
- free(compressedBuffer);\r
- free(chunkP);\r
- }\r
-\r
- if (nbFiles > 1)\r
- DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);\r
-\r
- if (BMK_pause) { DISPLAY("press enter...\n"); getchar(); }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-\r
+/*
+ bench.c - Demo program to benchmark open-source compression algorithm
+ Copyright (C) Yann Collet 2012-2013
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+
+//**************************************
+// Compiler Options
+//**************************************
+// Disable some Visual warning messages
+#define _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_DEPRECATE // VS2005
+
+// Unix Large Files support (>4GB)
+#define _FILE_OFFSET_BITS 64
+#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
+# define _LARGEFILE_SOURCE
+#elif ! defined(__LP64__) // No point defining Large file for 64 bit
+# define _LARGEFILE64_SOURCE
+#endif
+
+// S_ISREG & gettimeofday() are not supported by MSVC
+#if defined(_MSC_VER) || defined(_WIN32)
+# define BMK_LEGACY_TIMER 1
+#endif
+
+
+//**************************************
+// Includes
+//**************************************
+#include <stdlib.h> // malloc
+#include <stdio.h> // fprintf, fopen, ftello64
+#include <sys/types.h> // stat64
+#include <sys/stat.h> // stat64
+
+// Use ftime() if gettimeofday() is not available on your target
+#if defined(BMK_LEGACY_TIMER)
+# include <sys/timeb.h> // timeb, ftime
+#else
+# include <sys/time.h> // gettimeofday
+#endif
+
+#include "lz4.h"
+#define COMPRESSOR0 LZ4_compress
+#include "lz4hc.h"
+#define COMPRESSOR1 LZ4_compressHC
+#define DEFAULTCOMPRESSOR COMPRESSOR0
+
+#include "xxhash.h"
+
+
+//**************************************
+// Compiler specifics
+//**************************************
+#if !defined(S_ISREG)
+# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
+#endif
+
+// GCC does not support _rotl outside of Windows
+#if !defined(_WIN32)
+# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
+#endif
+
+
+//**************************************
+// Basic Types
+//**************************************
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+
+//**************************************
+// Constants
+//**************************************
+#define NBLOOPS 3
+#define TIMELOOP 2000
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+#define KNUTH 2654435761U
+#define MAX_MEM (2 GB - 64 MB)
+#define DEFAULT_CHUNKSIZE (4 MB)
+
+
+//**************************************
+// Local structures
+//**************************************
+struct chunkParameters
+{
+ U32 id;
+ char* origBuffer;
+ char* compressedBuffer;
+ int origSize;
+ int compressedSize;
+};
+
+struct compressionParameters
+{
+ int (*compressionFunction)(const char*, char*, int);
+ int (*decompressionFunction)(const char*, char*, int);
+};
+
+
+//**************************************
+// MACRO
+//**************************************
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+
+
+
+//**************************************
+// Benchmark Parameters
+//**************************************
+static int chunkSize = DEFAULT_CHUNKSIZE;
+static int nbIterations = NBLOOPS;
+static int BMK_pause = 0;
+
+void BMK_SetBlocksize(int bsize) { chunkSize = bsize; }
+
+void BMK_SetNbIterations(int nbLoops)
+{
+ nbIterations = nbLoops;
+ DISPLAY("- %i iterations -\n", nbIterations);
+}
+
+void BMK_SetPause() { BMK_pause = 1; }
+
+
+//*********************************************************
+// Private functions
+//*********************************************************
+
+#if defined(BMK_LEGACY_TIMER)
+
+static int BMK_GetMilliStart()
+{
+ // Based on Legacy ftime()
+ // Rolls over every ~ 12.1 days (0x100000/24/60/60)
+ // Use GetMilliSpan to correct for rollover
+ struct timeb tb;
+ int nCount;
+ ftime( &tb );
+ nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
+ return nCount;
+}
+
+#else
+
+static int BMK_GetMilliStart()
+{
+ // Based on newer gettimeofday()
+ // Use GetMilliSpan to correct for rollover
+ struct timeval tv;
+ int nCount;
+ gettimeofday(&tv, NULL);
+ nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
+ return nCount;
+}
+
+#endif
+
+
+static int BMK_GetMilliSpan( int nTimeStart )
+{
+ int nSpan = BMK_GetMilliStart() - nTimeStart;
+ if ( nSpan < 0 )
+ nSpan += 0x100000 * 1000;
+ return nSpan;
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+ size_t step = (64 MB);
+ BYTE* testmem=NULL;
+
+ requiredMem = (((requiredMem >> 26) + 1) << 26);
+ requiredMem += 2*step;
+ if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+
+ while (!testmem)
+ {
+ requiredMem -= step;
+ testmem = (BYTE*) malloc ((size_t)requiredMem);
+ }
+
+ free (testmem);
+ return (size_t) (requiredMem - step);
+}
+
+
+static U64 BMK_GetFileSize(char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
+ return (U64)statbuf.st_size;
+}
+
+
+//*********************************************************
+// Public function
+//*********************************************************
+
+int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel)
+{
+ int fileIdx=0;
+ char* orig_buff;
+ struct compressionParameters compP;
+ int cfunctionId;
+
+ U64 totals = 0;
+ U64 totalz = 0;
+ double totalc = 0.;
+ double totald = 0.;
+
+
+ // Init
+ if (cLevel <3) cfunctionId = 0; else cfunctionId = 1;
+ switch (cfunctionId)
+ {
+#ifdef COMPRESSOR0
+ case 0 : compP.compressionFunction = COMPRESSOR0; break;
+#endif
+#ifdef COMPRESSOR1
+ case 1 : compP.compressionFunction = COMPRESSOR1; break;
+#endif
+ default : compP.compressionFunction = DEFAULTCOMPRESSOR;
+ }
+ compP.decompressionFunction = LZ4_decompress_fast;
+
+ // Loop for each file
+ while (fileIdx<nbFiles)
+ {
+ FILE* inFile;
+ char* inFileName;
+ U64 inFileSize;
+ size_t benchedSize;
+ int nbChunks;
+ int maxCompressedChunkSize;
+ size_t readSize;
+ char* compressedBuffer; int compressedBuffSize;
+ struct chunkParameters* chunkP;
+ U32 crcOrig;
+
+ // Check file existence
+ inFileName = fileNamesTable[fileIdx++];
+ inFile = fopen( inFileName, "rb" );
+ if (inFile==NULL)
+ {
+ DISPLAY( "Pb opening %s\n", inFileName);
+ return 11;
+ }
+
+ // Memory allocation & restrictions
+ inFileSize = BMK_GetFileSize(inFileName);
+ benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2;
+ if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+ if (benchedSize < inFileSize)
+ {
+ DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
+ }
+
+ // Alloc
+ chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));
+ orig_buff = (char*)malloc((size_t )benchedSize);
+ nbChunks = (int) (benchedSize / chunkSize) + 1;
+ maxCompressedChunkSize = LZ4_compressBound(chunkSize);
+ compressedBuffSize = nbChunks * maxCompressedChunkSize;
+ compressedBuffer = (char*)malloc((size_t )compressedBuffSize);
+
+
+ if (!orig_buff || !compressedBuffer)
+ {
+ DISPLAY("\nError: not enough memory!\n");
+ free(orig_buff);
+ free(compressedBuffer);
+ free(chunkP);
+ fclose(inFile);
+ return 12;
+ }
+
+ // Init chunks data
+ {
+ int i;
+ size_t remaining = benchedSize;
+ char* in = orig_buff;
+ char* out = compressedBuffer;
+ for (i=0; i<nbChunks; i++)
+ {
+ chunkP[i].id = i;
+ chunkP[i].origBuffer = in; in += chunkSize;
+ if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+ chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
+ chunkP[i].compressedSize = 0;
+ }
+ }
+
+ // Fill input buffer
+ DISPLAY("Loading %s... \r", inFileName);
+ readSize = fread(orig_buff, 1, benchedSize, inFile);
+ fclose(inFile);
+
+ if (readSize != benchedSize)
+ {
+ DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
+ free(orig_buff);
+ free(compressedBuffer);
+ free(chunkP);
+ return 13;
+ }
+
+ // Calculating input Checksum
+ crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0);
+
+
+ // Bench
+ {
+ int loopNb, chunkNb;
+ size_t cSize=0;
+ double fastestC = 100000000., fastestD = 100000000.;
+ double ratio=0.;
+ U32 crcCheck=0;
+
+ DISPLAY("\r%79s\r", "");
+ for (loopNb = 1; loopNb <= nbIterations; loopNb++)
+ {
+ int nbLoops;
+ int milliTime;
+
+ // Compression
+ DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize);
+ { size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } // warmimg up memory
+
+ nbLoops = 0;
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
+ nbLoops++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops;
+ cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
+ ratio = (double)cSize/(double)benchedSize*100.;
+
+ DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.);
+
+ // Decompression
+ { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing area, for CRC checking
+
+ nbLoops = 0;
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);
+ nbLoops++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops;
+ DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
+
+ // CRC Checking
+ crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0);
+ if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }
+ }
+
+ if (crcOrig==crcCheck)
+ {
+ if (ratio<100.)
+ DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
+ else
+ DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.);
+ }
+ totals += benchedSize;
+ totalz += cSize;
+ totalc += fastestC;
+ totald += fastestD;
+ }
+
+ free(orig_buff);
+ free(compressedBuffer);
+ free(chunkP);
+ }
+
+ if (nbFiles > 1)
+ DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
+
+ if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); }
+
+ return 0;
+}
+
+
+
-/*\r
- bench.h - Demo program to benchmark open-source compression algorithm\r
- Copyright (C) Yann Collet 2012-2013\r
-\r
- This program is free software; you can redistribute it and/or modify\r
- it under the terms of the GNU General Public License as published by\r
- the Free Software Foundation; either version 2 of the License, or\r
- (at your option) any later version.\r
-\r
- This program is distributed in the hope that it will be useful,\r
- but WITHOUT ANY WARRANTY; without even the implied warranty of\r
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
- GNU General Public License for more details.\r
-\r
- You should have received a copy of the GNU General Public License along\r
- with this program; if not, write to the Free Software Foundation, Inc.,\r
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-#pragma once\r
-\r
-#if defined (__cplusplus)\r
-extern "C" {\r
-#endif\r
-\r
-\r
-int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel);\r
-\r
-// Parameters\r
-void BMK_SetBlocksize(int bsize);\r
-void BMK_SetNbIterations(int nbLoops);\r
-void BMK_SetPause();\r
-\r
-\r
-\r
-#if defined (__cplusplus)\r
-}\r
-#endif\r
+/*
+ bench.h - Demo program to benchmark open-source compression algorithm
+ Copyright (C) Yann Collet 2012-2013
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel);
+
+// Parameters
+void BMK_SetBlocksize(int bsize);
+void BMK_SetNbIterations(int nbLoops);
+void BMK_SetPause();
+
+
+
+#if defined (__cplusplus)
+}
+#endif
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library")\r
set(CPACK_PACKAGE_VERSION_MAJOR 0)\r
set(CPACK_PACKAGE_VERSION_MINOR 0)\r
-set(CPACK_PACKAGE_VERSION_PATCH r107)\r
+set(CPACK_PACKAGE_VERSION_PATCH r110)\r
#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL)\r
set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ")\r
include(CPack)\r
\r
-\r
cmake_minimum_required (VERSION 2.6)\r
INCLUDE (CheckTypeSize)\r
check_type_size("void *" SIZEOF_VOID_P)\r
MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P})\r
ENDIF()\r
\r
-set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")\r
-\r
-if (BUILD_SHARED_LIBS)\r
- message(STATUS "Enable shared library building")\r
-else(BUILD_SHARED_LIBS)\r
- message(STATUS "Disable shared library building")\r
-endif(BUILD_SHARED_LIBS)\r
+option(BUILD_TOOLS "Build the command line tools" ON)\r
+option(BUILD_LIBS "Build the libraries in addition to the tools" OFF)\r
\r
-if(UNIX AND BUILD_SHARED_LIBS)\r
+if(UNIX AND BUILD_LIBS)\r
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")\r
add_definitions(-fPIC)\r
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")\r
set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h )\r
set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c)\r
\r
-if(NOT BUILD_SHARED_LIBS)\r
+if(BUILD_TOOLS AND NOT BUILD_LIBS)\r
set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB})\r
endif()\r
\r
-if (CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")\r
- message(STATUS "Build 64bit executable binary")\r
- add_executable(lz4c64 ${LZ4_SRCS})\r
- install(TARGETS lz4c64 RUNTIME DESTINATION "bin/")\r
- if(NOT BUILD_SHARED_LIBS)\r
- message(STATUS "Build 32bit executable binary")\r
- add_executable(lz4c32 ${LZ4_SRCS})\r
- install(TARGETS lz4c32 RUNTIME DESTINATION "bin/")\r
-\r
- SET_TARGET_PROPERTIES(lz4c32 PROPERTIES\r
- COMPILE_FLAGS PROPERTIES COMPILE_FLAGS "-m32" LINK_FLAGS "-m32")\r
- endif()\r
-else()\r
- message(STATUS "Build 32bit executable binary")\r
- add_executable(lz4c32 ${LZ4_SRCS})\r
- install(TARGETS lz4c32 RUNTIME DESTINATION "bin/")\r
+if(BUILD_TOOLS)\r
+ add_executable(lz4 ${LZ4_SRCS})\r
+ set_target_properties(lz4 PROPERTIES COMPILE_DEFINITIONS DISABLE_LZ4C_LEGACY_OPTIONS)\r
+ install(TARGETS lz4 RUNTIME DESTINATION "bin/")\r
+ add_executable(lz4c ${LZ4_SRCS})\r
+ install(TARGETS lz4c RUNTIME DESTINATION "bin/")\r
endif()\r
\r
-if(BUILD_SHARED_LIBS)\r
- set(LZ4_SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}")\r
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit")\r
- target_link_libraries(lz4c64 liblz4)\r
- else()\r
- target_link_libraries(lz4c32 liblz4)\r
- endif()\r
- \r
- # for static library\r
- add_library(liblz4_static STATIC ${LZ4_SRCS_LIB})\r
- set_target_properties(liblz4_static PROPERTIES OUTPUT_NAME lz4)\r
-\r
- install(TARGETS liblz4_static\r
- LIBRARY DESTINATION lib\r
- ARCHIVE DESTINATION lib\r
- )\r
-\r
- # for shared library o\r
+if(BUILD_LIBS) \r
add_library(liblz4 ${LZ4_SRCS_LIB})\r
+ \r
set_target_properties(liblz4 PROPERTIES\r
- OUTPUT_NAME lz4\r
- SOVERSION ${LZ4_SOVERSION})\r
-\r
+ OUTPUT_NAME lz4\r
+ SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}"\r
+ )\r
+ \r
install(TARGETS liblz4\r
- LIBRARY DESTINATION lib\r
- ARCHIVE DESTINATION lib\r
+ LIBRARY DESTINATION lib\r
+ ARCHIVE DESTINATION lib\r
)\r
- \r
+\r
install(FILES\r
- ${SRC_DIR}/lz4.h\r
- ${SRC_DIR}/lz4hc.h\r
- DESTINATION include)\r
- \r
-endif(BUILD_SHARED_LIBS)\r
+ ${SRC_DIR}/lz4.h\r
+ ${SRC_DIR}/lz4hc.h\r
+ DESTINATION include\r
+ )\r
+ \r
+ if(BUILD_TOOLS)\r
+ target_link_libraries(lz4 liblz4)\r
+ target_link_libraries(lz4c liblz4)\r
+ endif()\r
+endif()\r
\r
\r
#warnings\r
ADD_DEFINITIONS("-W")\r
ADD_DEFINITIONS("-Wundef")\r
ADD_DEFINITIONS("-Wcast-align")\r
-ADD_DEFINITIONS("-Wno-implicit-function-declaration")\r
-ADD_DEFINITIONS("-Os -march=native -std=c99")\r
+ADD_DEFINITIONS("-std=c99")\r
+ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"")\r
INCLUDE_DIRECTORIES (${SRC_DIR})\r
\r
\r
-/*\r
- bench.c - Demo program to benchmark open-source compression algorithm\r
- Copyright (C) Yann Collet 2012-2013\r
- GPL v2 License\r
-\r
- This program is free software; you can redistribute it and/or modify\r
- it under the terms of the GNU General Public License as published by\r
- the Free Software Foundation; either version 2 of the License, or\r
- (at your option) any later version.\r
-\r
- This program is distributed in the hope that it will be useful,\r
- but WITHOUT ANY WARRANTY; without even the implied warranty of\r
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
- GNU General Public License for more details.\r
-\r
- You should have received a copy of the GNU General Public License along\r
- with this program; if not, write to the Free Software Foundation, Inc.,\r
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-// Disable some Visual warning messages\r
-#define _CRT_SECURE_NO_WARNINGS\r
-#define _CRT_SECURE_NO_DEPRECATE // VS2005\r
-\r
-// Unix Large Files support (>4GB)\r
-#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions\r
-# define _LARGEFILE_SOURCE \r
-# define _FILE_OFFSET_BITS 64\r
-#elif ! defined(__LP64__) // No point defining Large file for 64 bit\r
-# define _LARGEFILE64_SOURCE\r
-#endif\r
-\r
-// S_ISREG & gettimeofday() are not supported by MSVC\r
-#if defined(_MSC_VER) || defined(_WIN32)\r
-# define BMK_LEGACY_TIMER 1\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Includes\r
-//**************************************\r
-#include <stdlib.h> // malloc\r
-#include <stdio.h> // fprintf, fopen, ftello64\r
-#include <sys/types.h> // stat64\r
-#include <sys/stat.h> // stat64\r
-\r
-// Use ftime() if gettimeofday() is not available on your target\r
-#if defined(BMK_LEGACY_TIMER)\r
-# include <sys/timeb.h> // timeb, ftime\r
-#else\r
-# include <sys/time.h> // gettimeofday\r
-#endif\r
-\r
-#include "lz4.h"\r
-#define COMPRESSOR0 LZ4_compress\r
-#include "lz4hc.h"\r
-#define COMPRESSOR1 LZ4_compressHC\r
-#define DEFAULTCOMPRESSOR COMPRESSOR0\r
-\r
-#include "xxhash.h"\r
-\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-// S_ISREG & gettimeofday() are not supported by MSVC\r
-#if !defined(S_ISREG)\r
-# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)\r
-#endif\r
-\r
-// GCC does not support _rotl outside of Windows\r
-#if !defined(_WIN32)\r
-# define _rotl(x,r) ((x << r) | (x >> (32 - r)))\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Basic Types\r
-//**************************************\r
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99\r
-# include <stdint.h>\r
- typedef uint8_t BYTE;\r
- typedef uint16_t U16;\r
- typedef uint32_t U32;\r
- typedef int32_t S32;\r
- typedef uint64_t U64;\r
-#else\r
- typedef unsigned char BYTE;\r
- typedef unsigned short U16;\r
- typedef unsigned int U32;\r
- typedef signed int S32;\r
- typedef unsigned long long U64;\r
-#endif\r
-\r
-\r
-//****************************\r
-// Constants\r
-//****************************\r
-#define PROGRAM_DESCRIPTION "LZ4 speed analyzer"\r
-#ifndef LZ4_VERSION\r
-# define LZ4_VERSION ""\r
-#endif\r
-#define AUTHOR "Yann Collet"\r
-#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__\r
-\r
-#define NBLOOPS 6\r
-#define TIMELOOP 2500\r
-\r
-#define KNUTH 2654435761U\r
-#define MAX_MEM (1984<<20)\r
-#define DEFAULT_CHUNKSIZE (4<<20)\r
-\r
-#define ALL_COMPRESSORS -1\r
-#define ALL_DECOMPRESSORS -1\r
-\r
-\r
-//**************************************\r
-// Local structures\r
-//**************************************\r
-struct chunkParameters\r
-{\r
- U32 id;\r
- char* origBuffer;\r
- char* compressedBuffer;\r
- int origSize;\r
- int compressedSize;\r
-};\r
-\r
-\r
-//**************************************\r
-// MACRO\r
-//**************************************\r
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)\r
-\r
-\r
-\r
-//**************************************\r
-// Benchmark Parameters\r
-//**************************************\r
-static int chunkSize = DEFAULT_CHUNKSIZE;\r
-static int nbIterations = NBLOOPS;\r
-static int BMK_pause = 0;\r
-static int compressionTest = 1;\r
-static int decompressionTest = 1;\r
-static int compressionAlgo = ALL_COMPRESSORS;\r
-static int decompressionAlgo = ALL_DECOMPRESSORS;\r
-\r
-void BMK_SetBlocksize(int bsize)\r
-{\r
- chunkSize = bsize;\r
- DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);\r
-}\r
-\r
-void BMK_SetNbIterations(int nbLoops)\r
-{\r
- nbIterations = nbLoops;\r
- DISPLAY("- %i iterations -\n", nbIterations);\r
-}\r
-\r
-void BMK_SetPause()\r
-{\r
- BMK_pause = 1;\r
-}\r
-\r
-//*********************************************************\r
-// Private functions\r
-//*********************************************************\r
-\r
-#if defined(BMK_LEGACY_TIMER)\r
-\r
-static int BMK_GetMilliStart()\r
-{\r
- // Based on Legacy ftime()\r
- // Rolls over every ~ 12.1 days (0x100000/24/60/60)\r
- // Use GetMilliSpan to correct for rollover\r
- struct timeb tb;\r
- int nCount;\r
- ftime( &tb );\r
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);\r
- return nCount;\r
-}\r
-\r
-#else\r
-\r
-static int BMK_GetMilliStart()\r
-{\r
- // Based on newer gettimeofday()\r
- // Use GetMilliSpan to correct for rollover\r
- struct timeval tv;\r
- int nCount;\r
- gettimeofday(&tv, NULL);\r
- nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);\r
- return nCount;\r
-}\r
-\r
-#endif\r
-\r
-\r
-static int BMK_GetMilliSpan( int nTimeStart )\r
-{\r
- int nSpan = BMK_GetMilliStart() - nTimeStart;\r
- if ( nSpan < 0 )\r
- nSpan += 0x100000 * 1000;\r
- return nSpan;\r
-}\r
-\r
-\r
-static size_t BMK_findMaxMem(U64 requiredMem)\r
-{\r
- size_t step = (64U<<20); // 64 MB\r
- BYTE* testmem=NULL;\r
-\r
- requiredMem = (((requiredMem >> 25) + 1) << 26);\r
- if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;\r
-\r
- requiredMem += 2*step;\r
- while (!testmem)\r
- {\r
- requiredMem -= step;\r
- testmem = (BYTE*) malloc ((size_t)requiredMem);\r
- }\r
-\r
- free (testmem);\r
- return (size_t) (requiredMem - step);\r
-}\r
-\r
-\r
-static U64 BMK_GetFileSize(char* infilename)\r
-{\r
- int r;\r
-#if defined(_MSC_VER)\r
- struct _stat64 statbuf;\r
- r = _stat64(infilename, &statbuf);\r
-#else\r
- struct stat statbuf;\r
- r = stat(infilename, &statbuf);\r
-#endif\r
- if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...\r
- return (U64)statbuf.st_size;\r
-}\r
-\r
-\r
-//*********************************************************\r
-// Public function\r
-//*********************************************************\r
-\r
-static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)\r
-{\r
- return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));\r
-}\r
-\r
-static void* ctx;\r
-static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize)\r
-{\r
- return LZ4_compress_continue(ctx, in, out, inSize);\r
-}\r
-\r
-static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)\r
-{\r
- return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));\r
-}\r
-\r
-static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)\r
-{\r
- return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));\r
-}\r
-\r
-static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)\r
-{\r
- return LZ4_compressHC_continue(ctx, in, out, inSize);\r
-}\r
-\r
-static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)\r
-{\r
- return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));\r
-}\r
-\r
-static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)\r
-{\r
- (void)inSize;\r
- LZ4_decompress_fast(in, out, outSize);\r
- return outSize;\r
-}\r
-\r
-static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)\r
-{\r
- (void)inSize;\r
- LZ4_decompress_fast_withPrefix64k(in, out, outSize);\r
- return outSize;\r
-}\r
-\r
-static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)\r
-{\r
- return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);\r
-}\r
-\r
-int fullSpeedBench(char** fileNamesTable, int nbFiles)\r
-{\r
- int fileIdx=0;\r
- char* orig_buff;\r
-# define NB_COMPRESSION_ALGORITHMS 8\r
-# define MINCOMPRESSIONCHAR '0'\r
-# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)\r
- static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue", "LZ4_compressHC", "LZ4_compressHC_limitedOutput", "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };\r
- double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};\r
- double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};\r
-# define NB_DECOMPRESSION_ALGORITHMS 5\r
-# define MINDECOMPRESSIONCHAR '0'\r
-# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)\r
- static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" };\r
- double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};\r
-\r
- U64 totals = 0;\r
-\r
-\r
- // Loop for each file\r
- while (fileIdx<nbFiles)\r
- {\r
- FILE* inFile;\r
- char* inFileName;\r
- U64 inFileSize;\r
- size_t benchedSize;\r
- int nbChunks;\r
- int maxCompressedChunkSize;\r
- struct chunkParameters* chunkP;\r
- size_t readSize;\r
- char* compressed_buff; int compressedBuffSize;\r
- U32 crcOriginal;\r
-\r
- // Check file existence\r
- inFileName = fileNamesTable[fileIdx++];\r
- inFile = fopen( inFileName, "rb" );\r
- if (inFile==NULL)\r
- {\r
- DISPLAY( "Pb opening %s\n", inFileName);\r
- return 11;\r
- }\r
-\r
- // Memory allocation & restrictions\r
- inFileSize = BMK_GetFileSize(inFileName);\r
- benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;\r
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;\r
- if (benchedSize < inFileSize)\r
- {\r
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));\r
- }\r
-\r
- // Alloc\r
- chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));\r
- orig_buff = (char*) malloc((size_t)benchedSize);\r
- nbChunks = (int) (benchedSize / chunkSize) + 1;\r
- maxCompressedChunkSize = LZ4_compressBound(chunkSize);\r
- compressedBuffSize = nbChunks * maxCompressedChunkSize;\r
- compressed_buff = (char*)malloc((size_t)compressedBuffSize);\r
-\r
-\r
- if(!orig_buff || !compressed_buff)\r
- {\r
- DISPLAY("\nError: not enough memory!\n");\r
- free(orig_buff);\r
- free(compressed_buff);\r
- free(chunkP);\r
- fclose(inFile);\r
- return 12;\r
- }\r
-\r
- // Init chunks data\r
- {\r
- int i;\r
- size_t remaining = benchedSize;\r
- char* in = orig_buff;\r
- char* out = compressed_buff;\r
- for (i=0; i<nbChunks; i++)\r
- {\r
- chunkP[i].id = i;\r
- chunkP[i].origBuffer = in; in += chunkSize;\r
- if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }\r
- chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;\r
- chunkP[i].compressedSize = 0;\r
- }\r
- }\r
-\r
- // Fill input buffer\r
- DISPLAY("Loading %s... \r", inFileName);\r
- readSize = fread(orig_buff, 1, benchedSize, inFile);\r
- fclose(inFile);\r
-\r
- if(readSize != benchedSize)\r
- {\r
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);\r
- free(orig_buff);\r
- free(compressed_buff);\r
- free(chunkP);\r
- return 13;\r
- }\r
-\r
- // Calculating input Checksum\r
- crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0);\r
-\r
-\r
- // Bench\r
- {\r
- int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;\r
- size_t cSize=0;\r
- double ratio=0.;\r
-\r
- DISPLAY("\r%79s\r", "");\r
- DISPLAY(" %s : \n", inFileName);\r
-\r
- // Compression Algorithms\r
- for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)\r
- {\r
- char* cName = compressionNames[cAlgNb];\r
- int (*compressionFunction)(const char*, char*, int);\r
- void* (*initFunction)(const char*) = NULL;\r
- double bestTime = 100000000.;\r
-\r
- if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;\r
-\r
- switch(cAlgNb)\r
- {\r
- case 0: compressionFunction = LZ4_compress; break;\r
- case 1: compressionFunction = local_LZ4_compress_limitedOutput; break;\r
- case 2: compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;\r
- case 3: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;\r
- case 4: compressionFunction = LZ4_compressHC; break;\r
- case 5: compressionFunction = local_LZ4_compressHC_limitedOutput; break;\r
- case 6: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;\r
- case 7: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;\r
- default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;\r
- }\r
-\r
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)\r
- {\r
- double averageTime;\r
- int milliTime;\r
-\r
- DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);\r
- { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory\r
-\r
- nb_loops = 0;\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliStart() == milliTime);\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)\r
- {\r
- if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer);\r
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)\r
- {\r
- chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);\r
- if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);\r
- }\r
- if (initFunction!=NULL) free(ctx);\r
- nb_loops++;\r
- }\r
- milliTime = BMK_GetMilliSpan(milliTime);\r
-\r
- averageTime = (double)milliTime / nb_loops;\r
- if (averageTime < bestTime) bestTime = averageTime;\r
- cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;\r
- ratio = (double)cSize/(double)benchedSize*100.;\r
- DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);\r
- }\r
-\r
- if (ratio<100.)\r
- DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);\r
- else\r
- DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);\r
-\r
- totalCTime[cAlgNb] += bestTime;\r
- totalCSize[cAlgNb] += cSize;\r
- }\r
-\r
- // Prepare layout for decompression\r
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)\r
- {\r
- chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);\r
- if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);\r
- }\r
- { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking\r
-\r
- // Decompression Algorithms\r
- for (dAlgNb=0; (dAlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)\r
- {\r
- char* dName = decompressionNames[dAlgNb];\r
- int (*decompressionFunction)(const char*, char*, int, int);\r
- double bestTime = 100000000.;\r
-\r
- if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;\r
-\r
- switch(dAlgNb)\r
- {\r
- case 0: decompressionFunction = local_LZ4_decompress_fast; break;\r
- case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break;\r
- case 2: decompressionFunction = LZ4_decompress_safe; break;\r
- case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;\r
- case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break;\r
- default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;\r
- }\r
-\r
- for (loopNb = 1; loopNb <= nbIterations; loopNb++)\r
- {\r
- double averageTime;\r
- int milliTime;\r
- U32 crcDecoded;\r
-\r
- DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);\r
-\r
- nb_loops = 0;\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliStart() == milliTime);\r
- milliTime = BMK_GetMilliStart();\r
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)\r
- {\r
- for (chunkNb=0; chunkNb<nbChunks; chunkNb++)\r
- {\r
- int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);\r
- if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1);\r
- }\r
- nb_loops++;\r
- }\r
- milliTime = BMK_GetMilliSpan(milliTime);\r
-\r
- averageTime = (double)milliTime / nb_loops;\r
- if (averageTime < bestTime) bestTime = averageTime;\r
-\r
- DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);\r
-\r
- // CRC Checking\r
- crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);\r
- if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }\r
- }\r
-\r
- DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);\r
-\r
- totalDTime[dAlgNb] += bestTime;\r
- }\r
-\r
- totals += benchedSize;\r
- }\r
-\r
- free(orig_buff);\r
- free(compressed_buff);\r
- free(chunkP);\r
- }\r
-\r
- if (nbFiles > 1)\r
- {\r
- int AlgNb;\r
-\r
- DISPLAY(" ** TOTAL ** : \n");\r
- for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)\r
- {\r
- char* cName = compressionNames[AlgNb];\r
- if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;\r
- DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);\r
- }\r
- for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)\r
- {\r
- char* dName = decompressionNames[AlgNb];\r
- if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;\r
- DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);\r
- }\r
- }\r
-\r
- if (BMK_pause) { printf("press enter...\n"); getchar(); }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-int usage(char* exename)\r
-{\r
- DISPLAY( "Usage :\n");\r
- DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);\r
- DISPLAY( "Arguments :\n");\r
- DISPLAY( " -c : compression tests only\n");\r
- DISPLAY( " -d : decompression tests only\n");\r
- DISPLAY( " -H/-h : Help (this text + advanced options)\n");\r
- return 0;\r
-}\r
-\r
-int usage_advanced()\r
-{\r
- DISPLAY( "\nAdvanced options :\n");\r
- DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR);\r
- DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);\r
- DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);\r
- DISPLAY( " -B# : Block size [4-7](default : 7)\n");\r
- //DISPLAY( " -BD : Block dependency (improve compression ratio)\n");\r
- return 0;\r
-}\r
-\r
-int badusage(char* exename)\r
-{\r
- DISPLAY("Wrong parameters\n");\r
- usage(exename);\r
- return 0;\r
-}\r
-\r
-int main(int argc, char** argv)\r
-{\r
- int i,\r
- filenamesStart=2;\r
- char* exename=argv[0];\r
- char* input_filename=0;\r
-\r
- // Welcome message\r
- DISPLAY( WELCOME_MESSAGE);\r
-\r
- if (argc<2) { badusage(exename); return 1; }\r
-\r
- for(i=1; i<argc; i++)\r
- {\r
- char* argument = argv[i];\r
-\r
- if(!argument) continue; // Protection if argument empty\r
-\r
- // Decode command (note : aggregated commands are allowed)\r
- if (argument[0]=='-')\r
- {\r
- while (argument[1]!=0)\r
- {\r
- argument ++;\r
-\r
- switch(argument[0])\r
- {\r
- // Select compression algorithm only\r
- case 'c': \r
- decompressionTest = 0; \r
- if ((argument[1]>= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR))\r
- compressionAlgo = argument[1] - '0', argument++;\r
- break;\r
-\r
- // Select decompression algorithm only\r
- case 'd': \r
- compressionTest = 0;\r
- if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR))\r
- decompressionAlgo = argument[1] - '0', argument++;\r
- break;\r
-\r
- // Display help on usage\r
- case 'h' :\r
- case 'H': usage(exename); usage_advanced(); return 0;\r
-\r
- // Modify Block Properties\r
- case 'B':\r
- while (argument[1]!=0)\r
- switch(argument[1])\r
- {\r
- case '4':\r
- case '5':\r
- case '6':\r
- case '7':\r
- { \r
- int B = argument[1] - '0'; \r
- int S = 1 << (8 + 2*B); \r
- BMK_SetBlocksize(S); \r
- argument++;\r
- break;\r
- }\r
- case 'D': argument++; break;\r
- default : goto _exit_blockProperties;\r
- }\r
-_exit_blockProperties:\r
- break;\r
-\r
- // Modify Nb Iterations\r
- case 'i': \r
- if ((argument[1] >='1') && (argument[1] <='9'))\r
- {\r
- int iters = argument[1] - '0'; \r
- BMK_SetNbIterations(iters); \r
- argument++;\r
- }\r
- break;\r
-\r
- // Pause at the end (hidden option)\r
- case 'p': BMK_SetPause(); break;\r
-\r
- // Unrecognised command\r
- default : badusage(exename); return 1;\r
- }\r
- }\r
- continue;\r
- }\r
-\r
- // first provided filename is input\r
- if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }\r
-\r
- }\r
-\r
- // No input filename ==> Error\r
- if(!input_filename) { badusage(exename); return 1; }\r
-\r
- return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);\r
-\r
-}\r
-\r
+/*
+ bench.c - Demo program to benchmark open-source compression algorithm
+ Copyright (C) Yann Collet 2012-2013
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+
+//**************************************
+// Compiler Options
+//**************************************
+// Disable some Visual warning messages
+#define _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_DEPRECATE // VS2005
+
+// Unix Large Files support (>4GB)
+#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions
+# define _LARGEFILE_SOURCE
+# define _FILE_OFFSET_BITS 64
+#elif ! defined(__LP64__) // No point defining Large file for 64 bit
+# define _LARGEFILE64_SOURCE
+#endif
+
+// S_ISREG & gettimeofday() are not supported by MSVC
+#if defined(_MSC_VER) || defined(_WIN32)
+# define BMK_LEGACY_TIMER 1
+#endif
+
+
+//**************************************
+// Includes
+//**************************************
+#include <stdlib.h> // malloc
+#include <stdio.h> // fprintf, fopen, ftello64
+#include <sys/types.h> // stat64
+#include <sys/stat.h> // stat64
+
+// Use ftime() if gettimeofday() is not available on your target
+#if defined(BMK_LEGACY_TIMER)
+# include <sys/timeb.h> // timeb, ftime
+#else
+# include <sys/time.h> // gettimeofday
+#endif
+
+#include "lz4.h"
+#define COMPRESSOR0 LZ4_compress
+#include "lz4hc.h"
+#define COMPRESSOR1 LZ4_compressHC
+#define DEFAULTCOMPRESSOR COMPRESSOR0
+
+#include "xxhash.h"
+
+
+//**************************************
+// Compiler Options
+//**************************************
+// S_ISREG & gettimeofday() are not supported by MSVC
+#if !defined(S_ISREG)
+# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
+#endif
+
+// GCC does not support _rotl outside of Windows
+#if !defined(_WIN32)
+# define _rotl(x,r) ((x << r) | (x >> (32 - r)))
+#endif
+
+
+//**************************************
+// Basic Types
+//**************************************
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+
+//****************************
+// Constants
+//****************************
+#define PROGRAM_DESCRIPTION "LZ4 speed analyzer"
+#ifndef LZ4_VERSION
+# define LZ4_VERSION ""
+#endif
+#define AUTHOR "Yann Collet"
+#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__
+
+#define NBLOOPS 6
+#define TIMELOOP 2500
+
+#define KNUTH 2654435761U
+#define MAX_MEM (1984<<20)
+#define DEFAULT_CHUNKSIZE (4<<20)
+
+#define ALL_COMPRESSORS -1
+#define ALL_DECOMPRESSORS -1
+
+
+//**************************************
+// Local structures
+//**************************************
+struct chunkParameters
+{
+ U32 id;
+ char* origBuffer;
+ char* compressedBuffer;
+ int origSize;
+ int compressedSize;
+};
+
+
+//**************************************
+// MACRO
+//**************************************
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+
+
+
+//**************************************
+// Benchmark Parameters
+//**************************************
+static int chunkSize = DEFAULT_CHUNKSIZE;
+static int nbIterations = NBLOOPS;
+static int BMK_pause = 0;
+static int compressionTest = 1;
+static int decompressionTest = 1;
+static int compressionAlgo = ALL_COMPRESSORS;
+static int decompressionAlgo = ALL_DECOMPRESSORS;
+
+void BMK_SetBlocksize(int bsize)
+{
+ chunkSize = bsize;
+ DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10);
+}
+
+void BMK_SetNbIterations(int nbLoops)
+{
+ nbIterations = nbLoops;
+ DISPLAY("- %i iterations -\n", nbIterations);
+}
+
+void BMK_SetPause()
+{
+ BMK_pause = 1;
+}
+
+//*********************************************************
+// Private functions
+//*********************************************************
+
+#if defined(BMK_LEGACY_TIMER)
+
+static int BMK_GetMilliStart()
+{
+ // Based on Legacy ftime()
+ // Rolls over every ~ 12.1 days (0x100000/24/60/60)
+ // Use GetMilliSpan to correct for rollover
+ struct timeb tb;
+ int nCount;
+ ftime( &tb );
+ nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
+ return nCount;
+}
+
+#else
+
+static int BMK_GetMilliStart()
+{
+ // Based on newer gettimeofday()
+ // Use GetMilliSpan to correct for rollover
+ struct timeval tv;
+ int nCount;
+ gettimeofday(&tv, NULL);
+ nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
+ return nCount;
+}
+
+#endif
+
+
+static int BMK_GetMilliSpan( int nTimeStart )
+{
+ int nSpan = BMK_GetMilliStart() - nTimeStart;
+ if ( nSpan < 0 )
+ nSpan += 0x100000 * 1000;
+ return nSpan;
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+ size_t step = (64U<<20); // 64 MB
+ BYTE* testmem=NULL;
+
+ requiredMem = (((requiredMem >> 25) + 1) << 26);
+ if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+
+ requiredMem += 2*step;
+ while (!testmem)
+ {
+ requiredMem -= step;
+ testmem = (BYTE*) malloc ((size_t)requiredMem);
+ }
+
+ free (testmem);
+ return (size_t) (requiredMem - step);
+}
+
+
+static U64 BMK_GetFileSize(char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
+ return (U64)statbuf.st_size;
+}
+
+
+//*********************************************************
+// Public function
+//*********************************************************
+
+static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
+}
+
+static void* stateLZ4;
+static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_withState(stateLZ4, in, out, inSize);
+}
+
+static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize));
+}
+
+static void* ctx;
+static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_continue(ctx, in, out, inSize);
+}
+
+static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
+}
+
+static void* stateLZ4HC;
+static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_withState(stateLZ4HC, in, out, inSize);
+}
+
+static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize)
+{
+ return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize));
+}
+
+static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize));
+}
+
+static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_continue(ctx, in, out, inSize);
+}
+
+static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize)
+{
+ return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize));
+}
+
+static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
+{
+ (void)inSize;
+ LZ4_decompress_fast(in, out, outSize);
+ return outSize;
+}
+
+static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize)
+{
+ (void)inSize;
+ LZ4_decompress_fast_withPrefix64k(in, out, outSize);
+ return outSize;
+}
+
+static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
+{
+ return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
+}
+
+int fullSpeedBench(char** fileNamesTable, int nbFiles)
+{
+ int fileIdx=0;
+ char* orig_buff;
+# define NB_COMPRESSION_ALGORITHMS 12
+# define MINCOMPRESSIONCHAR '0'
+# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS)
+ static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput",
+ "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState",
+ "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue",
+ "LZ4_compressHC", "LZ4_compressHC_limitedOutput",
+ "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC",
+ "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" };
+ double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0};
+ double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0};
+# define NB_DECOMPRESSION_ALGORITHMS 5
+# define MINDECOMPRESSIONCHAR '0'
+# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS)
+ static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" };
+ double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0};
+
+ U64 totals = 0;
+
+
+ // Loop for each file
+ while (fileIdx<nbFiles)
+ {
+ FILE* inFile;
+ char* inFileName;
+ U64 inFileSize;
+ size_t benchedSize;
+ int nbChunks;
+ int maxCompressedChunkSize;
+ struct chunkParameters* chunkP;
+ size_t readSize;
+ char* compressed_buff; int compressedBuffSize;
+ U32 crcOriginal;
+
+
+ // Init
+ stateLZ4 = malloc(LZ4_sizeofState());
+ stateLZ4HC = malloc(LZ4_sizeofStateHC());
+
+ // Check file existence
+ inFileName = fileNamesTable[fileIdx++];
+ inFile = fopen( inFileName, "rb" );
+ if (inFile==NULL)
+ {
+ DISPLAY( "Pb opening %s\n", inFileName);
+ return 11;
+ }
+
+ // Memory allocation & restrictions
+ inFileSize = BMK_GetFileSize(inFileName);
+ benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2;
+ if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+ if (benchedSize < inFileSize)
+ {
+ DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
+ }
+
+ // Alloc
+ chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters));
+ orig_buff = (char*) malloc((size_t)benchedSize);
+ nbChunks = (int) (benchedSize / chunkSize) + 1;
+ maxCompressedChunkSize = LZ4_compressBound(chunkSize);
+ compressedBuffSize = nbChunks * maxCompressedChunkSize;
+ compressed_buff = (char*)malloc((size_t)compressedBuffSize);
+
+
+ if(!orig_buff || !compressed_buff)
+ {
+ DISPLAY("\nError: not enough memory!\n");
+ free(orig_buff);
+ free(compressed_buff);
+ free(chunkP);
+ fclose(inFile);
+ return 12;
+ }
+
+ // Init chunks data
+ {
+ int i;
+ size_t remaining = benchedSize;
+ char* in = orig_buff;
+ char* out = compressed_buff;
+ for (i=0; i<nbChunks; i++)
+ {
+ chunkP[i].id = i;
+ chunkP[i].origBuffer = in; in += chunkSize;
+ if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
+ chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
+ chunkP[i].compressedSize = 0;
+ }
+ }
+
+ // Fill input buffer
+ DISPLAY("Loading %s... \r", inFileName);
+ readSize = fread(orig_buff, 1, benchedSize, inFile);
+ fclose(inFile);
+
+ if(readSize != benchedSize)
+ {
+ DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
+ free(orig_buff);
+ free(compressed_buff);
+ free(chunkP);
+ return 13;
+ }
+
+ // Calculating input Checksum
+ crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0);
+
+
+ // Bench
+ {
+ int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
+ size_t cSize=0;
+ double ratio=0.;
+
+ DISPLAY("\r%79s\r", "");
+ DISPLAY(" %s : \n", inFileName);
+
+ // Compression Algorithms
+ for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++)
+ {
+ char* cName = compressionNames[cAlgNb];
+ int (*compressionFunction)(const char*, char*, int);
+ void* (*initFunction)(const char*) = NULL;
+ double bestTime = 100000000.;
+
+ if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue;
+
+ switch(cAlgNb)
+ {
+ case 0 : compressionFunction = LZ4_compress; break;
+ case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break;
+ case 2 : compressionFunction = local_LZ4_compress_withState; break;
+ case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break;
+ case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break;
+ case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break;
+ case 6 : compressionFunction = LZ4_compressHC; break;
+ case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break;
+ case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break;
+ case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break;
+ case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break;
+ case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break;
+ default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
+ }
+
+ for (loopNb = 1; loopNb <= nbIterations; loopNb++)
+ {
+ double averageTime;
+ int milliTime;
+
+ DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize);
+ { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warmimg up memory
+
+ nb_loops = 0;
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ if (initFunction!=NULL) ctx = initFunction(chunkP[0].origBuffer);
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ {
+ chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
+ if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", cName), exit(1);
+ }
+ if (initFunction!=NULL) free(ctx);
+ nb_loops++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ averageTime = (double)milliTime / nb_loops;
+ if (averageTime < bestTime) bestTime = averageTime;
+ cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
+ ratio = (double)cSize/(double)benchedSize*100.;
+ DISPLAY("%1i-%-19.19s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+ }
+
+ if (ratio<100.)
+ DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+ else
+ DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.);
+
+ totalCTime[cAlgNb] += bestTime;
+ totalCSize[cAlgNb] += cSize;
+ }
+
+ // Prepare layout for decompression
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ {
+ chunkP[chunkNb].compressedSize = LZ4_compress(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
+ if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressionNames[0]), exit(1);
+ }
+ { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking
+
+ // Decompression Algorithms
+ for (dAlgNb=0; (dAlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++)
+ {
+ char* dName = decompressionNames[dAlgNb];
+ int (*decompressionFunction)(const char*, char*, int, int);
+ double bestTime = 100000000.;
+
+ if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue;
+
+ switch(dAlgNb)
+ {
+ case 0: decompressionFunction = local_LZ4_decompress_fast; break;
+ case 1: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; break;
+ case 2: decompressionFunction = LZ4_decompress_safe; break;
+ case 3: decompressionFunction = LZ4_decompress_safe_withPrefix64k; break;
+ case 4: decompressionFunction = local_LZ4_decompress_safe_partial; break;
+ default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1;
+ }
+
+ for (loopNb = 1; loopNb <= nbIterations; loopNb++)
+ {
+ double averageTime;
+ int milliTime;
+ U32 crcDecoded;
+
+ DISPLAY("%1i-%-24.24s :%10i ->\r", loopNb, dName, (int)benchedSize);
+
+ nb_loops = 0;
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ {
+ int decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
+ if (chunkP[chunkNb].origSize != decodedSize) DISPLAY("ERROR ! %s() == %i != %i !! \n", dName, decodedSize, chunkP[chunkNb].origSize), exit(1);
+ }
+ nb_loops++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ averageTime = (double)milliTime / nb_loops;
+ if (averageTime < bestTime) bestTime = averageTime;
+
+ DISPLAY("%1i-%-24.24s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
+
+ // CRC Checking
+ crcDecoded = XXH32(orig_buff, (int)benchedSize, 0);
+ if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); }
+ }
+
+ DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.);
+
+ totalDTime[dAlgNb] += bestTime;
+ }
+
+ totals += benchedSize;
+ }
+
+ free(orig_buff);
+ free(compressed_buff);
+ free(chunkP);
+ }
+
+ if (nbFiles > 1)
+ {
+ int AlgNb;
+
+ DISPLAY(" ** TOTAL ** : \n");
+ for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++)
+ {
+ char* cName = compressionNames[AlgNb];
+ if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue;
+ DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.);
+ }
+ for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++)
+ {
+ char* dName = decompressionNames[AlgNb];
+ if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue;
+ DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.);
+ }
+ }
+
+ if (BMK_pause) { printf("press enter...\n"); getchar(); }
+
+ return 0;
+}
+
+
+int usage(char* exename)
+{
+ DISPLAY( "Usage :\n");
+ DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
+ DISPLAY( "Arguments :\n");
+ DISPLAY( " -c : compression tests only\n");
+ DISPLAY( " -d : decompression tests only\n");
+ DISPLAY( " -H/-h : Help (this text + advanced options)\n");
+ return 0;
+}
+
+int usage_advanced()
+{
+ DISPLAY( "\nAdvanced options :\n");
+ DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR);
+ DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR);
+ DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
+ DISPLAY( " -B# : Block size [4-7](default : 7)\n");
+ //DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
+ return 0;
+}
+
+int badusage(char* exename)
+{
+ DISPLAY("Wrong parameters\n");
+ usage(exename);
+ return 0;
+}
+
+int main(int argc, char** argv)
+{
+ int i,
+ filenamesStart=2;
+ char* exename=argv[0];
+ char* input_filename=0;
+
+ // Welcome message
+ DISPLAY( WELCOME_MESSAGE);
+
+ if (argc<2) { badusage(exename); return 1; }
+
+ for(i=1; i<argc; i++)
+ {
+ char* argument = argv[i];
+
+ if(!argument) continue; // Protection if argument empty
+
+ // Decode command (note : aggregated commands are allowed)
+ if (argument[0]=='-')
+ {
+ while (argument[1]!=0)
+ {
+ argument ++;
+
+ switch(argument[0])
+ {
+ // Select compression algorithm only
+ case 'c':
+ decompressionTest = 0;
+ if ((argument[1]>= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR))
+ compressionAlgo = argument[1] - '0', argument++;
+ break;
+
+ // Select decompression algorithm only
+ case 'd':
+ compressionTest = 0;
+ if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR))
+ decompressionAlgo = argument[1] - '0', argument++;
+ break;
+
+ // Display help on usage
+ case 'h' :
+ case 'H': usage(exename); usage_advanced(); return 0;
+
+ // Modify Block Properties
+ case 'B':
+ while (argument[1]!=0)
+ switch(argument[1])
+ {
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ {
+ int B = argument[1] - '0';
+ int S = 1 << (8 + 2*B);
+ BMK_SetBlocksize(S);
+ argument++;
+ break;
+ }
+ case 'D': argument++; break;
+ default : goto _exit_blockProperties;
+ }
+_exit_blockProperties:
+ break;
+
+ // Modify Nb Iterations
+ case 'i':
+ if ((argument[1] >='1') && (argument[1] <='9'))
+ {
+ int iters = argument[1] - '0';
+ BMK_SetNbIterations(iters);
+ argument++;
+ }
+ break;
+
+ // Pause at the end (hidden option)
+ case 'p': BMK_SetPause(); break;
+
+ // Unrecognised command
+ default : badusage(exename); return 1;
+ }
+ }
+ continue;
+ }
+
+ // first provided filename is input
+ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
+
+ }
+
+ // No input filename ==> Error
+ if(!input_filename) { badusage(exename); return 1; }
+
+ return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);
+
+}
+
-/*\r
- fuzzer.c - Fuzzer test tool for LZ4\r
- Copyright (C) Yann Collet - Andrew Mahone 2012-2013\r
- Code started by Andrew Mahone, modified by Yann Collet\r
- GPL v2 License\r
-\r
- This program is free software; you can redistribute it and/or modify\r
- it under the terms of the GNU General Public License as published by\r
- the Free Software Foundation; either version 2 of the License, or\r
- (at your option) any later version.\r
-\r
- This program is distributed in the hope that it will be useful,\r
- but WITHOUT ANY WARRANTY; without even the implied warranty of\r
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
- GNU General Public License for more details.\r
-\r
- You should have received a copy of the GNU General Public License along\r
- with this program; if not, write to the Free Software Foundation, Inc.,\r
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-\r
-//**************************************\r
-// Remove Visual warning messages\r
-//**************************************\r
-#define _CRT_SECURE_NO_WARNINGS // fgets\r
-\r
-\r
-//**************************************\r
-// Includes\r
-//**************************************\r
-#include <stdlib.h>\r
-#include <stdio.h> // fgets, sscanf\r
-#include <sys/timeb.h> // timeb\r
-#include "lz4.h"\r
-#include "lz4hc.h"\r
-\r
-\r
-//**************************************\r
-// Constants\r
-//**************************************\r
-#ifndef LZ4_VERSION\r
-# define LZ4_VERSION ""\r
-#endif\r
-\r
-#define NB_ATTEMPTS (1<<17)\r
-#define LEN ((1<<15))\r
-#define SEQ_POW 2\r
-#define NUM_SEQ (1 << SEQ_POW)\r
-#define SEQ_MSK ((NUM_SEQ) - 1)\r
-#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0)\r
-#define NEW_SEQ(x) ((((x) >> 10) %10) == 0)\r
-#define PAGE_SIZE 4096\r
-#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))\r
-#define PRIME1 2654435761U\r
-#define PRIME2 2246822519U\r
-#define PRIME3 3266489917U\r
-\r
-\r
-//*********************************************************\r
-// Functions\r
-//*********************************************************\r
-static int FUZ_GetMilliStart()\r
-{\r
- struct timeb tb;\r
- int nCount;\r
- ftime( &tb );\r
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);\r
- return nCount;\r
-}\r
-\r
-\r
-static int FUZ_GetMilliSpan( int nTimeStart )\r
-{\r
- int nSpan = FUZ_GetMilliStart() - nTimeStart;\r
- if ( nSpan < 0 )\r
- nSpan += 0x100000 * 1000;\r
- return nSpan;\r
-}\r
-\r
-\r
-unsigned int FUZ_rand(unsigned int* src)\r
-{\r
- *src = ((*src) * PRIME1) + PRIME2;\r
- return *src;\r
-}\r
-\r
-\r
-int test_canary(unsigned char *buf)\r
-{\r
- int i;\r
- for (i = 0; i < 2048; i++)\r
- if (buf[i] != buf[i + 2048])\r
- return 0;\r
- return 1;\r
-}\r
-\r
-\r
-int FUZ_SecurityTest()\r
-{\r
- char* output;\r
- char* input;\r
- int i, r;\r
-\r
- printf("Overflow test (issue 52)...\n");\r
- input = (char*) malloc (20<<20);\r
- output = (char*) malloc (20<<20);\r
- input[0] = 0x0F;\r
- input[1] = 0x00;\r
- input[2] = 0x00;\r
- for(i = 3; i < 16840000; i++)\r
- input[i] = 0xff;\r
- r = LZ4_decompress_fast(input, output, 20<<20);\r
-\r
- free(input);\r
- free(output);\r
- printf(" Passed (return = %i < 0)\n",r);\r
- return 0;\r
-}\r
-\r
-\r
-//int main(int argc, char *argv[]) {\r
-int main() {\r
- unsigned long long bytes = 0;\r
- unsigned long long cbytes = 0;\r
- unsigned long long hcbytes = 0;\r
- unsigned char buf[LEN];\r
- unsigned char testOut[LEN+1];\r
-# define FUZ_max LZ4_COMPRESSBOUND(LEN)\r
-# define FUZ_avail ROUND_PAGE(FUZ_max)\r
- const int off_full = FUZ_avail - FUZ_max;\r
- unsigned char cbuf[FUZ_avail + PAGE_SIZE];\r
- unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();\r
- int i, j, k, ret, len, lenHC, attemptNb;\r
- char userInput[30] = {0};\r
-# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }\r
-# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb);\r
-\r
- printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);\r
- printf("Select an Initialisation number (default : random) : ");\r
- fflush(stdout);\r
- if ( fgets(userInput, sizeof userInput, stdin) )\r
- {\r
- if ( sscanf(userInput, "%d", &seed) == 1 ) {}\r
- else seed = FUZ_GetMilliSpan(timestamp);\r
- }\r
- printf("Seed = %u\n", seed);\r
- randState = seed;\r
-\r
- //FUZ_SecurityTest();\r
-\r
- for (i = 0; i < 2048; i++)\r
- cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16;\r
-\r
- for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++) \r
- {\r
- int testNb = 0;\r
-\r
- printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);\r
- \r
- for (j = 0; j < NUM_SEQ; j++) {\r
- seeds[j] = FUZ_rand(&randState) << 8;\r
- seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535;\r
- }\r
- for (j = 0; j < LEN; j++) {\r
- k = FUZ_rand(&randState);\r
- if (j == 0 || NEW_SEQ(k))\r
- cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK];\r
- if (MOD_SEQ(k)) {\r
- k = (FUZ_rand(&randState) >> 16) & SEQ_MSK;\r
- seeds[k] = FUZ_rand(&randState) << 8;\r
- seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535;\r
- }\r
- buf[j] = FUZ_rand(&cur_seq) >> 16;\r
- }\r
-\r
- // Test compression HC\r
- FUZ_DISPLAYTEST; // 1\r
- ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);\r
- FUZ_CHECKTEST(ret==0, "HC compression failed despite sufficient space");\r
- lenHC = ret;\r
-\r
- // Test compression\r
- FUZ_DISPLAYTEST; // 2\r
- ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);\r
- FUZ_CHECKTEST(ret==0, "compression failed despite sufficient space");\r
- len = ret;\r
-\r
- // Test decoding with output size being exactly what's necessary => must work\r
- FUZ_DISPLAYTEST; // 3\r
- ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN);\r
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");\r
-\r
- // Test decoding with one byte missing => must fail\r
- FUZ_DISPLAYTEST; // 4\r
- ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1);\r
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");\r
-\r
- // Test decoding with one byte too much => must fail\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1);\r
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");\r
-\r
- // Test decoding with enough output size => must work\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);\r
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");\r
-\r
- // Test decoding with output size being exactly what's necessary => must work\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN);\r
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");\r
-\r
- // Test decoding with output size being one byte too short => must fail\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);\r
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");\r
-\r
- // Test decoding with input size being one byte too short => must fail\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);\r
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short");\r
-\r
- // Test decoding with input size being one byte too large => must fail\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);\r
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");\r
- //if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }\r
-\r
- // Test partial decoding with target output size being max/2 => must work\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN);\r
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");\r
-\r
- // Test partial decoding with target output size being just below max => must work\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN);\r
- FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");\r
-\r
- // Test compression with output size being exactly what's necessary (should work)\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);\r
- FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");\r
- FUZ_CHECKTEST(ret==0, "compression failed despite sufficient space");\r
-\r
- // Test HC compression with output size being exactly what's necessary (should work)\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);\r
- FUZ_CHECKTEST(ret==0, "HC compression failed despite sufficient space");\r
-\r
- // Test compression with just one missing byte into output buffer => must fail\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);\r
- FUZ_CHECKTEST(ret, "compression overran output buffer");\r
- FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");\r
-\r
- // Test HC compression with just one missing byte into output buffer => must fail\r
- FUZ_DISPLAYTEST;\r
- ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1);\r
- FUZ_CHECKTEST(ret, "HC compression overran output buffer");\r
-\r
- bytes += LEN;\r
- cbytes += len;\r
- hcbytes += lenHC;\r
- FUZ_rand(&randState);\r
- }\r
-\r
- printf("all tests completed successfully \n");\r
- printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);\r
- printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);\r
- getchar();\r
- return 0;\r
-\r
-_output_error:\r
- getchar();\r
- return 1;\r
-}\r
+/*
+ fuzzer.c - Fuzzer test tool for LZ4
+ Copyright (C) Yann Collet - Andrew Mahone 2012-2013
+ Code started by Andrew Mahone, modified by Yann Collet
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+
+//**************************************
+// Remove Visual warning messages
+//**************************************
+#define _CRT_SECURE_NO_WARNINGS // fgets
+
+
+//**************************************
+// Includes
+//**************************************
+#include <stdlib.h>
+#include <stdio.h> // fgets, sscanf
+#include <sys/timeb.h> // timeb
+#include "lz4.h"
+#include "lz4hc.h"
+
+
+//**************************************
+// Constants
+//**************************************
+#ifndef LZ4_VERSION
+# define LZ4_VERSION ""
+#endif
+
+#define NB_ATTEMPTS (1<<17)
+#define LEN ((1<<15))
+#define SEQ_POW 2
+#define NUM_SEQ (1 << SEQ_POW)
+#define SEQ_MSK ((NUM_SEQ) - 1)
+#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0)
+#define NEW_SEQ(x) ((((x) >> 10) %10) == 0)
+#define PAGE_SIZE 4096
+#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))
+#define PRIME1 2654435761U
+#define PRIME2 2246822519U
+#define PRIME3 3266489917U
+
+
+//*********************************************************
+// Functions
+//*********************************************************
+static int FUZ_GetMilliStart()
+{
+ struct timeb tb;
+ int nCount;
+ ftime( &tb );
+ nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
+ return nCount;
+}
+
+
+static int FUZ_GetMilliSpan( int nTimeStart )
+{
+ int nSpan = FUZ_GetMilliStart() - nTimeStart;
+ if ( nSpan < 0 )
+ nSpan += 0x100000 * 1000;
+ return nSpan;
+}
+
+
+unsigned int FUZ_rand(unsigned int* src)
+{
+ *src = ((*src) * PRIME1) + PRIME2;
+ return *src;
+}
+
+
+int test_canary(unsigned char *buf)
+{
+ int i;
+ for (i = 0; i < 2048; i++)
+ if (buf[i] != buf[i + 2048])
+ return 0;
+ return 1;
+}
+
+
+int FUZ_SecurityTest()
+{
+ char* output;
+ char* input;
+ int i, r;
+
+ printf("Overflow test (issue 52)...\n");
+ input = (char*) malloc (20<<20);
+ output = (char*) malloc (20<<20);
+ input[0] = 0x0F;
+ input[1] = 0x00;
+ input[2] = 0x00;
+ for(i = 3; i < 16840000; i++)
+ input[i] = 0xff;
+ r = LZ4_decompress_fast(input, output, 20<<20);
+
+ free(input);
+ free(output);
+ printf(" Passed (return = %i < 0)\n",r);
+ return 0;
+}
+
+
+//int main(int argc, char *argv[]) {
+int main() {
+ unsigned long long bytes = 0;
+ unsigned long long cbytes = 0;
+ unsigned long long hcbytes = 0;
+ unsigned char buf[LEN];
+ unsigned char testOut[LEN+1];
+# define FUZ_max LZ4_COMPRESSBOUND(LEN)
+# define FUZ_avail ROUND_PAGE(FUZ_max)
+ const int off_full = FUZ_avail - FUZ_max;
+ unsigned char cbuf[FUZ_avail + PAGE_SIZE];
+ unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart();
+ int i, j, k, ret, len, lenHC, attemptNb;
+ char userInput[30] = {0};
+# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; }
+# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb);
+ void* stateLZ4 = malloc(LZ4_sizeofState());
+ void* stateLZ4HC = malloc(LZ4_sizeofStateHC());
+
+ printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION);
+ printf("Select an Initialisation number (default : random) : ");
+ fflush(stdout);
+ if ( fgets(userInput, sizeof userInput, stdin) )
+ {
+ if ( sscanf(userInput, "%d", &seed) == 1 ) {}
+ else seed = FUZ_GetMilliSpan(timestamp);
+ }
+ printf("Seed = %u\n", seed);
+ randState = seed;
+
+ //FUZ_SecurityTest();
+
+ for (i = 0; i < 2048; i++)
+ cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16;
+
+ for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++)
+ {
+ int testNb = 0;
+
+ printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS);
+
+ for (j = 0; j < NUM_SEQ; j++) {
+ seeds[j] = FUZ_rand(&randState) << 8;
+ seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535;
+ }
+ for (j = 0; j < LEN; j++) {
+ k = FUZ_rand(&randState);
+ if (j == 0 || NEW_SEQ(k))
+ cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK];
+ if (MOD_SEQ(k)) {
+ k = (FUZ_rand(&randState) >> 16) & SEQ_MSK;
+ seeds[k] = FUZ_rand(&randState) << 8;
+ seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535;
+ }
+ buf[j] = FUZ_rand(&cur_seq) >> 16;
+ }
+
+ // Test compression HC
+ FUZ_DISPLAYTEST; // 1
+ ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
+ FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
+ lenHC = ret;
+
+ // Test compression HC using external state
+ FUZ_DISPLAYTEST; // 1
+ ret = LZ4_compressHC_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[off_full], LEN);
+ FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
+
+ // Test compression using external state
+ FUZ_DISPLAYTEST; // 2
+ ret = LZ4_compress_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN);
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
+
+ // Test compression
+ FUZ_DISPLAYTEST; // 2
+ ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max);
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
+ len = ret;
+
+ // Test decoding with output size being exactly what's necessary => must work
+ FUZ_DISPLAYTEST; // 3
+ ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN);
+ FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
+
+ // Test decoding with one byte missing => must fail
+ FUZ_DISPLAYTEST; // 4
+ ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
+
+ // Test decoding with one byte too much => must fail
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
+
+ // Test decoding with enough output size => must work
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1);
+ FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
+
+ // Test decoding with output size being exactly what's necessary => must work
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN);
+ FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
+
+ // Test decoding with output size being one byte too short => must fail
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
+
+ // Test decoding with input size being one byte too short => must fail
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short");
+
+ // Test decoding with input size being one byte too large => must fail
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN);
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
+ //if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; }
+
+ // Test partial decoding with target output size being max/2 => must work
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN);
+ FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
+
+ // Test partial decoding with target output size being just below max => must work
+ FUZ_DISPLAYTEST;
+ ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN);
+ FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
+
+ // Test compression with output size being exactly what's necessary (should work)
+ FUZ_DISPLAYTEST;
+ ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len);
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space");
+ FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
+
+ // Test compression with output size being exactly what's necessary and external state (should work)
+ FUZ_DISPLAYTEST; // 2
+ ret = LZ4_compress_limitedOutput_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN, len);
+ FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space");
+ FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
+
+ // Test HC compression with output size being exactly what's necessary (should work)
+ FUZ_DISPLAYTEST;
+ ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
+ FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
+
+ // Test HC compression with output size being exactly what's necessary (should work)
+ FUZ_DISPLAYTEST;
+ ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC);
+ FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
+
+ // Test compression with just one missing byte into output buffer => must fail
+ FUZ_DISPLAYTEST;
+ ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1);
+ FUZ_CHECKTEST(ret, "compression overran output buffer");
+ FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer");
+
+ // Test HC compression with just one missing byte into output buffer => must fail
+ FUZ_DISPLAYTEST;
+ ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1);
+ FUZ_CHECKTEST(ret, "HC compression overran output buffer");
+
+ bytes += LEN;
+ cbytes += len;
+ hcbytes += lenHC;
+ FUZ_rand(&randState);
+ }
+
+ printf("all tests completed successfully \n");
+ printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
+ printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
+ getchar();
+ return 0;
+
+_output_error:
+ getchar();
+ return 1;
+}
-\"\r
-\" lz4.1: This is a manual page for 'lz4' program. This file is part of the\r
-\" lz4 <https://code.google.com/p/lz4/> project.\r
-\"\r
-\r
-\" No hyphenation\r
-.hy 0\r
-.nr HY 0\r
-\r
-.TH lz4 man\r
-.SH NAME\r
-\fBlz4\fR - Extremely fast compression algorithm\r
-\r
-.SH SYNOPSIS\r
-.TP 5\r
-\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>\r
-\r
-.SH DESCRIPTION\r
-.PP\r
-\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on\r
-the \fBLZ77\fR family of compression scheme. At the compression speed of 400\r
-MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features\r
-an extremely fast decoder, with speed in multiple GB/s per core, typically\r
-reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports\r
-following options\r
-\r
-.SH OPTIONS\r
-.TP\r
-.B \-1\r
- fast compression (default)\r
-.TP\r
-.B \-9\r
- high compression\r
-.TP\r
-.B \-d\r
- decompression\r
-.TP\r
-.B \-f\r
- overwrite output without prompting\r
-.TP\r
-.B \-h/\-H\r
- display help/long help and exit\r
-.TP\r
-.B \-V\r
- display Version number and exit\r
-.TP\r
-.B \-v\r
- verbose mode\r
-.TP\r
-.B \-q\r
- suppress warnings; specify twice to suppress errors too\r
-.TP\r
-.B \-c\r
- force write to standard output, even if it is the console\r
-.TP\r
-.B \-t\r
- test compressed file integrity\r
-.TP\r
-.B \-z\r
- force compression\r
-.TP\r
-.B \-B#\r
- block size [4-7](default : 7)\r
-.TP\r
-.B \-BD\r
- block dependency (improve compression ratio)\r
-.TP\r
-.B \-BX\r
- enable block checksum (default:disabled)\r
-.TP\r
-.B \-Sx\r
- disable stream checksum (default:enabled)\r
-.TP\r
-.B \-b\r
- benchmark file(s)\r
-.TP\r
-.B \-i#\r
- iteration loops [1-9](default : 3), benchmark mode only\r
-\r
-.SH BUGS\r
-Report bugs at:- https://code.google.com/p/lz4/\r
-\r
-.SH AUTHOR\r
+\"
+\" lz4.1: This is a manual page for 'lz4' program. This file is part of the
+\" lz4 <https://code.google.com/p/lz4/> project.
+\"
+
+\" No hyphenation
+.hy 0
+.nr HY 0
+
+.TH lz4 man
+.SH NAME
+\fBlz4\fR - Extremely fast compression algorithm
+
+.SH SYNOPSIS
+.TP 5
+\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] <OUTPUT-FILE>
+
+.SH DESCRIPTION
+.PP
+\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on
+the \fBLZ77\fR family of compression scheme. At the compression speed of 400
+MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features
+an extremely fast decoder, with speed in multiple GB/s per core, typically
+reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports
+following options
+
+.SH OPTIONS
+.TP
+.B \-1
+ fast compression (default)
+.TP
+.B \-9
+ high compression
+.TP
+.B \-d
+ decompression
+.TP
+.B \-f
+ overwrite output without prompting
+.TP
+.B \-h/\-H
+ display help/long help and exit
+.TP
+.B \-V
+ display Version number and exit
+.TP
+.B \-v
+ verbose mode
+.TP
+.B \-q
+ suppress warnings; specify twice to suppress errors too
+.TP
+.B \-c
+ force write to standard output, even if it is the console
+.TP
+.B \-t
+ test compressed file integrity
+.TP
+.B \-z
+ force compression
+.TP
+.B \-l
+ use Legacy format (useful for Linux Kernel compression)
+.TP
+.B \-B#
+ block size [4-7](default : 7)
+.TP
+.B \-BD
+ block dependency (improve compression ratio)
+.TP
+.B \-BX
+ enable block checksum (default:disabled)
+.TP
+.B \-Sx
+ disable stream checksum (default:enabled)
+.TP
+.B \-b
+ benchmark file(s)
+.TP
+.B \-i#
+ iteration loops [1-9](default : 3), benchmark mode only
+
+.SH BUGS
+Report bugs at:- https://code.google.com/p/lz4/
+
+.SH AUTHOR
Yann Collet
\ No newline at end of file
-/*\r
- LZ4 - Fast LZ compression algorithm\r
- Copyright (C) 2011-2013, Yann Collet.\r
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)\r
-\r
- Redistribution and use in source and binary forms, with or without\r
- modification, are permitted provided that the following conditions are\r
- met:\r
-\r
- * Redistributions of source code must retain the above copyright\r
- notice, this list of conditions and the following disclaimer.\r
- * Redistributions in binary form must reproduce the above\r
- copyright notice, this list of conditions and the following disclaimer\r
- in the documentation and/or other materials provided with the\r
- distribution.\r
-\r
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\r
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\r
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\r
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\r
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
- You can contact the author at :\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c\r
-*/\r
-\r
-//**************************************\r
-// Tuning parameters\r
-//**************************************\r
-// MEMORY_USAGE :\r
-// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)\r
-// Increasing memory usage improves compression ratio\r
-// Reduced memory usage can improve speed, due to cache effect\r
-// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache\r
-#define MEMORY_USAGE 14\r
-\r
-// HEAPMODE :\r
-// Select how default compression functions will allocate memory for their hash table,\r
-// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).\r
-#define HEAPMODE 0\r
-\r
-\r
-//**************************************\r
-// CPU Feature Detection\r
-//**************************************\r
-// 32 or 64 bits ?\r
-#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \\r
- || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \\r
- || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \\r
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode\r
-# define LZ4_ARCH64 1\r
-#else\r
-# define LZ4_ARCH64 0\r
-#endif\r
-\r
-// Little Endian or Big Endian ?\r
-// Overwrite the #define below if you know your architecture endianess\r
-#if defined (__GLIBC__)\r
-# include <endian.h>\r
-# if (__BYTE_ORDER == __BIG_ENDIAN)\r
-# define LZ4_BIG_ENDIAN 1\r
-# endif\r
-#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))\r
-# define LZ4_BIG_ENDIAN 1\r
-#elif defined(__sparc) || defined(__sparc__) \\r
- || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \\r
- || defined(__hpux) || defined(__hppa) \\r
- || defined(_MIPSEB) || defined(__s390__)\r
-# define LZ4_BIG_ENDIAN 1\r
-#else\r
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.\r
-#endif\r
-\r
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.\r
-// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property\r
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance\r
-#if defined(__ARM_FEATURE_UNALIGNED)\r
-# define LZ4_FORCE_UNALIGNED_ACCESS 1\r
-#endif\r
-\r
-// Define this parameter if your target system or compiler does not support hardware bit count\r
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count\r
-# define LZ4_FORCE_SW_BITCOUNT\r
-#endif\r
-\r
-// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :\r
-// This option may provide a small boost to performance for some big endian cpu, although probably modest.\r
-// You may set this option to 1 if data will remain within closed environment.\r
-// This option is useless on Little_Endian CPU (such as x86)\r
-//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1\r
-\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99\r
-/* "restrict" is a known keyword */\r
-#else\r
-# define restrict // Disable restrict\r
-#endif\r
-\r
-#ifdef _MSC_VER // Visual Studio\r
-# define FORCE_INLINE static __forceinline\r
-# include <intrin.h> // For Visual 2005\r
-# if LZ4_ARCH64 // 64-bits\r
-# pragma intrinsic(_BitScanForward64) // For Visual 2005\r
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005\r
-# else // 32-bits\r
-# pragma intrinsic(_BitScanForward) // For Visual 2005\r
-# pragma intrinsic(_BitScanReverse) // For Visual 2005\r
-# endif\r
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant\r
-#else \r
-# ifdef __GNUC__\r
-# define FORCE_INLINE static inline __attribute__((always_inline))\r
-# else\r
-# define FORCE_INLINE static inline\r
-# endif\r
-#endif\r
-\r
-#ifdef _MSC_VER\r
-# define lz4_bswap16(x) _byteswap_ushort(x)\r
-#else\r
-# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))\r
-#endif\r
-\r
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)\r
-\r
-#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)\r
-# define expect(expr,value) (__builtin_expect ((expr),(value)) )\r
-#else\r
-# define expect(expr,value) (expr)\r
-#endif\r
-\r
-#define likely(expr) expect((expr) != 0, 1)\r
-#define unlikely(expr) expect((expr) != 0, 0)\r
-\r
-\r
-//**************************************\r
-// Memory routines\r
-//**************************************\r
-#include <stdlib.h> // malloc, calloc, free\r
-#define ALLOCATOR(n,s) calloc(n,s)\r
-#define FREEMEM free\r
-#include <string.h> // memset, memcpy\r
-#define MEM_INIT memset\r
-\r
-\r
-//**************************************\r
-// Includes\r
-//**************************************\r
-#include "lz4.h"\r
-\r
-\r
-//**************************************\r
-// Basic Types\r
-//**************************************\r
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99\r
-# include <stdint.h>\r
- typedef uint8_t BYTE;\r
- typedef uint16_t U16;\r
- typedef uint32_t U32;\r
- typedef int32_t S32;\r
- typedef uint64_t U64;\r
-#else\r
- typedef unsigned char BYTE;\r
- typedef unsigned short U16;\r
- typedef unsigned int U32;\r
- typedef signed int S32;\r
- typedef unsigned long long U64;\r
-#endif\r
-\r
-#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)\r
-# define _PACKED __attribute__ ((packed))\r
-#else\r
-# define _PACKED\r
-#endif\r
-\r
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)\r
-# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC)\r
-# pragma pack(1)\r
-# else\r
-# pragma pack(push, 1)\r
-# endif\r
-#endif\r
-\r
-typedef struct { U16 v; } _PACKED U16_S;\r
-typedef struct { U32 v; } _PACKED U32_S;\r
-typedef struct { U64 v; } _PACKED U64_S;\r
-typedef struct {size_t v;} _PACKED size_t_S;\r
-\r
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)\r
-# if defined(__SUNPRO_C) || defined(__SUNPRO_CC)\r
-# pragma pack(0)\r
-# else\r
-# pragma pack(pop)\r
-# endif\r
-#endif\r
-\r
-#define A16(x) (((U16_S *)(x))->v)\r
-#define A32(x) (((U32_S *)(x))->v)\r
-#define A64(x) (((U64_S *)(x))->v)\r
-#define AARCH(x) (((size_t_S *)(x))->v)\r
-\r
-\r
-//**************************************\r
-// Constants\r
-//**************************************\r
-#define LZ4_HASHLOG (MEMORY_USAGE-2)\r
-#define HASHTABLESIZE (1 << MEMORY_USAGE)\r
-#define HASHNBCELLS4 (1 << LZ4_HASHLOG)\r
-\r
-#define MINMATCH 4\r
-\r
-#define COPYLENGTH 8\r
-#define LASTLITERALS 5\r
-#define MFLIMIT (COPYLENGTH+MINMATCH)\r
-const int LZ4_minLength = (MFLIMIT+1);\r
-\r
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1))\r
-#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data\r
-\r
-#define MAXD_LOG 16\r
-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)\r
-\r
-#define ML_BITS 4\r
-#define ML_MASK ((1U<<ML_BITS)-1)\r
-#define RUN_BITS (8-ML_BITS)\r
-#define RUN_MASK ((1U<<RUN_BITS)-1)\r
-\r
-#define KB *(1U<<10)\r
-#define MB *(1U<<20)\r
-#define GB *(1U<<30)\r
-\r
-\r
-//**************************************\r
-// Structures and local types\r
-//**************************************\r
-\r
-typedef struct {\r
- U32 hashTable[HASHNBCELLS4];\r
- const BYTE* bufferStart;\r
- const BYTE* base;\r
- const BYTE* nextBlock;\r
-} LZ4_Data_Structure;\r
-\r
-typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive;\r
-typedef enum { byPtr, byU32, byU16 } tableType_t;\r
-\r
-typedef enum { noPrefix = 0, withPrefix = 1 } prefix64k_directive;\r
-\r
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;\r
-typedef enum { full = 0, partial = 1 } earlyEnd_directive;\r
-\r
-\r
-//**************************************\r
-// Architecture-specific macros\r
-//**************************************\r
-#define STEPSIZE sizeof(size_t)\r
-#define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; }\r
-#define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); }\r
-#define LZ4_SECURECOPY(d,s,e) { if ((STEPSIZE==4)||(d<e)) LZ4_WILDCOPY(d,s,e); }\r
-\r
-#if LZ4_ARCH64 // 64-bit\r
-# define HTYPE U32\r
-# define INITBASE(base) const BYTE* const base = ip\r
-#else // 32-bit\r
-# define HTYPE const BYTE*\r
-# define INITBASE(base) const int base = 0\r
-#endif\r
-\r
-#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))\r
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }\r
-# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }\r
-#else // Little Endian\r
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }\r
-# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Macros\r
-//**************************************\r
-#define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } // at the end, d>=e;\r
-\r
-\r
-//****************************\r
-// Private functions\r
-//****************************\r
-#if LZ4_ARCH64\r
-\r
-FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)\r
-{\r
-# if defined(LZ4_BIG_ENDIAN)\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r = 0;\r
- _BitScanReverse64( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_clzll(val) >> 3);\r
-# else\r
- int r;\r
- if (!(val>>32)) { r=4; } else { r=0; val>>=32; }\r
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }\r
- r += (!val);\r
- return r;\r
-# endif\r
-# else\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r = 0;\r
- _BitScanForward64( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_ctzll(val) >> 3);\r
-# else\r
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };\r
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];\r
-# endif\r
-# endif\r
-}\r
-\r
-#else\r
-\r
-FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)\r
-{\r
-# if defined(LZ4_BIG_ENDIAN)\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r = 0;\r
- _BitScanReverse( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_clz(val) >> 3);\r
-# else\r
- int r;\r
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }\r
- r += (!val);\r
- return r;\r
-# endif\r
-# else\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r;\r
- _BitScanForward( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_ctz(val) >> 3);\r
-# else\r
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };\r
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];\r
-# endif\r
-# endif\r
-}\r
-\r
-#endif\r
-\r
-\r
-//****************************\r
-// Compression functions\r
-//****************************\r
-FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)\r
-{\r
- if (tableType == byU16)\r
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));\r
- else\r
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));\r
-}\r
-\r
-FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }\r
-\r
-FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)\r
-{\r
- switch (tableType)\r
- {\r
- case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; }\r
- case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; }\r
- case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; }\r
- }\r
-}\r
-\r
-FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)\r
-{\r
- U32 h = LZ4_hashPosition(p, tableType);\r
- LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);\r
-}\r
-\r
-FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)\r
-{\r
- if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }\r
- if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }\r
- { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return\r
-}\r
-\r
-FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)\r
-{\r
- U32 h = LZ4_hashPosition(p, tableType);\r
- return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);\r
-}\r
-\r
-\r
-FORCE_INLINE int LZ4_compress_generic(\r
- void* ctx,\r
- const char* source,\r
- char* dest,\r
- int inputSize,\r
- int maxOutputSize,\r
-\r
- limitedOutput_directive limitedOutput,\r
- tableType_t tableType,\r
- prefix64k_directive prefix)\r
-{\r
- const BYTE* ip = (const BYTE*) source;\r
- const BYTE* const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source;\r
- const BYTE* const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source);\r
- const BYTE* anchor = (const BYTE*) source;\r
- const BYTE* const iend = ip + inputSize;\r
- const BYTE* const mflimit = iend - MFLIMIT;\r
- const BYTE* const matchlimit = iend - LASTLITERALS;\r
-\r
- BYTE* op = (BYTE*) dest;\r
- BYTE* const oend = op + maxOutputSize;\r
-\r
- int length;\r
- const int skipStrength = SKIPSTRENGTH;\r
- U32 forwardH;\r
-\r
- // Init conditions\r
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; // Unsupported input size, too large (or negative)\r
- if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; // must continue from end of previous block\r
- if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; // do it now, due to potential early exit\r
- if ((tableType == byU16) && (inputSize>=LZ4_64KLIMIT)) return 0; // Size too large (not within 64K limit)\r
- if (inputSize<LZ4_minLength) goto _last_literals; // Input too small, no compression (all literals)\r
-\r
- // First Byte\r
- LZ4_putPosition(ip, ctx, tableType, base);\r
- ip++; forwardH = LZ4_hashPosition(ip, tableType);\r
-\r
- // Main Loop\r
- for ( ; ; )\r
- {\r
- int findMatchAttempts = (1U << skipStrength) + 3;\r
- const BYTE* forwardIp = ip;\r
- const BYTE* ref;\r
- BYTE* token;\r
-\r
- // Find a match\r
- do {\r
- U32 h = forwardH;\r
- int step = findMatchAttempts++ >> skipStrength;\r
- ip = forwardIp;\r
- forwardIp = ip + step;\r
-\r
- if unlikely(forwardIp > mflimit) { goto _last_literals; }\r
-\r
- forwardH = LZ4_hashPosition(forwardIp, tableType);\r
- ref = LZ4_getPositionOnHash(h, ctx, tableType, base);\r
- LZ4_putPositionOnHash(ip, h, ctx, tableType, base);\r
-\r
- } while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip)));\r
-\r
- // Catch up\r
- while ((ip>anchor) && (ref > lowLimit) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; }\r
-\r
- // Encode Literal length\r
- length = (int)(ip - anchor);\r
- token = op++;\r
- if ((limitedOutput) && unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend)) return 0; // Check output limit\r
- if (length>=(int)RUN_MASK) \r
- { \r
- int len = length-RUN_MASK; \r
- *token=(RUN_MASK<<ML_BITS); \r
- for(; len >= 255 ; len-=255) *op++ = 255; \r
- *op++ = (BYTE)len; \r
- }\r
- else *token = (BYTE)(length<<ML_BITS);\r
-\r
- // Copy Literals\r
- { BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; }\r
-\r
-_next_match:\r
- // Encode Offset\r
- LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));\r
-\r
- // Start Counting\r
- ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified\r
- anchor = ip;\r
- while likely(ip<matchlimit-(STEPSIZE-1))\r
- {\r
- size_t diff = AARCH(ref) ^ AARCH(ip);\r
- if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }\r
- ip += LZ4_NbCommonBytes(diff);\r
- goto _endCount;\r
- }\r
- if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }\r
- if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }\r
- if ((ip<matchlimit) && (*ref == *ip)) ip++;\r
-_endCount:\r
-\r
- // Encode MatchLength\r
- length = (int)(ip - anchor);\r
- if ((limitedOutput) && unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend)) return 0; // Check output limit\r
- if (length>=(int)ML_MASK) \r
- { \r
- *token += ML_MASK; \r
- length -= ML_MASK; \r
- for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; } \r
- if (length >= 255) { length-=255; *op++ = 255; } \r
- *op++ = (BYTE)length; \r
- }\r
- else *token += (BYTE)(length);\r
-\r
- // Test end of chunk\r
- if (ip > mflimit) { anchor = ip; break; }\r
-\r
- // Fill table\r
- LZ4_putPosition(ip-2, ctx, tableType, base);\r
-\r
- // Test next position\r
- ref = LZ4_getPosition(ip, ctx, tableType, base); \r
- LZ4_putPosition(ip, ctx, tableType, base);\r
- if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }\r
-\r
- // Prepare next loop\r
- anchor = ip++;\r
- forwardH = LZ4_hashPosition(ip, tableType);\r
- }\r
-\r
-_last_literals:\r
- // Encode Last Literals\r
- {\r
- int lastRun = (int)(iend - anchor);\r
- if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit\r
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }\r
- else *op++ = (BYTE)(lastRun<<ML_BITS);\r
- memcpy(op, anchor, iend - anchor);\r
- op += iend-anchor;\r
- }\r
-\r
- // End\r
- return (int) (((char*)op)-dest);\r
-}\r
-\r
-\r
-int LZ4_compress(const char* source, char* dest, int inputSize)\r
-{\r
-#if (HEAPMODE)\r
- void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries\r
-#else\r
- U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries\r
-#endif\r
- int result;\r
-\r
- if (inputSize < (int)LZ4_64KLIMIT)\r
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noPrefix);\r
- else\r
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);\r
-\r
-#if (HEAPMODE)\r
- FREEMEM(ctx);\r
-#endif\r
- return result;\r
-}\r
-\r
-int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize)\r
-{\r
- return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, 0, notLimited, byU32, withPrefix);\r
-}\r
-\r
-\r
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)\r
-{\r
-#if (HEAPMODE)\r
- void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries\r
-#else\r
- U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries\r
-#endif\r
- int result;\r
-\r
- if (inputSize < (int)LZ4_64KLIMIT)\r
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noPrefix);\r
- else\r
- result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);\r
-\r
-#if (HEAPMODE)\r
- FREEMEM(ctx);\r
-#endif\r
- return result;\r
-}\r
-\r
-int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize)\r
-{\r
- return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix);\r
-}\r
-\r
-\r
-//****************************\r
-// Stream functions\r
-//****************************\r
-\r
-FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)\r
-{\r
- MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable));\r
- lz4ds->bufferStart = base;\r
- lz4ds->base = base;\r
- lz4ds->nextBlock = base;\r
-}\r
-\r
-\r
-void* LZ4_create (const char* inputBuffer)\r
-{\r
- void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure));\r
- LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer);\r
- return lz4ds;\r
-}\r
-\r
-\r
-int LZ4_free (void* LZ4_Data)\r
-{\r
- FREEMEM(LZ4_Data);\r
- return (0);\r
-}\r
-\r
-\r
-char* LZ4_slideInputBuffer (void* LZ4_Data)\r
-{\r
- LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data;\r
- size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB);\r
-\r
- if ( (lz4ds->base - delta > lz4ds->base) // underflow control\r
- || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) // close to 32-bits limit\r
- {\r
- size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base;\r
- int nH;\r
-\r
- for (nH=0; nH < HASHNBCELLS4; nH++)\r
- {\r
- if ((size_t)(lz4ds->hashTable[nH]) < deltaLimit) lz4ds->hashTable[nH] = 0;\r
- else lz4ds->hashTable[nH] -= (U32)deltaLimit;\r
- }\r
- memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);\r
- lz4ds->base = lz4ds->bufferStart;\r
- lz4ds->nextBlock = lz4ds->base + 64 KB;\r
- }\r
- else\r
- {\r
- memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);\r
- lz4ds->nextBlock -= delta;\r
- lz4ds->base -= delta;\r
- }\r
-\r
- return (char*)(lz4ds->nextBlock);\r
-}\r
-\r
-\r
-//****************************\r
-// Decompression functions\r
-//****************************\r
-\r
-// This generic decompression function cover all use cases.\r
-// It shall be instanciated several times, using different sets of directives\r
-// Note that it is essential this generic function is really inlined, \r
-// in order to remove useless branches during compilation optimisation.\r
-FORCE_INLINE int LZ4_decompress_generic(\r
- const char* source,\r
- char* dest,\r
- int inputSize, //\r
- int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer.\r
-\r
- int endOnInput, // endOnOutputSize, endOnInputSize\r
- int prefix64k, // noPrefix, withPrefix\r
- int partialDecoding, // full, partial\r
- int targetOutputSize // only used if partialDecoding==partial\r
- )\r
-{\r
- // Local Variables\r
- const BYTE* restrict ip = (const BYTE*) source;\r
- const BYTE* ref;\r
- const BYTE* const iend = ip + inputSize;\r
-\r
- BYTE* op = (BYTE*) dest;\r
- BYTE* const oend = op + outputSize;\r
- BYTE* cpy;\r
- BYTE* oexit = op + targetOutputSize;\r
-\r
- const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64\r
- static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};\r
-\r
-\r
- // Special cases\r
- if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything\r
- if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer\r
- if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1);\r
-\r
-\r
- // Main Loop\r
- while (1)\r
- {\r
- unsigned token;\r
- size_t length;\r
-\r
- // get runlength\r
- token = *ip++;\r
- if ((length=(token>>ML_BITS)) == RUN_MASK)\r
- { \r
- unsigned s=255; \r
- while (((endOnInput)?ip<iend:1) && (s==255))\r
- { \r
- s = *ip++; \r
- length += s; \r
- } \r
- }\r
-\r
- // copy literals\r
- cpy = op+length;\r
- if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )\r
- || ((!endOnInput) && (cpy>oend-COPYLENGTH)))\r
- {\r
- if (partialDecoding)\r
- {\r
- if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer\r
- if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer\r
- }\r
- else\r
- {\r
- if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there\r
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed\r
- }\r
- memcpy(op, ip, length);\r
- ip += length;\r
- op += length;\r
- break; // Necessarily EOF, due to parsing restrictions\r
- }\r
- LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;\r
-\r
- // get offset\r
- LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;\r
- if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer\r
-\r
- // get matchlength\r
- if ((length=(token&ML_MASK)) == ML_MASK) \r
- { \r
- while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) // Ensure enough bytes remain for LASTLITERALS + token\r
- {\r
- unsigned s = *ip++; \r
- length += s; \r
- if (s==255) continue; \r
- break; \r
- }\r
- }\r
-\r
- // copy repeated sequence\r
- if unlikely((op-ref)<(int)STEPSIZE)\r
- {\r
- const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref];\r
- op[0] = ref[0];\r
- op[1] = ref[1];\r
- op[2] = ref[2];\r
- op[3] = ref[3];\r
- op += 4, ref += 4; ref -= dec32table[op-ref];\r
- A32(op) = A32(ref); \r
- op += STEPSIZE-4; ref -= dec64;\r
- } else { LZ4_COPYSTEP(op,ref); }\r
- cpy = op + length - (STEPSIZE-4);\r
-\r
- if unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4))\r
- {\r
- if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals\r
- LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH));\r
- while(op<cpy) *op++=*ref++;\r
- op=cpy;\r
- continue;\r
- }\r
- LZ4_WILDCOPY(op, ref, cpy);\r
- op=cpy; // correction\r
- }\r
-\r
- // end of decoding\r
- if (endOnInput)\r
- return (int) (((char*)op)-dest); // Nb of output bytes decoded\r
- else\r
- return (int) (((char*)ip)-source); // Nb of input bytes read\r
-\r
- // Overflow error detected\r
-_output_error:\r
- return (int) (-(((char*)ip)-source))-1;\r
-}\r
-\r
-\r
-int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize)\r
-{\r
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, full, 0);\r
-}\r
-\r
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int inputSize, int maxOutputSize)\r
-{\r
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, withPrefix, full, 0);\r
-}\r
-\r
-int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize)\r
-{\r
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize);\r
-}\r
-\r
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize)\r
-{\r
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);\r
-}\r
-\r
-int LZ4_decompress_fast(const char* source, char* dest, int outputSize)\r
-{\r
-#ifdef _MSC_VER // This version is faster with Visual\r
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0);\r
-#else\r
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);\r
-#endif\r
-}\r
-\r
+/*
+ LZ4 - Fast LZ compression algorithm
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+//**************************************
+// Tuning parameters
+//**************************************
+// MEMORY_USAGE :
+// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+// Increasing memory usage improves compression ratio
+// Reduced memory usage can improve speed, due to cache effect
+// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+#define MEMORY_USAGE 14
+
+// HEAPMODE :
+// Select how default compression functions will allocate memory for their hash table,
+// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
+#define HEAPMODE 0
+
+
+//**************************************
+// CPU Feature Detection
+//**************************************
+// 32 or 64 bits ?
+#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
+ || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
+ || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
+ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
+# define LZ4_ARCH64 1
+#else
+# define LZ4_ARCH64 0
+#endif
+
+// Little Endian or Big Endian ?
+// Overwrite the #define below if you know your architecture endianess
+#if defined (__GLIBC__)
+# include <endian.h>
+# if (__BYTE_ORDER == __BIG_ENDIAN)
+# define LZ4_BIG_ENDIAN 1
+# endif
+#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))
+# define LZ4_BIG_ENDIAN 1
+#elif defined(__sparc) || defined(__sparc__) \
+ || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \
+ || defined(__hpux) || defined(__hppa) \
+ || defined(_MIPSEB) || defined(__s390__)
+# define LZ4_BIG_ENDIAN 1
+#else
+// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
+#endif
+
+// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property
+// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+#if defined(__ARM_FEATURE_UNALIGNED)
+# define LZ4_FORCE_UNALIGNED_ACCESS 1
+#endif
+
+// Define this parameter if your target system or compiler does not support hardware bit count
+#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
+# define LZ4_FORCE_SW_BITCOUNT
+#endif
+
+// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
+// This option may provide a small boost to performance for some big endian cpu, although probably modest.
+// You may set this option to 1 if data will remain within closed environment.
+// This option is useless on Little_Endian CPU (such as x86)
+//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1
+
+
+//**************************************
+// Compiler Options
+//**************************************
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99
+/* "restrict" is a known keyword */
+#else
+# define restrict // Disable restrict
+#endif
+
+#ifdef _MSC_VER // Visual Studio
+# define FORCE_INLINE static __forceinline
+# include <intrin.h> // For Visual 2005
+# if LZ4_ARCH64 // 64-bits
+# pragma intrinsic(_BitScanForward64) // For Visual 2005
+# pragma intrinsic(_BitScanReverse64) // For Visual 2005
+# else // 32-bits
+# pragma intrinsic(_BitScanForward) // For Visual 2005
+# pragma intrinsic(_BitScanReverse) // For Visual 2005
+# endif
+# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+#else
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+#endif
+
+#ifdef _MSC_VER
+# define lz4_bswap16(x) _byteswap_ushort(x)
+#else
+# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
+#endif
+
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
+# define expect(expr,value) (__builtin_expect ((expr),(value)) )
+#else
+# define expect(expr,value) (expr)
+#endif
+
+#define likely(expr) expect((expr) != 0, 1)
+#define unlikely(expr) expect((expr) != 0, 0)
+
+
+//**************************************
+// Memory routines
+//**************************************
+#include <stdlib.h> // malloc, calloc, free
+#define ALLOCATOR(n,s) calloc(n,s)
+#define FREEMEM free
+#include <string.h> // memset, memcpy
+#define MEM_INIT memset
+
+
+//**************************************
+// Includes
+//**************************************
+#include "lz4.h"
+
+
+//**************************************
+// Basic Types
+//**************************************
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)
+# define _PACKED __attribute__ ((packed))
+#else
+# define _PACKED
+#endif
+
+#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+# pragma pack(1)
+# else
+# pragma pack(push, 1)
+# endif
+#endif
+
+typedef struct { U16 v; } _PACKED U16_S;
+typedef struct { U32 v; } _PACKED U32_S;
+typedef struct { U64 v; } _PACKED U64_S;
+typedef struct {size_t v;} _PACKED size_t_S;
+
+#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+# pragma pack(0)
+# else
+# pragma pack(pop)
+# endif
+#endif
+
+#define A16(x) (((U16_S *)(x))->v)
+#define A32(x) (((U32_S *)(x))->v)
+#define A64(x) (((U64_S *)(x))->v)
+#define AARCH(x) (((size_t_S *)(x))->v)
+
+
+//**************************************
+// Constants
+//**************************************
+#define LZ4_HASHLOG (MEMORY_USAGE-2)
+#define HASHTABLESIZE (1 << MEMORY_USAGE)
+#define HASHNBCELLS4 (1 << LZ4_HASHLOG)
+
+#define MINMATCH 4
+
+#define COPYLENGTH 8
+#define LASTLITERALS 5
+#define MFLIMIT (COPYLENGTH+MINMATCH)
+const int LZ4_minLength = (MFLIMIT+1);
+
+#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1))
+#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data
+
+#define MAXD_LOG 16
+#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+
+//**************************************
+// Structures and local types
+//**************************************
+
+typedef struct {
+ U32 hashTable[HASHNBCELLS4];
+ const BYTE* bufferStart;
+ const BYTE* base;
+ const BYTE* nextBlock;
+} LZ4_Data_Structure;
+
+typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive;
+typedef enum { byPtr, byU32, byU16 } tableType_t;
+
+typedef enum { noPrefix = 0, withPrefix = 1 } prefix64k_directive;
+
+typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
+typedef enum { full = 0, partial = 1 } earlyEnd_directive;
+
+
+//**************************************
+// Architecture-specific macros
+//**************************************
+#define STEPSIZE sizeof(size_t)
+#define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; }
+#define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); }
+#define LZ4_SECURECOPY(d,s,e) { if ((STEPSIZE==4)||(d<e)) LZ4_WILDCOPY(d,s,e); }
+
+#if LZ4_ARCH64 // 64-bit
+# define HTYPE U32
+# define INITBASE(base) const BYTE* const base = ip
+#else // 32-bit
+# define HTYPE const BYTE*
+# define INITBASE(base) const int base = 0
+#endif
+
+#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))
+# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
+# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
+#else // Little Endian
+# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
+# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
+#endif
+
+
+//**************************************
+// Macros
+//**************************************
+#define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } // at the end, d>=e;
+
+
+//****************************
+// Private functions
+//****************************
+#if LZ4_ARCH64
+
+FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
+{
+# if defined(LZ4_BIG_ENDIAN)
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanReverse64( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_clzll(val) >> 3);
+# else
+ int r;
+ if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
+ if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+ r += (!val);
+ return r;
+# endif
+# else
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanForward64( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_ctzll(val) >> 3);
+# else
+ static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+ return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+# endif
+# endif
+}
+
+#else
+
+FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
+{
+# if defined(LZ4_BIG_ENDIAN)
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanReverse( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_clz(val) >> 3);
+# else
+ int r;
+ if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+ r += (!val);
+ return r;
+# endif
+# else
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r;
+ _BitScanForward( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_ctz(val) >> 3);
+# else
+ static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
+ return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+# endif
+# endif
+}
+
+#endif
+
+
+//****************************
+// Compression functions
+//****************************
+FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType)
+{
+ if (tableType == byU16)
+ return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
+ else
+ return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
+}
+
+FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
+
+FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ switch (tableType)
+ {
+ case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; }
+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; }
+ case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; }
+ }
+}
+
+FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ U32 h = LZ4_hashPosition(p, tableType);
+ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
+}
+
+FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
+ if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
+ { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return
+}
+
+FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ U32 h = LZ4_hashPosition(p, tableType);
+ return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
+}
+
+
+FORCE_INLINE int LZ4_compress_generic(
+ void* ctx,
+ const char* source,
+ char* dest,
+ int inputSize,
+ int maxOutputSize,
+
+ limitedOutput_directive limitedOutput,
+ tableType_t tableType,
+ prefix64k_directive prefix)
+{
+ const BYTE* ip = (const BYTE*) source;
+ const BYTE* const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source;
+ const BYTE* const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source);
+ const BYTE* anchor = (const BYTE*) source;
+ const BYTE* const iend = ip + inputSize;
+ const BYTE* const mflimit = iend - MFLIMIT;
+ const BYTE* const matchlimit = iend - LASTLITERALS;
+
+ BYTE* op = (BYTE*) dest;
+ BYTE* const oend = op + maxOutputSize;
+
+ int length;
+ const int skipStrength = SKIPSTRENGTH;
+ U32 forwardH;
+
+ // Init conditions
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; // Unsupported input size, too large (or negative)
+ if ((prefix==withPrefix) && (ip != ((LZ4_Data_Structure*)ctx)->nextBlock)) return 0; // must continue from end of previous block
+ if (prefix==withPrefix) ((LZ4_Data_Structure*)ctx)->nextBlock=iend; // do it now, due to potential early exit
+ if ((tableType == byU16) && (inputSize>=LZ4_64KLIMIT)) return 0; // Size too large (not within 64K limit)
+ if (inputSize<LZ4_minLength) goto _last_literals; // Input too small, no compression (all literals)
+
+ // First Byte
+ LZ4_putPosition(ip, ctx, tableType, base);
+ ip++; forwardH = LZ4_hashPosition(ip, tableType);
+
+ // Main Loop
+ for ( ; ; )
+ {
+ int findMatchAttempts = (1U << skipStrength) + 3;
+ const BYTE* forwardIp = ip;
+ const BYTE* ref;
+ BYTE* token;
+
+ // Find a match
+ do {
+ U32 h = forwardH;
+ int step = findMatchAttempts++ >> skipStrength;
+ ip = forwardIp;
+ forwardIp = ip + step;
+
+ if unlikely(forwardIp > mflimit) { goto _last_literals; }
+
+ forwardH = LZ4_hashPosition(forwardIp, tableType);
+ ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
+ LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
+
+ } while ((ref + MAX_DISTANCE < ip) || (A32(ref) != A32(ip)));
+
+ // Catch up
+ while ((ip>anchor) && (ref > lowLimit) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; }
+
+ // Encode Literal length
+ length = (int)(ip - anchor);
+ token = op++;
+ if ((limitedOutput) && unlikely(op + length + (2 + 1 + LASTLITERALS) + (length/255) > oend)) return 0; // Check output limit
+ if (length>=(int)RUN_MASK)
+ {
+ int len = length-RUN_MASK;
+ *token=(RUN_MASK<<ML_BITS);
+ for(; len >= 255 ; len-=255) *op++ = 255;
+ *op++ = (BYTE)len;
+ }
+ else *token = (BYTE)(length<<ML_BITS);
+
+ // Copy Literals
+ { BYTE* end=(op)+(length); LZ4_WILDCOPY(op,anchor,end); op=end; }
+
+_next_match:
+ // Encode Offset
+ LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
+
+ // Start Counting
+ ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified
+ anchor = ip;
+ while likely(ip<matchlimit-(STEPSIZE-1))
+ {
+ size_t diff = AARCH(ref) ^ AARCH(ip);
+ if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
+ ip += LZ4_NbCommonBytes(diff);
+ goto _endCount;
+ }
+ if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
+ if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
+ if ((ip<matchlimit) && (*ref == *ip)) ip++;
+_endCount:
+
+ // Encode MatchLength
+ length = (int)(ip - anchor);
+ if ((limitedOutput) && unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend)) return 0; // Check output limit
+ if (length>=(int)ML_MASK)
+ {
+ *token += ML_MASK;
+ length -= ML_MASK;
+ for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; }
+ if (length >= 255) { length-=255; *op++ = 255; }
+ *op++ = (BYTE)length;
+ }
+ else *token += (BYTE)(length);
+
+ // Test end of chunk
+ if (ip > mflimit) { anchor = ip; break; }
+
+ // Fill table
+ LZ4_putPosition(ip-2, ctx, tableType, base);
+
+ // Test next position
+ ref = LZ4_getPosition(ip, ctx, tableType, base);
+ LZ4_putPosition(ip, ctx, tableType, base);
+ if ((ref + MAX_DISTANCE >= ip) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
+
+ // Prepare next loop
+ anchor = ip++;
+ forwardH = LZ4_hashPosition(ip, tableType);
+ }
+
+_last_literals:
+ // Encode Last Literals
+ {
+ int lastRun = (int)(iend - anchor);
+ if ((limitedOutput) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit
+ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
+ else *op++ = (BYTE)(lastRun<<ML_BITS);
+ memcpy(op, anchor, iend - anchor);
+ op += iend-anchor;
+ }
+
+ // End
+ return (int) (((char*)op)-dest);
+}
+
+
+int LZ4_compress(const char* source, char* dest, int inputSize)
+{
+#if (HEAPMODE)
+ void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries
+#else
+ U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries
+#endif
+ int result;
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noPrefix);
+ else
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);
+
+#if (HEAPMODE)
+ FREEMEM(ctx);
+#endif
+ return result;
+}
+
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+#if (HEAPMODE)
+ void* ctx = ALLOCATOR(HASHNBCELLS4, 4); // Aligned on 4-bytes boundaries
+#else
+ U32 ctx[1U<<(MEMORY_USAGE-2)] = {0}; // Ensure data is aligned on 4-bytes boundaries
+#endif
+ int result;
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, byU16, noPrefix);
+ else
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);
+
+#if (HEAPMODE)
+ FREEMEM(ctx);
+#endif
+ return result;
+}
+
+
+//*****************************
+// Using an external allocation
+//*****************************
+
+int LZ4_sizeofState() { return 1 << MEMORY_USAGE; }
+
+
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
+{
+ if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary
+ MEM_INIT(state, 0, LZ4_sizeofState());
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noPrefix);
+ else
+ return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);
+}
+
+
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ if (((size_t)(state)&3) != 0) return 0; // Error : state is not aligned on 4-bytes boundary
+ MEM_INIT(state, 0, LZ4_sizeofState());
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, byU16, noPrefix);
+ else
+ return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limited, (sizeof(void*)==8) ? byU32 : byPtr, noPrefix);
+}
+
+
+//****************************
+// Stream functions
+//****************************
+
+int LZ4_sizeofStreamState()
+{
+ return sizeof(LZ4_Data_Structure);
+}
+
+FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base)
+{
+ MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable));
+ lz4ds->bufferStart = base;
+ lz4ds->base = base;
+ lz4ds->nextBlock = base;
+}
+
+int LZ4_resetStreamState(void* state, const char* inputBuffer)
+{
+ if ((((size_t)state) & 3) != 0) return 1; // Error : pointer is not aligned on 4-bytes boundary
+ LZ4_init((LZ4_Data_Structure*)state, (const BYTE*)inputBuffer);
+ return 0;
+}
+
+void* LZ4_create (const char* inputBuffer)
+{
+ void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure));
+ LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer);
+ return lz4ds;
+}
+
+
+int LZ4_free (void* LZ4_Data)
+{
+ FREEMEM(LZ4_Data);
+ return (0);
+}
+
+
+char* LZ4_slideInputBuffer (void* LZ4_Data)
+{
+ LZ4_Data_Structure* lz4ds = (LZ4_Data_Structure*)LZ4_Data;
+ size_t delta = lz4ds->nextBlock - (lz4ds->bufferStart + 64 KB);
+
+ if ( (lz4ds->base - delta > lz4ds->base) // underflow control
+ || ((size_t)(lz4ds->nextBlock - lz4ds->base) > 0xE0000000) ) // close to 32-bits limit
+ {
+ size_t deltaLimit = (lz4ds->nextBlock - 64 KB) - lz4ds->base;
+ int nH;
+
+ for (nH=0; nH < HASHNBCELLS4; nH++)
+ {
+ if ((size_t)(lz4ds->hashTable[nH]) < deltaLimit) lz4ds->hashTable[nH] = 0;
+ else lz4ds->hashTable[nH] -= (U32)deltaLimit;
+ }
+ memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
+ lz4ds->base = lz4ds->bufferStart;
+ lz4ds->nextBlock = lz4ds->base + 64 KB;
+ }
+ else
+ {
+ memcpy((void*)(lz4ds->bufferStart), (const void*)(lz4ds->nextBlock - 64 KB), 64 KB);
+ lz4ds->nextBlock -= delta;
+ lz4ds->base -= delta;
+ }
+
+ return (char*)(lz4ds->nextBlock);
+}
+
+
+int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize)
+{
+ return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, 0, notLimited, byU32, withPrefix);
+}
+
+
+int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_compress_generic(LZ4_Data, source, dest, inputSize, maxOutputSize, limited, byU32, withPrefix);
+}
+
+
+//****************************
+// Decompression functions
+//****************************
+
+// This generic decompression function cover all use cases.
+// It shall be instanciated several times, using different sets of directives
+// Note that it is essential this generic function is really inlined,
+// in order to remove useless branches during compilation optimisation.
+FORCE_INLINE int LZ4_decompress_generic(
+ const char* source,
+ char* dest,
+ int inputSize, //
+ int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer.
+
+ int endOnInput, // endOnOutputSize, endOnInputSize
+ int prefix64k, // noPrefix, withPrefix
+ int partialDecoding, // full, partial
+ int targetOutputSize // only used if partialDecoding==partial
+ )
+{
+ // Local Variables
+ const BYTE* restrict ip = (const BYTE*) source;
+ const BYTE* ref;
+ const BYTE* const iend = ip + inputSize;
+
+ BYTE* op = (BYTE*) dest;
+ BYTE* const oend = op + outputSize;
+ BYTE* cpy;
+ BYTE* oexit = op + targetOutputSize;
+
+ const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64
+ static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
+
+
+ // Special cases
+ if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything
+ if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer
+ if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1);
+
+
+ // Main Loop
+ while (1)
+ {
+ unsigned token;
+ size_t length;
+
+ // get runlength
+ token = *ip++;
+ if ((length=(token>>ML_BITS)) == RUN_MASK)
+ {
+ unsigned s=255;
+ while (((endOnInput)?ip<iend:1) && (s==255))
+ {
+ s = *ip++;
+ length += s;
+ }
+ }
+
+ // copy literals
+ cpy = op+length;
+ if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
+ || ((!endOnInput) && (cpy>oend-COPYLENGTH)))
+ {
+ if (partialDecoding)
+ {
+ if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer
+ if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer
+ }
+ else
+ {
+ if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there
+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed
+ }
+ memcpy(op, ip, length);
+ ip += length;
+ op += length;
+ break; // Necessarily EOF, due to parsing restrictions
+ }
+ LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;
+
+ // get offset
+ LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
+ if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer
+
+ // get matchlength
+ if ((length=(token&ML_MASK)) == ML_MASK)
+ {
+ while ((!endOnInput) || (ip<iend-(LASTLITERALS+1))) // Ensure enough bytes remain for LASTLITERALS + token
+ {
+ unsigned s = *ip++;
+ length += s;
+ if (s==255) continue;
+ break;
+ }
+ }
+
+ // copy repeated sequence
+ if unlikely((op-ref)<(int)STEPSIZE)
+ {
+ const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref];
+ op[0] = ref[0];
+ op[1] = ref[1];
+ op[2] = ref[2];
+ op[3] = ref[3];
+ op += 4, ref += 4; ref -= dec32table[op-ref];
+ A32(op) = A32(ref);
+ op += STEPSIZE-4; ref -= dec64;
+ } else { LZ4_COPYSTEP(op,ref); }
+ cpy = op + length - (STEPSIZE-4);
+
+ if unlikely(cpy>oend-COPYLENGTH-(STEPSIZE-4))
+ {
+ if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals
+ LZ4_SECURECOPY(op, ref, (oend-COPYLENGTH));
+ while(op<cpy) *op++=*ref++;
+ op=cpy;
+ continue;
+ }
+ LZ4_WILDCOPY(op, ref, cpy);
+ op=cpy; // correction
+ }
+
+ // end of decoding
+ if (endOnInput)
+ return (int) (((char*)op)-dest); // Nb of output bytes decoded
+ else
+ return (int) (((char*)ip)-source); // Nb of input bytes read
+
+ // Overflow error detected
+_output_error:
+ return (int) (-(((char*)ip)-source))-1;
+}
+
+
+int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, full, 0);
+}
+
+int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, withPrefix, full, 0);
+}
+
+int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize)
+{
+ return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize);
+}
+
+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize)
+{
+ return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
+}
+
+int LZ4_decompress_fast(const char* source, char* dest, int outputSize)
+{
+#ifdef _MSC_VER // This version is faster with Visual
+ return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0);
+#else
+ return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
+#endif
+}
+
-/*\r
- LZ4 - Fast LZ compression algorithm\r
- Header File\r
- Copyright (C) 2011-2013, Yann Collet.\r
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)\r
-\r
- Redistribution and use in source and binary forms, with or without\r
- modification, are permitted provided that the following conditions are\r
- met:\r
-\r
- * Redistributions of source code must retain the above copyright\r
- notice, this list of conditions and the following disclaimer.\r
- * Redistributions in binary form must reproduce the above\r
- copyright notice, this list of conditions and the following disclaimer\r
- in the documentation and/or other materials provided with the\r
- distribution.\r
-\r
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\r
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\r
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\r
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\r
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-#pragma once\r
-\r
-#if defined (__cplusplus)\r
-extern "C" {\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio\r
-# define inline __inline // Visual C is not C99, but supports some kind of inline\r
-#endif\r
-\r
-\r
-//****************************\r
-// Simple Functions\r
-//****************************\r
-\r
-int LZ4_compress (const char* source, char* dest, int inputSize);\r
-int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);\r
-\r
-/*\r
-LZ4_compress() :\r
- Compresses 'inputSize' bytes from 'source' into 'dest'.\r
- Destination buffer must be already allocated,\r
- and must be sized to handle worst cases situations (input data not compressible)\r
- Worst case size evaluation is provided by function LZ4_compressBound()\r
- inputSize : Max supported value is LZ4_MAX_INPUT_VALUE\r
- return : the number of bytes written in buffer dest\r
- or 0 if the compression fails\r
-\r
-LZ4_decompress_safe() :\r
- maxOutputSize : is the size of the destination buffer (which must be already allocated)\r
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)\r
- If the source stream is detected malformed, the function will stop decoding and return a negative result.\r
- This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets\r
-*/\r
-\r
-\r
-//****************************\r
-// Advanced Functions\r
-//****************************\r
-#define LZ4_MAX_INPUT_SIZE 0x7E000000 // 2 113 929 216 bytes\r
-#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)\r
-static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }\r
-\r
-/*\r
-LZ4_compressBound() :\r
- Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)\r
- primarily useful for memory allocation of output buffer.\r
- inline function is recommended for the general case,\r
- macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).\r
-\r
- isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE\r
- return : maximum output size in a "worst case" scenario\r
- or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)\r
-*/\r
-\r
-\r
-int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);\r
-\r
-/*\r
-LZ4_compress_limitedOutput() :\r
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.\r
- If it cannot achieve it, compression will stop, and result of the function will be zero.\r
- This function never writes outside of provided output buffer.\r
-\r
- inputSize : Max supported value is LZ4_MAX_INPUT_VALUE\r
- maxOutputSize : is the size of the destination buffer (which must be already allocated)\r
- return : the number of bytes written in buffer 'dest'\r
- or 0 if the compression fails\r
-*/\r
-\r
-\r
-int LZ4_decompress_fast (const char* source, char* dest, int outputSize);\r
-\r
-/*\r
-LZ4_decompress_fast() :\r
- outputSize : is the original (uncompressed) size\r
- return : the number of bytes read from the source buffer (in other words, the compressed size)\r
- If the source stream is malformed, the function will stop decoding and return a negative result.\r
- note : This function is a bit faster than LZ4_decompress_safe()\r
- This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.\r
- Use this function preferably into a trusted environment (data to decode comes from a trusted source).\r
- Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.\r
-*/\r
-\r
-int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);\r
-\r
-/*\r
-LZ4_decompress_safe_partial() :\r
- This function decompress a compressed block of size 'inputSize' at position 'source'\r
- into output buffer 'dest' of size 'maxOutputSize'.\r
- The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,\r
- reducing decompression time.\r
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)\r
- Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.\r
- Always control how many bytes were decoded.\r
- If the source stream is detected malformed, the function will stop decoding and return a negative result.\r
- This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets\r
-*/\r
-\r
-\r
-//****************************\r
-// Stream Functions\r
-//****************************\r
-\r
-void* LZ4_create (const char* inputBuffer);\r
-int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);\r
-int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);\r
-char* LZ4_slideInputBuffer (void* LZ4_Data);\r
-int LZ4_free (void* LZ4_Data);\r
-\r
-/* \r
-These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.\r
-In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :\r
-\r
-void* LZ4_create (const char* inputBuffer);\r
-The result of the function is the (void*) pointer on the LZ4 Data Structure.\r
-This pointer will be needed in all other functions.\r
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.\r
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.\r
-The input buffer must be already allocated, and size at least 192KB.\r
-'inputBuffer' will also be the 'const char* source' of the first block.\r
-\r
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.\r
-To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().\r
-Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(), \r
-but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.\r
-If next block does not begin immediately after the previous one, the compression will fail (return 0).\r
-\r
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : \r
-char* LZ4_slideInputBuffer(void* LZ4_Data);\r
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.\r
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.\r
-==> The memory position where the next input data block must start is provided as the result of the function.\r
-\r
-Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.\r
-\r
-When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.\r
-*/\r
-\r
-\r
-int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);\r
-int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);\r
-\r
-/*\r
-*_withPrefix64k() :\r
- These decoding functions work the same as their "normal name" versions,\r
- but can use up to 64KB of data in front of 'char* dest'.\r
- These functions are necessary to decode inter-dependant blocks.\r
-*/\r
-\r
-\r
-//****************************\r
-// Obsolete Functions\r
-//****************************\r
-\r
-static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }\r
-static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }\r
-\r
-/*\r
-These functions are deprecated and should no longer be used.\r
-They are provided here for compatibility with existing user programs.\r
-*/\r
-\r
-\r
-\r
-#if defined (__cplusplus)\r
-}\r
-#endif\r
+/*
+ LZ4 - Fast LZ compression algorithm
+ Header File
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+//**************************************
+// Compiler Options
+//**************************************
+#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio
+# define inline __inline // Visual C is not C99, but supports some kind of inline
+#endif
+
+
+//****************************
+// Simple Functions
+//****************************
+
+int LZ4_compress (const char* source, char* dest, int inputSize);
+int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+LZ4_compress() :
+ Compresses 'inputSize' bytes from 'source' into 'dest'.
+ Destination buffer must be already allocated,
+ and must be sized to handle worst cases situations (input data not compressible)
+ Worst case size evaluation is provided by function LZ4_compressBound()
+ inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ return : the number of bytes written in buffer dest
+ or 0 if the compression fails
+
+LZ4_decompress_safe() :
+ maxOutputSize : is the size of the destination buffer (which must be already allocated)
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
+*/
+
+
+//****************************
+// Advanced Functions
+//****************************
+#define LZ4_MAX_INPUT_SIZE 0x7E000000 // 2 113 929 216 bytes
+#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
+static inline int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
+
+/*
+LZ4_compressBound() :
+ Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
+ primarily useful for memory allocation of output buffer.
+ inline function is recommended for the general case,
+ macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation).
+
+ isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
+ return : maximum output size in a "worst case" scenario
+ or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+*/
+
+
+int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+LZ4_compress_limitedOutput() :
+ Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This function never writes outside of provided output buffer.
+
+ inputSize : Max supported value is LZ4_MAX_INPUT_VALUE
+ maxOutputSize : is the size of the destination buffer (which must be already allocated)
+ return : the number of bytes written in buffer 'dest'
+ or 0 if the compression fails
+*/
+
+
+int LZ4_decompress_fast (const char* source, char* dest, int outputSize);
+
+/*
+LZ4_decompress_fast() :
+ outputSize : is the original (uncompressed) size
+ return : the number of bytes read from the source buffer (in other words, the compressed size)
+ If the source stream is malformed, the function will stop decoding and return a negative result.
+ note : This function is a bit faster than LZ4_decompress_safe()
+ This function never writes outside of output buffers, but may read beyond input buffer in case of malicious data packet.
+ Use this function preferably into a trusted environment (data to decode comes from a trusted source).
+ Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
+*/
+
+int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
+
+/*
+LZ4_decompress_safe_partial() :
+ This function decompress a compressed block of size 'inputSize' at position 'source'
+ into output buffer 'dest' of size 'maxOutputSize'.
+ The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+ reducing decompression time.
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+ Always control how many bytes were decoded.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+
+
+//*****************************
+// Using an external allocation
+//*****************************
+int LZ4_sizeofState();
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofState();
+
+Note that tables must be aligned on 4-bytes boundaries, otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withState() and LZ4_compress_limitedOutput_withState() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+//****************************
+// Streaming Functions
+//****************************
+
+void* LZ4_create (const char* inputBuffer);
+int LZ4_compress_continue (void* LZ4_Data, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBuffer (void* LZ4_Data);
+int LZ4_free (void* LZ4_Data);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4 Data Structure, thanks to the function :
+
+void* LZ4_create (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4 Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compress_continue() or LZ4_compress_limitedOutput_continue().
+Their behavior are identical to LZ4_compress() or LZ4_compress_limitedOutput(),
+but require the LZ4 Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBuffer(void* LZ4_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compress_continue() or LZ4_compress_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_free() will release the memory used by the LZ4 Data Structure.
+*/
+
+int LZ4_sizeofStreamState();
+int LZ4_resetStreamState(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_create (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamState();
+Note also that space must be 4-bytes aligned.
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamState(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned on 4-bytes boundaries, and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamState() must be 0 is OK.
+Any other value means there was an error (typically, pointer is not aligned on 4-bytes boundaries).
+*/
+
+
+int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
+
+/*
+*_withPrefix64k() :
+ These decoding functions work the same as their "normal name" versions,
+ but can use up to 64KB of data in front of 'char* dest'.
+ These functions are necessary to decode inter-dependant blocks.
+*/
+
+
+//****************************
+// Obsolete Functions
+//****************************
+
+static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
+static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+
+/*
+These functions are deprecated and should no longer be used.
+They are provided here for compatibility with existing user programs.
+*/
+
+
+
+#if defined (__cplusplus)
+}
+#endif
-/*\r
- LZ4cli.c - LZ4 Command Line Interface\r
- Copyright (C) Yann Collet 2011-2013\r
- GPL v2 License\r
-\r
- This program is free software; you can redistribute it and/or modify\r
- it under the terms of the GNU General Public License as published by\r
- the Free Software Foundation; either version 2 of the License, or\r
- (at your option) any later version.\r
-\r
- This program is distributed in the hope that it will be useful,\r
- but WITHOUT ANY WARRANTY; without even the implied warranty of\r
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
- GNU General Public License for more details.\r
-\r
- You should have received a copy of the GNU General Public License along\r
- with this program; if not, write to the Free Software Foundation, Inc.,\r
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\r
-\r
- You can contact the author at :\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c\r
-*/\r
-/*\r
- Note : this is stand-alone program.\r
- It is not part of LZ4 compression library, it is a user program of the LZ4 library.\r
- The license of LZ4 library is BSD.\r
- The license of xxHash library is BSD.\r
- The license of this compression CLI program is GPLv2.\r
-*/\r
-\r
-//**************************************\r
-// Tuning parameters\r
-//**************************************\r
-// DISABLE_LZ4C_LEGACY_OPTIONS :\r
-// Control the availability of -c0, -c1 and -hc legacy arguments\r
-// Default : Legacy options are enabled\r
-// #define DISABLE_LZ4C_LEGACY_OPTIONS\r
-\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-// Disable some Visual warning messages\r
-#ifdef _MSC_VER // Visual Studio\r
-# define _CRT_SECURE_NO_WARNINGS\r
-# define _CRT_SECURE_NO_DEPRECATE // VS2005\r
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant\r
-#endif\r
-\r
-#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix\r
-#define _POSIX_SOURCE 1 // for fileno() within <stdio.h> on unix\r
-\r
-\r
-//****************************\r
-// Includes\r
-//****************************\r
-#include <stdio.h> // fprintf, fopen, fread, _fileno, stdin, stdout\r
-#include <stdlib.h> // malloc\r
-#include <string.h> // strcmp, strlen\r
-#include <time.h> // clock\r
-#include "lz4.h"\r
-#include "lz4hc.h"\r
-#include "xxhash.h"\r
-#include "bench.h"\r
-\r
-\r
-//****************************\r
-// OS-specific Includes\r
-//****************************\r
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)\r
-# include <fcntl.h> // _O_BINARY\r
-# include <io.h> // _setmode, _isatty\r
-# ifdef __MINGW32__\r
- int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h>\r
-# endif\r
-# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)\r
-# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))\r
-#else\r
-# include <unistd.h> // isatty\r
-# define SET_BINARY_MODE(file)\r
-# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Compiler-specific functions\r
-//**************************************\r
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)\r
-\r
-#if defined(_MSC_VER) // Visual Studio\r
-# define swap32 _byteswap_ulong\r
-#elif GCC_VERSION >= 403\r
-# define swap32 __builtin_bswap32\r
-#else\r
- static inline unsigned int swap32(unsigned int x) \r
- {\r
- return ((x << 24) & 0xff000000 ) |\r
- ((x << 8) & 0x00ff0000 ) |\r
- ((x >> 8) & 0x0000ff00 ) |\r
- ((x >> 24) & 0x000000ff );\r
- }\r
-#endif\r
-\r
-\r
-//****************************\r
-// Constants\r
-//****************************\r
-#define COMPRESSOR_NAME "LZ4 Compression CLI"\r
-#ifndef LZ4_VERSION\r
-# define LZ4_VERSION "v1.0.9"\r
-#endif\r
-#define AUTHOR "Yann Collet"\r
-#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__\r
-#define LZ4_EXTENSION ".lz4"\r
-\r
-#define KB *(1U<<10)\r
-#define MB *(1U<<20)\r
-#define GB *(1U<<30)\r
-\r
-#define _1BIT 0x01\r
-#define _2BITS 0x03\r
-#define _3BITS 0x07\r
-#define _4BITS 0x0F\r
-#define _8BITS 0xFF\r
-\r
-#define MAGICNUMBER_SIZE 4\r
-#define LZ4S_MAGICNUMBER 0x184D2204\r
-#define LZ4S_SKIPPABLE0 0x184D2A50\r
-#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0\r
-#define LEGACY_MAGICNUMBER 0x184C2102\r
-\r
-#define CACHELINE 64\r
-#define LEGACY_BLOCKSIZE (8 MB)\r
-#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)\r
-#define LZ4S_BLOCKSIZEID_DEFAULT 7\r
-#define LZ4S_CHECKSUM_SEED 0\r
-#define LZ4S_EOS 0\r
-#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)\r
-\r
-\r
-//**************************************\r
-// Architecture Macros\r
-//**************************************\r
-static const int one = 1;\r
-#define CPU_LITTLE_ENDIAN (*(char*)(&one))\r
-#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)\r
-#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i))\r
-\r
-\r
-//**************************************\r
-// Macros\r
-//**************************************\r
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)\r
-#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }\r
-\r
-\r
-//**************************************\r
-// Special input/output\r
-//**************************************\r
-#define NULL_OUTPUT "null"\r
-char stdinmark[] = "stdin";\r
-char stdoutmark[] = "stdout";\r
-#ifdef _WIN32\r
-char nulmark[] = "nul";\r
-#else\r
-char nulmark[] = "/dev/null";\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Local Parameters\r
-//**************************************\r
-static char* programName;\r
-static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information\r
-static int overwrite = 0;\r
-static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;\r
-static int blockChecksum = 0;\r
-static int streamChecksum = 1;\r
-static int blockIndependence = 1;\r
-\r
-\r
-//**************************************\r
-// Exceptions\r
-//**************************************\r
-#define DEBUG 0\r
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);\r
-#define EXM_THROW(error, ...) \\r
-{ \\r
- DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \\r
- DISPLAYLEVEL(1, "Error %i : ", error); \\r
- DISPLAYLEVEL(1, __VA_ARGS__); \\r
- DISPLAYLEVEL(1, "\n"); \\r
- exit(error); \\r
-}\r
-\r
-\r
-//**************************************\r
-// Version modifiers\r
-//**************************************\r
-#define EXTENDED_ARGUMENTS\r
-#define EXTENDED_HELP\r
-#define EXTENDED_FORMAT\r
-#define DEFAULT_COMPRESSOR compress_file\r
-#define DEFAULT_DECOMPRESSOR decodeLZ4S\r
-\r
-\r
-//****************************\r
-// Functions\r
-//****************************\r
-int usage()\r
-{\r
- DISPLAY( "Usage :\n");\r
- DISPLAY( " %s [arg] [input] [output]\n", programName);\r
- DISPLAY( "\n");\r
- DISPLAY( "input : a filename\n");\r
- DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);\r
- DISPLAY( "Arguments :\n");\r
- DISPLAY( " -1 : Fast compression (default) \n");\r
- DISPLAY( " -9 : High compression \n");\r
- DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION);\r
- DISPLAY( " -z : force compression\n");\r
- DISPLAY( " -f : overwrite output without prompting \n");\r
- DISPLAY( " -h/-H : display help/long help and exit\n");\r
- return 0;\r
-}\r
-\r
-int usage_advanced()\r
-{\r
- DISPLAY(WELCOME_MESSAGE);\r
- usage();\r
- DISPLAY( "\n");\r
- DISPLAY( "Advanced arguments :\n");\r
- DISPLAY( " -V : display Version number and exit\n");\r
- DISPLAY( " -v : verbose mode\n");\r
- DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");\r
- DISPLAY( " -c : force write to standard output, even if it is the console\n");\r
- DISPLAY( " -t : test compressed file integrity\n");\r
- DISPLAY( " -B# : Block size [4-7](default : 7)\n");\r
- DISPLAY( " -BD : Block dependency (improve compression ratio)\n");\r
- DISPLAY( " -BX : enable block checksum (default:disabled)\n");\r
- DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");\r
- DISPLAY( "Benchmark arguments :\n");\r
- DISPLAY( " -b : benchmark file(s)\n");\r
- DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");\r
-#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)\r
- DISPLAY( "Legacy arguments :\n");\r
- DISPLAY( " -c0 : fast compression\n");\r
- DISPLAY( " -c1 : high compression\n");\r
- DISPLAY( " -hc : high compression\n");\r
- DISPLAY( " -y : overwrite output without prompting \n");\r
- DISPLAY( " -s : suppress warnings \n");\r
-#endif // DISABLE_LZ4C_LEGACY_OPTIONS\r
- EXTENDED_HELP;\r
- return 0;\r
-}\r
-\r
-int usage_longhelp()\r
-{\r
- DISPLAY( "\n");\r
- DISPLAY( "Which values can get [output] ? \n");\r
- DISPLAY( "[output] : a filename\n");\r
- DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark);\r
- DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT);\r
- DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n");\r
- DISPLAY( " - if stdout is not the console, then [output] = stdout \n");\r
- DISPLAY( " - if stdout is console : \n");\r
- DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION);\r
- DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION);\r
- DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION);\r
- DISPLAY( "\n");\r
- DISPLAY( "Compression levels : \n");\r
- DISPLAY( "There are technically 2 accessible compression levels.\n");\r
- DISPLAY( "-0 ... -2 => Fast compression\n");\r
- DISPLAY( "-3 ... -9 => High compression\n");\r
- DISPLAY( "\n");\r
- DISPLAY( "stdin, stdout and the console : \n");\r
- DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n");\r
- DISPLAY( "%s will refuse to read from console, or write to console \n", programName);\r
- DISPLAY( "except if '-c' command is specified, to force output to console \n");\r
- DISPLAY( "\n");\r
- DISPLAY( "Simple example :\n");\r
- DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n");\r
- DISPLAY( " %s filename\n", programName);\r
- DISPLAY( "\n");\r
- DISPLAY( "Arguments can be appended together, or provided independently. For example :\n");\r
- DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n");\r
- DISPLAY( " %s -f9 filename \n", programName);\r
- DISPLAY( " is equivalent to :\n");\r
- DISPLAY( " %s -f -9 filename \n", programName);\r
- DISPLAY( "\n");\r
- DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName);\r
- DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");\r
- DISPLAY( " generator | %s | consumer \n", programName);\r
-#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)\r
- DISPLAY( "\n");\r
- DISPLAY( "Warning :\n");\r
- DISPLAY( "Legacy arguments take precedence. Therefore : \n");\r
- DISPLAY( " %s -hc filename\n", programName);\r
- DISPLAY( "means 'compress filename in high compression mode'\n");\r
- DISPLAY( "It is not equivalent to :\n");\r
- DISPLAY( " %s -h -c filename\n", programName);\r
- DISPLAY( "which would display help text and exit\n");\r
-#endif // DISABLE_LZ4C_LEGACY_OPTIONS\r
- return 0;\r
-}\r
-\r
-int badusage()\r
-{\r
- DISPLAYLEVEL(1, "Incorrect parameters\n");\r
- if (displayLevel >= 1) usage();\r
- exit(1);\r
-}\r
-\r
-\r
-static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }\r
-static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }\r
-static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }\r
-\r
-\r
-int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)\r
-{\r
-\r
- if (!strcmp (input_filename, stdinmark)) \r
- {\r
- DISPLAYLEVEL(4,"Using stdin for input\n");\r
- *pfinput = stdin;\r
- SET_BINARY_MODE(stdin);\r
- } \r
- else \r
- {\r
- *pfinput = fopen(input_filename, "rb");\r
- }\r
-\r
- if (!strcmp (output_filename, stdoutmark)) \r
- {\r
- DISPLAYLEVEL(4,"Using stdout for output\n");\r
- *pfoutput = stdout;\r
- SET_BINARY_MODE(stdout);\r
- } \r
- else \r
- {\r
- // Check if destination file already exists\r
- *pfoutput=0;\r
- if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );\r
- if (*pfoutput!=0) \r
- { \r
- fclose(*pfoutput); \r
- if (!overwrite)\r
- {\r
- char ch;\r
- DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);\r
- DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");\r
- if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible\r
- ch = (char)getchar();\r
- if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);\r
- }\r
- }\r
- *pfoutput = fopen( output_filename, "wb" );\r
- }\r
-\r
- if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);\r
- if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); \r
-\r
- return 0;\r
-}\r
-\r
-\r
-\r
-int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel)\r
-{\r
- int (*compressionFunction)(const char*, char*, int);\r
- unsigned long long filesize = 0;\r
- unsigned long long compressedfilesize = MAGICNUMBER_SIZE;\r
- char* in_buff;\r
- char* out_buff;\r
- FILE* finput;\r
- FILE* foutput;\r
- int displayLevel = (compressionlevel>0);\r
- clock_t start, end;\r
- size_t sizeCheck;\r
-\r
-\r
- // Init\r
- if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;\r
- start = clock();\r
- get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
- if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;\r
-\r
- // Allocate Memory\r
- in_buff = (char*)malloc(LEGACY_BLOCKSIZE);\r
- out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));\r
- if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");\r
-\r
- // Write Archive Header\r
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER);\r
- sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);\r
- if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");\r
-\r
- // Main Loop\r
- while (1)\r
- {\r
- unsigned int outSize;\r
- // Read Block\r
- int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);\r
- if( inSize<=0 ) break;\r
- filesize += inSize;\r
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));\r
-\r
- // Compress Block\r
- outSize = compressionFunction(in_buff, out_buff+4, inSize);\r
- compressedfilesize += outSize+4;\r
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
-\r
- // Write Block\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
- sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);\r
- if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");\r
- }\r
-\r
- // Status\r
- end = clock();\r
- DISPLAYLEVEL(2, "\r%79s\r", "");\r
- DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
- {\r
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
- DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
- }\r
-\r
- // Close & Free\r
- free(in_buff);\r
- free(out_buff);\r
- fclose(finput);\r
- fclose(foutput);\r
-\r
- return 0;\r
-}\r
-\r
-\r
-int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)\r
-{\r
- void* (*initFunction) (const char*);\r
- int (*compressionFunction)(void*, const char*, char*, int, int);\r
- char* (*translateFunction) (void*);\r
- int (*freeFunction) (void*);\r
- void* ctx;\r
- unsigned long long filesize = 0;\r
- unsigned long long compressedfilesize = 0;\r
- unsigned int checkbits;\r
- char* in_buff, *in_start, *in_end;\r
- char* out_buff;\r
- FILE* finput;\r
- FILE* foutput;\r
- clock_t start, end;\r
- unsigned int blockSize, inputBufferSize;\r
- size_t sizeCheck, header_size;\r
- void* streamChecksumState=NULL;\r
-\r
-\r
- // Init\r
- start = clock();\r
- if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;\r
- if (compressionlevel>=3)\r
- {\r
- initFunction = LZ4_createHC;\r
- compressionFunction = LZ4_compressHC_limitedOutput_continue;\r
- translateFunction = LZ4_slideInputBufferHC;\r
- freeFunction = LZ4_freeHC;\r
- }\r
- else\r
- {\r
- initFunction = LZ4_create;\r
- compressionFunction = LZ4_compress_limitedOutput_continue;\r
- translateFunction = LZ4_slideInputBuffer;\r
- freeFunction = LZ4_free;\r
- }\r
- get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
- blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);\r
-\r
- // Allocate Memory\r
- inputBufferSize = blockSize + 64 KB;\r
- if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;\r
- in_buff = (char*)malloc(inputBufferSize);\r
- out_buff = (char*)malloc(blockSize+CACHELINE);\r
- if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");\r
- in_start = in_buff; in_end = in_buff + inputBufferSize;\r
- if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
- ctx = initFunction(in_buff);\r
-\r
- // Write Archive Header\r
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention\r
- *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')\r
- *(out_buff+4) |= (blockIndependence & _1BIT) << 5;\r
- *(out_buff+4) |= (blockChecksum & _1BIT) << 4;\r
- *(out_buff+4) |= (streamChecksum & _1BIT) << 2;\r
- *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);\r
- checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);\r
- checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);\r
- *(out_buff+6) = (unsigned char) checkbits;\r
- header_size = 7;\r
- sizeCheck = fwrite(out_buff, 1, header_size, foutput);\r
- if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");\r
- compressedfilesize += header_size;\r
-\r
- // Main Loop\r
- while (1)\r
- {\r
- unsigned int outSize;\r
- unsigned int inSize;\r
- // Read Block\r
- if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);\r
- inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);\r
- if( inSize==0 ) break; // No more input : end of compression\r
- filesize += inSize;\r
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));\r
- if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);\r
-\r
- // Compress Block\r
- outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);\r
- if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;\r
- if (blockChecksum) compressedfilesize+=4;\r
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
-\r
- // Write Block\r
- if (outSize > 0)\r
- {\r
- int sizeToWrite;\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
- if (blockChecksum)\r
- {\r
- unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);\r
- * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);\r
- }\r
- sizeToWrite = 4 + outSize + (4*blockChecksum);\r
- sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);\r
- if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");\r
-\r
- }\r
- else // Copy Original\r
- {\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");\r
- sizeCheck = fwrite(in_start, 1, inSize, foutput);\r
- if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");\r
- if (blockChecksum)\r
- {\r
- unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");\r
- }\r
- }\r
- in_start += inSize;\r
- }\r
-\r
- // End of Stream mark\r
- * (unsigned int*) out_buff = LZ4S_EOS;\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");\r
- compressedfilesize += 4;\r
- if (streamChecksum)\r
- {\r
- unsigned int checksum = XXH32_digest(streamChecksumState);\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");\r
- compressedfilesize += 4;\r
- }\r
-\r
- // Status\r
- end = clock();\r
- DISPLAYLEVEL(2, "\r%79s\r", "");\r
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
- {\r
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
- }\r
-\r
- // Close & Free\r
- freeFunction(ctx);\r
- free(in_buff);\r
- free(out_buff);\r
- fclose(finput);\r
- fclose(foutput);\r
-\r
- return 0;\r
-}\r
-\r
-\r
-int compress_file(char* input_filename, char* output_filename, int compressionlevel)\r
-{\r
- int (*compressionFunction)(const char*, char*, int, int);\r
- unsigned long long filesize = 0;\r
- unsigned long long compressedfilesize = 0;\r
- unsigned int checkbits;\r
- char* in_buff;\r
- char* out_buff;\r
- char* headerBuffer;\r
- FILE* finput;\r
- FILE* foutput;\r
- clock_t start, end;\r
- int blockSize;\r
- size_t sizeCheck, header_size, readSize;\r
- void* streamChecksumState=NULL;\r
-\r
- // Branch out\r
- if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel);\r
-\r
- // Init\r
- start = clock();\r
- if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;\r
- if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput;\r
- get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
- blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);\r
-\r
- // Allocate Memory\r
- in_buff = (char*)malloc(blockSize);\r
- out_buff = (char*)malloc(blockSize+CACHELINE);\r
- headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE);\r
- if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory");\r
- if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
-\r
- // Write Archive Header\r
- *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention\r
- *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01')\r
- *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5;\r
- *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4;\r
- *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2;\r
- *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4);\r
- checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED);\r
- checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);\r
- *(headerBuffer+6) = (unsigned char) checkbits;\r
- header_size = 7;\r
-\r
- // Write header\r
- sizeCheck = fwrite(headerBuffer, 1, header_size, foutput);\r
- if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");\r
- compressedfilesize += header_size;\r
-\r
- // read first block\r
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);\r
-\r
- // Main Loop\r
- while (readSize>0)\r
- {\r
- unsigned int outSize;\r
-\r
- filesize += readSize;\r
- DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));\r
- if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize);\r
-\r
- // Compress Block\r
- outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1);\r
- if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;\r
- if (blockChecksum) compressedfilesize+=4;\r
- DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
-\r
- // Write Block\r
- if (outSize > 0)\r
- {\r
- int sizeToWrite;\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
- if (blockChecksum)\r
- {\r
- unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);\r
- * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);\r
- }\r
- sizeToWrite = 4 + outSize + (4*blockChecksum);\r
- sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);\r
- if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");\r
- }\r
- else // Copy Original Uncompressed\r
- {\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");\r
- sizeCheck = fwrite(in_buff, 1, readSize, foutput);\r
- if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block");\r
- if (blockChecksum)\r
- {\r
- unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED);\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");\r
- }\r
- }\r
-\r
- // Read next block\r
- readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);\r
- }\r
-\r
- // End of Stream mark\r
- * (unsigned int*) out_buff = LZ4S_EOS;\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");\r
- compressedfilesize += 4;\r
- if (streamChecksum)\r
- {\r
- unsigned int checksum = XXH32_digest(streamChecksumState);\r
- * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
- sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
- if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");\r
- compressedfilesize += 4;\r
- }\r
-\r
- // Close & Free\r
- free(in_buff);\r
- free(out_buff);\r
- free(headerBuffer);\r
- fclose(finput);\r
- fclose(foutput);\r
-\r
- // Final Status\r
- end = clock();\r
- DISPLAYLEVEL(2, "\r%79s\r", "");\r
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
- (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
- {\r
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
- }\r
-\r
- return 0;\r
-}\r
-\r
-\r
-unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)\r
-{\r
- unsigned long long filesize = 0;\r
- char* in_buff;\r
- char* out_buff;\r
- unsigned int blockSize;\r
-\r
-\r
- // Allocate Memory\r
- in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));\r
- out_buff = (char*)malloc(LEGACY_BLOCKSIZE);\r
- if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");\r
-\r
- // Main Loop\r
- while (1)\r
- {\r
- int decodeSize;\r
- size_t sizeCheck;\r
-\r
- // Block Size\r
- sizeCheck = fread(&blockSize, 1, 4, finput);\r
- if (sizeCheck==0) break; // Nothing to read : file read is completed\r
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian\r
- if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) \r
- { // Cannot read next block : maybe new stream ?\r
- fseek(finput, -4, SEEK_CUR);\r
- break;\r
- }\r
-\r
- // Read Block\r
- sizeCheck = fread(in_buff, 1, blockSize, finput);\r
-\r
- // Decode Block\r
- decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);\r
- if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");\r
- filesize += decodeSize;\r
-\r
- // Write Block\r
- sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);\r
- if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");\r
- }\r
-\r
- // Free\r
- free(in_buff);\r
- free(out_buff);\r
-\r
- return filesize;\r
-}\r
-\r
-\r
-unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)\r
-{\r
- unsigned long long filesize = 0;\r
- char* in_buff;\r
- char* out_buff, *out_start, *out_end;\r
- unsigned char descriptor[LZ4S_MAXHEADERSIZE];\r
- size_t nbReadBytes;\r
- int decodedBytes=0;\r
- unsigned int maxBlockSize;\r
- size_t sizeCheck;\r
- int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;\r
- void* streamChecksumState=NULL;\r
- int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;\r
- unsigned int prefix64k = 0;\r
-\r
- // Decode stream descriptor\r
- nbReadBytes = fread(descriptor, 1, 3, finput);\r
- if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");\r
- {\r
- int version = (descriptor[0] >> 6) & _2BITS;\r
- int streamSize = (descriptor[0] >> 3) & _1BIT;\r
- int reserved1 = (descriptor[0] >> 1) & _1BIT;\r
- int dictionary = (descriptor[0] >> 0) & _1BIT;\r
-\r
- int reserved2 = (descriptor[1] >> 7) & _1BIT;\r
- int blockSizeId = (descriptor[1] >> 4) & _3BITS;\r
- int reserved3 = (descriptor[1] >> 0) & _4BITS;\r
- int checkBits = (descriptor[2] >> 0) & _8BITS;\r
- int checkBits_xxh32;\r
-\r
- blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;\r
- blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;\r
- streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;\r
-\r
- if (version != 1) EXM_THROW(62, "Wrong version number");\r
- if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); \r
- if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");\r
- if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); \r
- if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");\r
- if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); \r
- if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");\r
- maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);\r
- // Checkbits verification\r
- descriptor[1] &= 0xF0;\r
- checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);\r
- checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32);\r
- if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");\r
- }\r
-\r
- if (!blockIndependenceFlag)\r
- {\r
- decompressionFunction = LZ4_decompress_safe_withPrefix64k;\r
- prefix64k = 64 KB;\r
- }\r
-\r
- // Allocate Memory\r
- {\r
- unsigned int outbuffSize = prefix64k+maxBlockSize;\r
- in_buff = (char*)malloc(maxBlockSize);\r
- if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;\r
- out_buff = (char*)malloc(outbuffSize); \r
- out_end = out_buff + outbuffSize;\r
- out_start = out_buff + prefix64k;\r
- if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");\r
- }\r
- if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
-\r
- // Main Loop\r
- while (1)\r
- {\r
- unsigned int blockSize, uncompressedFlag;\r
-\r
- // Block Size\r
- nbReadBytes = fread(&blockSize, 1, 4, finput);\r
- if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");\r
- if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed\r
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian\r
- uncompressedFlag = blockSize >> 31;\r
- blockSize &= 0x7FFFFFFF;\r
- if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size");\r
-\r
- // Read Block\r
- nbReadBytes = fread(in_buff, 1, blockSize, finput);\r
- if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" );\r
-\r
- // Check Block\r
- if (blockChecksumFlag)\r
- {\r
- unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);\r
- unsigned int readChecksum;\r
- sizeCheck = fread(&readChecksum, 1, 4, finput);\r
- if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");\r
- readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian\r
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");\r
- }\r
-\r
- if (uncompressedFlag)\r
- {\r
- // Write uncompressed Block\r
- sizeCheck = fwrite(in_buff, 1, blockSize, foutput);\r
- if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");\r
- filesize += blockSize;\r
- if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);\r
- if (!blockIndependenceFlag)\r
- {\r
- if (blockSize >= prefix64k)\r
- {\r
- memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks\r
- out_start = out_buff + prefix64k;\r
- continue;\r
- }\r
- else\r
- {\r
- memcpy(out_start, in_buff, blockSize);\r
- decodedBytes = blockSize;\r
- }\r
- }\r
- }\r
- else\r
- {\r
- // Decode Block\r
- decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);\r
- if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");\r
- filesize += decodedBytes;\r
- if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);\r
-\r
- // Write Block\r
- sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);\r
- if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");\r
- }\r
-\r
- if (!blockIndependenceFlag)\r
- {\r
- out_start += decodedBytes;\r
- if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) \r
- {\r
- memcpy(out_buff, out_start - prefix64k, prefix64k); \r
- out_start = out_buff + prefix64k; \r
- }\r
- }\r
- }\r
-\r
- // Stream Checksum\r
- if (streamChecksumFlag)\r
- {\r
- unsigned int checksum = XXH32_digest(streamChecksumState);\r
- unsigned int readChecksum;\r
- sizeCheck = fread(&readChecksum, 1, 4, finput);\r
- if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");\r
- readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian\r
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");\r
- }\r
-\r
- // Free\r
- free(in_buff);\r
- free(out_buff);\r
-\r
- return filesize;\r
-}\r
-\r
-\r
-unsigned long long selectDecoder( FILE* finput, FILE* foutput)\r
-{\r
- unsigned int magicNumber, size;\r
- int errorNb;\r
- size_t nbReadBytes;\r
-\r
- // Check Archive Header\r
- nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);\r
- if (nbReadBytes==0) return 0; // EOF\r
- if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");\r
- magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format\r
- if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers\r
-\r
- switch(magicNumber)\r
- {\r
- case LZ4S_MAGICNUMBER:\r
- return DEFAULT_DECOMPRESSOR(finput, foutput);\r
- case LEGACY_MAGICNUMBER:\r
- DISPLAYLEVEL(4, "Detected : Legacy format \n");\r
- return decodeLegacyStream(finput, foutput);\r
- case LZ4S_SKIPPABLE0:\r
- DISPLAYLEVEL(4, "Skipping detected skippable area \n");\r
- nbReadBytes = fread(&size, 1, 4, finput);\r
- if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");\r
- size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format\r
- errorNb = fseek(finput, size, SEEK_CUR);\r
- if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");\r
- return selectDecoder(finput, foutput);\r
- EXTENDED_FORMAT;\r
- default:\r
- if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream\r
- DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");\r
- return 0;\r
- }\r
-}\r
-\r
-\r
-int decodeFile(char* input_filename, char* output_filename)\r
-{\r
- unsigned long long filesize = 0, decodedSize=0;\r
- FILE* finput;\r
- FILE* foutput;\r
- clock_t start, end;\r
-\r
-\r
- // Init\r
- start = clock();\r
- get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
-\r
- // Loop over multiple streams\r
- do \r
- {\r
- decodedSize = selectDecoder(finput, foutput);\r
- filesize += decodedSize;\r
- } while (decodedSize);\r
-\r
- // Final Status\r
- end = clock();\r
- DISPLAYLEVEL(2, "\r%79s\r", "");\r
- DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);\r
- {\r
- double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
- DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
- }\r
-\r
- // Close\r
- fclose(finput);\r
- fclose(foutput);\r
-\r
- // Error status = OK\r
- return 0;\r
-}\r
-\r
-\r
-void waitEnter()\r
-{\r
- DISPLAY("Press enter to continue...\n");\r
- getchar();\r
-}\r
-\r
-\r
-int main(int argc, char** argv)\r
-{\r
- int i,\r
- cLevel=0,\r
- decode=0,\r
- bench=0,\r
- filenamesStart=2,\r
- legacy_format=0,\r
- forceStdout=0,\r
- forceCompress=0,\r
- pause=0;\r
- char* input_filename=0;\r
- char* output_filename=0;\r
- char nullOutput[] = NULL_OUTPUT;\r
- char extension[] = LZ4_EXTENSION;\r
-\r
- // Init\r
- programName = argv[0];\r
-\r
- for(i=1; i<argc; i++)\r
- {\r
- char* argument = argv[i];\r
-\r
- if(!argument) continue; // Protection if argument empty\r
-\r
- // Decode command (note : aggregated commands are allowed)\r
- if (argument[0]=='-')\r
- {\r
- // '-' means stdin/stdout\r
- if (argument[1]==0)\r
- {\r
- if (!input_filename) input_filename=stdinmark;\r
- else output_filename=stdoutmark;\r
- }\r
-\r
- while (argument[1]!=0)\r
- {\r
- argument ++;\r
-\r
-#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)\r
- // Legacy options (-c0, -c1, -hc, -y, -s)\r
- if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression)\r
- if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression)\r
- if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression)\r
- if (*argument=='y') { overwrite=1; continue; } // -y (answer 'yes' to overwrite permission)\r
- if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)\r
-#endif // DISABLE_LZ4C_LEGACY_OPTIONS\r
-\r
- switch(argument[0])\r
- {\r
- // Display help\r
- case 'V': DISPLAY(WELCOME_MESSAGE); return 0; // Version\r
- case 'h': usage_advanced(); return 0;\r
- case 'H': usage_advanced(); usage_longhelp(); return 0;\r
-\r
- // Compression (default)\r
- case 'z': forceCompress = 1; break;\r
-\r
- // Compression level\r
- case '0': \r
- case '1':\r
- case '2':\r
- case '3':\r
- case '4':\r
- case '5':\r
- case '6':\r
- case '7':\r
- case '8':\r
- case '9': cLevel=*argument -'0'; break;\r
-\r
- // Use Legacy format (hidden option)\r
- case 'l': legacy_format=1; break;\r
-\r
- // Decoding\r
- case 'd': decode=1; break;\r
-\r
- // Force stdout, even if stdout==console\r
- case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break;\r
-\r
- // Test\r
- case 't': decode=1; output_filename=nulmark; break;\r
-\r
- // Overwrite\r
- case 'f': overwrite=1; break;\r
-\r
- // Verbose mode\r
- case 'v': displayLevel=4; break;\r
-\r
- // Quiet mode\r
- case 'q': displayLevel--; break;\r
-\r
- // keep source file (default anyway, so useless) (for xz/lzma compatibility)\r
- case 'k': break;\r
-\r
- // Modify Block Properties\r
- case 'B':\r
- while (argument[1]!=0)\r
- {\r
- int exitBlockProperties=0;\r
- switch(argument[1])\r
- {\r
- case '4':\r
- case '5':\r
- case '6':\r
- case '7':\r
- { \r
- int B = argument[1] - '0'; \r
- int S = 1 << (8 + 2*B); \r
- BMK_SetBlocksize(S); \r
- blockSizeId = B;\r
- argument++;\r
- break;\r
- }\r
- case 'D': blockIndependence = 0, argument++; break;\r
- case 'X': blockChecksum = 1, argument ++; break;\r
- default : exitBlockProperties=1;\r
- }\r
- if (exitBlockProperties) break;\r
- }\r
- break;\r
-\r
- // Modify Stream properties\r
- case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); }\r
-\r
- // Benchmark\r
- case 'b': bench=1; break;\r
-\r
- // Modify Nb Iterations (benchmark only)\r
- case 'i': \r
- if ((argument[1] >='1') && (argument[1] <='9'))\r
- {\r
- int iters = argument[1] - '0'; \r
- BMK_SetNbIterations(iters); \r
- argument++;\r
- }\r
- break;\r
-\r
- // Pause at the end (hidden option)\r
- case 'p': pause=1; BMK_SetPause(); break;\r
-\r
- EXTENDED_ARGUMENTS;\r
-\r
- // Unrecognised command\r
- default : badusage();\r
- }\r
- }\r
- continue;\r
- }\r
-\r
- // first provided filename is input\r
- if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }\r
-\r
- // second provided filename is output\r
- if (!output_filename)\r
- {\r
- output_filename=argument;\r
- if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;\r
- continue;\r
- }\r
- }\r
-\r
- DISPLAYLEVEL(3, WELCOME_MESSAGE);\r
- DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2)));\r
-\r
- // No input filename ==> use stdin\r
- if(!input_filename) { input_filename=stdinmark; }\r
-\r
- // Check if input or output are defined as console; trigger an error in this case\r
- if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();\r
-\r
- // Check if benchmark is selected\r
- if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);\r
-\r
- // No output filename ==> try to select one automatically (when possible)\r
- while (!output_filename) \r
- {\r
- if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } // Default to stdout whenever possible (i.e. not a console)\r
- if ((!decode) && !(forceCompress)) // auto-determine compression or decompression, based on file extension\r
- {\r
- size_t l = strlen(input_filename);\r
- if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1;\r
- }\r
- if (!decode) // compression to file\r
- {\r
- size_t l = strlen(input_filename);\r
- output_filename = (char*)calloc(1,l+5);\r
- strcpy(output_filename, input_filename);\r
- strcpy(output_filename+l, LZ4_EXTENSION);\r
- DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);\r
- break;\r
- }\r
- // decompression to file (automatic name will work only if input filename has format extension ".lz4")\r
- {\r
- size_t outl;\r
- size_t inl = strlen(input_filename);\r
- output_filename = (char*)calloc(1,inl+1);\r
- strcpy(output_filename, input_filename);\r
- outl = inl;\r
- if (inl>4)\r
- while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0;\r
- if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }\r
- DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);\r
- }\r
- }\r
-\r
- // No warning message in pure pipe mode (stdin + stdout)\r
- if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;\r
-\r
- // Check if input or output are defined as console; trigger an error in this case\r
- if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();\r
- if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();\r
-\r
- // Decompress input if selected\r
- if (decode) decodeFile(input_filename, output_filename);\r
- else\r
- // compression is default action\r
- {\r
- if (legacy_format)\r
- {\r
- DISPLAYLEVEL(2, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n");\r
- legacy_compress_file(input_filename, output_filename, cLevel);\r
- }\r
- else\r
- {\r
- DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);\r
- }\r
- }\r
-\r
- if (pause) waitEnter();\r
-}\r
+/*
+ LZ4cli.c - LZ4 Command Line Interface
+ Copyright (C) Yann Collet 2011-2013
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+/*
+ Note : this is stand-alone program.
+ It is not part of LZ4 compression library, it is a user program of the LZ4 library.
+ The license of LZ4 library is BSD.
+ The license of xxHash library is BSD.
+ The license of this compression CLI program is GPLv2.
+*/
+
+//**************************************
+// Tuning parameters
+//**************************************
+// DISABLE_LZ4C_LEGACY_OPTIONS :
+// Control the availability of -c0, -c1 and -hc legacy arguments
+// Default : Legacy options are enabled
+// #define DISABLE_LZ4C_LEGACY_OPTIONS
+
+
+//**************************************
+// Compiler Options
+//**************************************
+// Disable some Visual warning messages
+#ifdef _MSC_VER // Visual Studio
+# define _CRT_SECURE_NO_WARNINGS
+# define _CRT_SECURE_NO_DEPRECATE // VS2005
+# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+#endif
+
+#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix
+#define _POSIX_SOURCE 1 // for fileno() within <stdio.h> on unix
+
+
+//****************************
+// Includes
+//****************************
+#include <stdio.h> // fprintf, fopen, fread, _fileno, stdin, stdout
+#include <stdlib.h> // malloc
+#include <string.h> // strcmp, strlen
+#include <time.h> // clock
+#include "lz4.h"
+#include "lz4hc.h"
+#include "xxhash.h"
+#include "bench.h"
+
+
+//****************************
+// OS-specific Includes
+//****************************
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+# include <fcntl.h> // _O_BINARY
+# include <io.h> // _setmode, _isatty
+# ifdef __MINGW32__
+ int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h>
+# endif
+# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
+# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+# include <unistd.h> // isatty
+# define SET_BINARY_MODE(file)
+# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#endif
+
+
+//**************************************
+// Compiler-specific functions
+//**************************************
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#if defined(_MSC_VER) // Visual Studio
+# define swap32 _byteswap_ulong
+#elif GCC_VERSION >= 403
+# define swap32 __builtin_bswap32
+#else
+ static inline unsigned int swap32(unsigned int x)
+ {
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );
+ }
+#endif
+
+
+//****************************
+// Constants
+//****************************
+#define COMPRESSOR_NAME "LZ4 Compression CLI"
+#ifndef LZ4_VERSION
+# define LZ4_VERSION "v1.1.0"
+#endif
+#define AUTHOR "Yann Collet"
+#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__
+#define LZ4_EXTENSION ".lz4"
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+#define _1BIT 0x01
+#define _2BITS 0x03
+#define _3BITS 0x07
+#define _4BITS 0x0F
+#define _8BITS 0xFF
+
+#define MAGICNUMBER_SIZE 4
+#define LZ4S_MAGICNUMBER 0x184D2204
+#define LZ4S_SKIPPABLE0 0x184D2A50
+#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0
+#define LEGACY_MAGICNUMBER 0x184C2102
+
+#define CACHELINE 64
+#define LEGACY_BLOCKSIZE (8 MB)
+#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)
+#define LZ4S_BLOCKSIZEID_DEFAULT 7
+#define LZ4S_CHECKSUM_SEED 0
+#define LZ4S_EOS 0
+#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1)
+
+
+//**************************************
+// Architecture Macros
+//**************************************
+static const int one = 1;
+#define CPU_LITTLE_ENDIAN (*(char*)(&one))
+#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN)
+#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i))
+
+
+//**************************************
+// Macros
+//**************************************
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+
+
+//**************************************
+// Special input/output
+//**************************************
+#define NULL_OUTPUT "null"
+char stdinmark[] = "stdin";
+char stdoutmark[] = "stdout";
+#ifdef _WIN32
+char nulmark[] = "nul";
+#else
+char nulmark[] = "/dev/null";
+#endif
+
+
+//**************************************
+// Local Parameters
+//**************************************
+static char* programName;
+static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information
+static int overwrite = 0;
+static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
+static int blockChecksum = 0;
+static int streamChecksum = 1;
+static int blockIndependence = 1;
+
+
+//**************************************
+// Exceptions
+//**************************************
+#define DEBUG 0
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
+
+
+//**************************************
+// Version modifiers
+//**************************************
+#define EXTENDED_ARGUMENTS
+#define EXTENDED_HELP
+#define EXTENDED_FORMAT
+#define DEFAULT_COMPRESSOR compress_file
+#define DEFAULT_DECOMPRESSOR decodeLZ4S
+
+
+//****************************
+// Functions
+//****************************
+int usage()
+{
+ DISPLAY( "Usage :\n");
+ DISPLAY( " %s [arg] [input] [output]\n", programName);
+ DISPLAY( "\n");
+ DISPLAY( "input : a filename\n");
+ DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark);
+ DISPLAY( "Arguments :\n");
+ DISPLAY( " -1 : Fast compression (default) \n");
+ DISPLAY( " -9 : High compression \n");
+ DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION);
+ DISPLAY( " -z : force compression\n");
+ DISPLAY( " -f : overwrite output without prompting \n");
+ DISPLAY( " -h/-H : display help/long help and exit\n");
+ return 0;
+}
+
+int usage_advanced()
+{
+ DISPLAY(WELCOME_MESSAGE);
+ usage();
+ DISPLAY( "\n");
+ DISPLAY( "Advanced arguments :\n");
+ DISPLAY( " -V : display Version number and exit\n");
+ DISPLAY( " -v : verbose mode\n");
+ DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
+ DISPLAY( " -c : force write to standard output, even if it is the console\n");
+ DISPLAY( " -t : test compressed file integrity\n");
+ DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
+ DISPLAY( " -B# : Block size [4-7](default : 7)\n");
+ DISPLAY( " -BD : Block dependency (improve compression ratio)\n");
+ DISPLAY( " -BX : enable block checksum (default:disabled)\n");
+ DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");
+ DISPLAY( "Benchmark arguments :\n");
+ DISPLAY( " -b : benchmark file(s)\n");
+ DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");
+#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)
+ DISPLAY( "Legacy arguments :\n");
+ DISPLAY( " -c0 : fast compression\n");
+ DISPLAY( " -c1 : high compression\n");
+ DISPLAY( " -hc : high compression\n");
+ DISPLAY( " -y : overwrite output without prompting \n");
+ DISPLAY( " -s : suppress warnings \n");
+#endif // DISABLE_LZ4C_LEGACY_OPTIONS
+ EXTENDED_HELP;
+ return 0;
+}
+
+int usage_longhelp()
+{
+ DISPLAY( "\n");
+ DISPLAY( "Which values can get [output] ? \n");
+ DISPLAY( "[output] : a filename\n");
+ DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark);
+ DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT);
+ DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n");
+ DISPLAY( " - if stdout is not the console, then [output] = stdout \n");
+ DISPLAY( " - if stdout is console : \n");
+ DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION);
+ DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION);
+ DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION);
+ DISPLAY( "\n");
+ DISPLAY( "Compression levels : \n");
+ DISPLAY( "There are technically 2 accessible compression levels.\n");
+ DISPLAY( "-0 ... -2 => Fast compression\n");
+ DISPLAY( "-3 ... -9 => High compression\n");
+ DISPLAY( "\n");
+ DISPLAY( "stdin, stdout and the console : \n");
+ DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n");
+ DISPLAY( "%s will refuse to read from console, or write to console \n", programName);
+ DISPLAY( "except if '-c' command is specified, to force output to console \n");
+ DISPLAY( "\n");
+ DISPLAY( "Simple example :\n");
+ DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n");
+ DISPLAY( " %s filename\n", programName);
+ DISPLAY( "\n");
+ DISPLAY( "Arguments can be appended together, or provided independently. For example :\n");
+ DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n");
+ DISPLAY( " %s -f9 filename \n", programName);
+ DISPLAY( " is equivalent to :\n");
+ DISPLAY( " %s -f -9 filename \n", programName);
+ DISPLAY( "\n");
+ DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName);
+ DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n");
+ DISPLAY( " generator | %s | consumer \n", programName);
+#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)
+ DISPLAY( "\n");
+ DISPLAY( "Warning :\n");
+ DISPLAY( "Legacy arguments take precedence. Therefore : \n");
+ DISPLAY( " %s -hc filename\n", programName);
+ DISPLAY( "means 'compress filename in high compression mode'\n");
+ DISPLAY( "It is not equivalent to :\n");
+ DISPLAY( " %s -h -c filename\n", programName);
+ DISPLAY( "which would display help text and exit\n");
+#endif // DISABLE_LZ4C_LEGACY_OPTIONS
+ return 0;
+}
+
+int badusage()
+{
+ DISPLAYLEVEL(1, "Incorrect parameters\n");
+ if (displayLevel >= 1) usage();
+ exit(1);
+}
+
+
+static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }
+static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }
+static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }
+
+
+int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)
+{
+
+ if (!strcmp (input_filename, stdinmark))
+ {
+ DISPLAYLEVEL(4,"Using stdin for input\n");
+ *pfinput = stdin;
+ SET_BINARY_MODE(stdin);
+ }
+ else
+ {
+ *pfinput = fopen(input_filename, "rb");
+ }
+
+ if (!strcmp (output_filename, stdoutmark))
+ {
+ DISPLAYLEVEL(4,"Using stdout for output\n");
+ *pfoutput = stdout;
+ SET_BINARY_MODE(stdout);
+ }
+ else
+ {
+ // Check if destination file already exists
+ *pfoutput=0;
+ if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" );
+ if (*pfoutput!=0)
+ {
+ fclose(*pfoutput);
+ if (!overwrite)
+ {
+ char ch;
+ DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
+ DISPLAYLEVEL(2, "Overwrite ? (Y/N) : ");
+ if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible
+ ch = (char)getchar();
+ if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename);
+ }
+ }
+ *pfoutput = fopen( output_filename, "wb" );
+ }
+
+ if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
+ if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
+
+ return 0;
+}
+
+
+
+int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel)
+{
+ int (*compressionFunction)(const char*, char*, int);
+ unsigned long long filesize = 0;
+ unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
+ char* in_buff;
+ char* out_buff;
+ FILE* finput;
+ FILE* foutput;
+ int displayLevel = (compressionlevel>0);
+ clock_t start, end;
+ size_t sizeCheck;
+
+
+ // Init
+ if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC;
+ start = clock();
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3;
+
+ // Allocate Memory
+ in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
+ out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+ if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory");
+
+ // Write Archive Header
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER);
+ sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput);
+ if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header");
+
+ // Main Loop
+ while (1)
+ {
+ unsigned int outSize;
+ // Read Block
+ int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
+ if( inSize<=0 ) break;
+ filesize += inSize;
+ DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
+
+ // Compress Block
+ outSize = compressionFunction(in_buff, out_buff+4, inSize);
+ compressedfilesize += outSize+4;
+ DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ // Write Block
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
+ sizeCheck = fwrite(out_buff, 1, outSize+4, foutput);
+ if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block");
+ }
+
+ // Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ // Close & Free
+ free(in_buff);
+ free(out_buff);
+ fclose(finput);
+ fclose(foutput);
+
+ return 0;
+}
+
+
+int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)
+{
+ void* (*initFunction) (const char*);
+ int (*compressionFunction)(void*, const char*, char*, int, int);
+ char* (*translateFunction) (void*);
+ int (*freeFunction) (void*);
+ void* ctx;
+ unsigned long long filesize = 0;
+ unsigned long long compressedfilesize = 0;
+ unsigned int checkbits;
+ char* in_buff, *in_start, *in_end;
+ char* out_buff;
+ FILE* finput;
+ FILE* foutput;
+ clock_t start, end;
+ unsigned int blockSize, inputBufferSize;
+ size_t sizeCheck, header_size;
+ void* streamChecksumState=NULL;
+
+
+ // Init
+ start = clock();
+ if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
+ if (compressionlevel>=3)
+ {
+ initFunction = LZ4_createHC;
+ compressionFunction = LZ4_compressHC_limitedOutput_continue;
+ translateFunction = LZ4_slideInputBufferHC;
+ freeFunction = LZ4_freeHC;
+ }
+ else
+ {
+ initFunction = LZ4_create;
+ compressionFunction = LZ4_compress_limitedOutput_continue;
+ translateFunction = LZ4_slideInputBuffer;
+ freeFunction = LZ4_free;
+ }
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
+
+ // Allocate Memory
+ inputBufferSize = blockSize + 64 KB;
+ if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;
+ in_buff = (char*)malloc(inputBufferSize);
+ out_buff = (char*)malloc(blockSize+CACHELINE);
+ if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");
+ in_start = in_buff; in_end = in_buff + inputBufferSize;
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
+ ctx = initFunction(in_buff);
+
+ // Write Archive Header
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
+ *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')
+ *(out_buff+4) |= (blockIndependence & _1BIT) << 5;
+ *(out_buff+4) |= (blockChecksum & _1BIT) << 4;
+ *(out_buff+4) |= (streamChecksum & _1BIT) << 2;
+ *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4);
+ checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
+ *(out_buff+6) = (unsigned char) checkbits;
+ header_size = 7;
+ sizeCheck = fwrite(out_buff, 1, header_size, foutput);
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
+ compressedfilesize += header_size;
+
+ // Main Loop
+ while (1)
+ {
+ unsigned int outSize;
+ unsigned int inSize;
+ // Read Block
+ if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);
+ inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);
+ if( inSize==0 ) break; // No more input : end of compression
+ filesize += inSize;
+ DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
+ if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);
+
+ // Compress Block
+ outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;
+ if (blockChecksum) compressedfilesize+=4;
+ DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ // Write Block
+ if (outSize > 0)
+ {
+ int sizeToWrite;
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
+ }
+ sizeToWrite = 4 + outSize + (4*blockChecksum);
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
+
+ }
+ else // Copy Original
+ {
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
+ sizeCheck = fwrite(in_start, 1, inSize, foutput);
+ if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
+ }
+ }
+ in_start += inSize;
+ }
+
+ // End of Stream mark
+ * (unsigned int*) out_buff = LZ4S_EOS;
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
+ compressedfilesize += 4;
+ if (streamChecksum)
+ {
+ unsigned int checksum = XXH32_digest(streamChecksumState);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
+ compressedfilesize += 4;
+ }
+
+ // Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ // Close & Free
+ freeFunction(ctx);
+ free(in_buff);
+ free(out_buff);
+ fclose(finput);
+ fclose(foutput);
+
+ return 0;
+}
+
+
+int compress_file(char* input_filename, char* output_filename, int compressionlevel)
+{
+ int (*compressionFunction)(const char*, char*, int, int);
+ unsigned long long filesize = 0;
+ unsigned long long compressedfilesize = 0;
+ unsigned int checkbits;
+ char* in_buff;
+ char* out_buff;
+ char* headerBuffer;
+ FILE* finput;
+ FILE* foutput;
+ clock_t start, end;
+ int blockSize;
+ size_t sizeCheck, header_size, readSize;
+ void* streamChecksumState=NULL;
+
+ // Branch out
+ if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel);
+
+ // Init
+ start = clock();
+ if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3;
+ if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput;
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);
+
+ // Allocate Memory
+ in_buff = (char*)malloc(blockSize);
+ out_buff = (char*)malloc(blockSize+CACHELINE);
+ headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE);
+ if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory");
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
+
+ // Write Archive Header
+ *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention
+ *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01')
+ *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5;
+ *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4;
+ *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2;
+ *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4);
+ checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED);
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);
+ *(headerBuffer+6) = (unsigned char) checkbits;
+ header_size = 7;
+
+ // Write header
+ sizeCheck = fwrite(headerBuffer, 1, header_size, foutput);
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");
+ compressedfilesize += header_size;
+
+ // read first block
+ readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
+
+ // Main Loop
+ while (readSize>0)
+ {
+ unsigned int outSize;
+
+ filesize += readSize;
+ DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20));
+ if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize);
+
+ // Compress Block
+ outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1);
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4;
+ if (blockChecksum) compressedfilesize+=4;
+ DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100);
+
+ // Write Block
+ if (outSize > 0)
+ {
+ int sizeToWrite;
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);
+ }
+ sizeToWrite = 4 + outSize + (4*blockChecksum);
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");
+ }
+ else // Copy Original Uncompressed
+ {
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");
+ sizeCheck = fwrite(in_buff, 1, readSize, foutput);
+ if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block");
+ if (blockChecksum)
+ {
+ unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");
+ }
+ }
+
+ // Read next block
+ readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput);
+ }
+
+ // End of Stream mark
+ * (unsigned int*) out_buff = LZ4S_EOS;
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");
+ compressedfilesize += 4;
+ if (streamChecksum)
+ {
+ unsigned int checksum = XXH32_digest(streamChecksumState);
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");
+ compressedfilesize += 4;
+ }
+
+ // Close & Free
+ free(in_buff);
+ free(out_buff);
+ free(headerBuffer);
+ fclose(finput);
+ fclose(foutput);
+
+ // Final Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ return 0;
+}
+
+
+unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput)
+{
+ unsigned long long filesize = 0;
+ char* in_buff;
+ char* out_buff;
+ unsigned int blockSize;
+
+
+ // Allocate Memory
+ in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE));
+ out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
+ if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
+
+ // Main Loop
+ while (1)
+ {
+ int decodeSize;
+ size_t sizeCheck;
+
+ // Block Size
+ sizeCheck = fread(&blockSize, 1, 4, finput);
+ if (sizeCheck==0) break; // Nothing to read : file read is completed
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian
+ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE))
+ { // Cannot read next block : maybe new stream ?
+ fseek(finput, -4, SEEK_CUR);
+ break;
+ }
+
+ // Read Block
+ sizeCheck = fread(in_buff, 1, blockSize, finput);
+
+ // Decode Block
+ decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE);
+ if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !");
+ filesize += decodeSize;
+
+ // Write Block
+ sizeCheck = fwrite(out_buff, 1, decodeSize, foutput);
+ if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n");
+ }
+
+ // Free
+ free(in_buff);
+ free(out_buff);
+
+ return filesize;
+}
+
+
+unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
+{
+ unsigned long long filesize = 0;
+ char* in_buff;
+ char* out_buff, *out_start, *out_end;
+ unsigned char descriptor[LZ4S_MAXHEADERSIZE];
+ size_t nbReadBytes;
+ int decodedBytes=0;
+ unsigned int maxBlockSize;
+ size_t sizeCheck;
+ int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
+ void* streamChecksumState=NULL;
+ int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;
+ unsigned int prefix64k = 0;
+
+ // Decode stream descriptor
+ nbReadBytes = fread(descriptor, 1, 3, finput);
+ if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");
+ {
+ int version = (descriptor[0] >> 6) & _2BITS;
+ int streamSize = (descriptor[0] >> 3) & _1BIT;
+ int reserved1 = (descriptor[0] >> 1) & _1BIT;
+ int dictionary = (descriptor[0] >> 0) & _1BIT;
+
+ int reserved2 = (descriptor[1] >> 7) & _1BIT;
+ int blockSizeId = (descriptor[1] >> 4) & _3BITS;
+ int reserved3 = (descriptor[1] >> 0) & _4BITS;
+ int checkBits = (descriptor[2] >> 0) & _8BITS;
+ int checkBits_xxh32;
+
+ blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;
+ blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;
+ streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;
+
+ if (version != 1) EXM_THROW(62, "Wrong version number");
+ if (streamSize == 1) EXM_THROW(64, "Does not support stream size");
+ if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");
+ if (dictionary == 1) EXM_THROW(66, "Does not support dictionary");
+ if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");
+ if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size");
+ if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");
+ maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);
+ // Checkbits verification
+ descriptor[1] &= 0xF0;
+ checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);
+ checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32);
+ if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
+ }
+
+ if (!blockIndependenceFlag)
+ {
+ decompressionFunction = LZ4_decompress_safe_withPrefix64k;
+ prefix64k = 64 KB;
+ }
+
+ // Allocate Memory
+ {
+ unsigned int outbuffSize = prefix64k+maxBlockSize;
+ in_buff = (char*)malloc(maxBlockSize);
+ if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;
+ out_buff = (char*)malloc(outbuffSize);
+ out_end = out_buff + outbuffSize;
+ out_start = out_buff + prefix64k;
+ if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
+ }
+ if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
+
+ // Main Loop
+ while (1)
+ {
+ unsigned int blockSize, uncompressedFlag;
+
+ // Block Size
+ nbReadBytes = fread(&blockSize, 1, 4, finput);
+ if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");
+ if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian
+ uncompressedFlag = blockSize >> 31;
+ blockSize &= 0x7FFFFFFF;
+ if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size");
+
+ // Read Block
+ nbReadBytes = fread(in_buff, 1, blockSize, finput);
+ if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" );
+
+ // Check Block
+ if (blockChecksumFlag)
+ {
+ unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);
+ unsigned int readChecksum;
+ sizeCheck = fread(&readChecksum, 1, 4, finput);
+ if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");
+ readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
+ if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");
+ }
+
+ if (uncompressedFlag)
+ {
+ // Write uncompressed Block
+ sizeCheck = fwrite(in_buff, 1, blockSize, foutput);
+ if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");
+ filesize += blockSize;
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
+ if (!blockIndependenceFlag)
+ {
+ if (blockSize >= prefix64k)
+ {
+ memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks
+ out_start = out_buff + prefix64k;
+ continue;
+ }
+ else
+ {
+ memcpy(out_start, in_buff, blockSize);
+ decodedBytes = blockSize;
+ }
+ }
+ }
+ else
+ {
+ // Decode Block
+ decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);
+ if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
+ filesize += decodedBytes;
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
+
+ // Write Block
+ sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
+ if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
+ }
+
+ if (!blockIndependenceFlag)
+ {
+ out_start += decodedBytes;
+ if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
+ {
+ memcpy(out_buff, out_start - prefix64k, prefix64k);
+ out_start = out_buff + prefix64k;
+ }
+ }
+ }
+
+ // Stream Checksum
+ if (streamChecksumFlag)
+ {
+ unsigned int checksum = XXH32_digest(streamChecksumState);
+ unsigned int readChecksum;
+ sizeCheck = fread(&readChecksum, 1, 4, finput);
+ if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
+ readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
+ if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");
+ }
+
+ // Free
+ free(in_buff);
+ free(out_buff);
+
+ return filesize;
+}
+
+
+unsigned long long selectDecoder( FILE* finput, FILE* foutput)
+{
+ unsigned int magicNumber, size;
+ int errorNb;
+ size_t nbReadBytes;
+
+ // Check Archive Header
+ nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);
+ if (nbReadBytes==0) return 0; // EOF
+ if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");
+ magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format
+ if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers
+
+ switch(magicNumber)
+ {
+ case LZ4S_MAGICNUMBER:
+ return DEFAULT_DECOMPRESSOR(finput, foutput);
+ case LEGACY_MAGICNUMBER:
+ DISPLAYLEVEL(4, "Detected : Legacy format \n");
+ return decodeLegacyStream(finput, foutput);
+ case LZ4S_SKIPPABLE0:
+ DISPLAYLEVEL(4, "Skipping detected skippable area \n");
+ nbReadBytes = fread(&size, 1, 4, finput);
+ if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
+ size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format
+ errorNb = fseek(finput, size, SEEK_CUR);
+ if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
+ return selectDecoder(finput, foutput);
+ EXTENDED_FORMAT;
+ default:
+ if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream
+ DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
+ return 0;
+ }
+}
+
+
+int decodeFile(char* input_filename, char* output_filename)
+{
+ unsigned long long filesize = 0, decodedSize=0;
+ FILE* finput;
+ FILE* foutput;
+ clock_t start, end;
+
+
+ // Init
+ start = clock();
+ get_fileHandle(input_filename, output_filename, &finput, &foutput);
+
+ // Loop over multiple streams
+ do
+ {
+ decodedSize = selectDecoder(finput, foutput);
+ filesize += decodedSize;
+ } while (decodedSize);
+
+ // Final Status
+ end = clock();
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
+ {
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+ DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
+ }
+
+ // Close
+ fclose(finput);
+ fclose(foutput);
+
+ // Error status = OK
+ return 0;
+}
+
+
+void waitEnter()
+{
+ DISPLAY("Press enter to continue...\n");
+ getchar();
+}
+
+
+int main(int argc, char** argv)
+{
+ int i,
+ cLevel=0,
+ decode=0,
+ bench=0,
+ filenamesStart=2,
+ legacy_format=0,
+ forceStdout=0,
+ forceCompress=0,
+ pause=0;
+ char* input_filename=0;
+ char* output_filename=0;
+ char* dynNameSpace=0;
+ char nullOutput[] = NULL_OUTPUT;
+ char extension[] = LZ4_EXTENSION;
+
+ // Init
+ programName = argv[0];
+
+ for(i=1; i<argc; i++)
+ {
+ char* argument = argv[i];
+
+ if(!argument) continue; // Protection if argument empty
+
+ // Decode command (note : aggregated commands are allowed)
+ if (argument[0]=='-')
+ {
+ // '-' means stdin/stdout
+ if (argument[1]==0)
+ {
+ if (!input_filename) input_filename=stdinmark;
+ else output_filename=stdoutmark;
+ }
+
+ while (argument[1]!=0)
+ {
+ argument ++;
+
+#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS)
+ // Legacy options (-c0, -c1, -hc, -y, -s)
+ if ((argument[0]=='c') && (argument[1]=='0')) { cLevel=0; argument++; continue; } // -c0 (fast compression)
+ if ((argument[0]=='c') && (argument[1]=='1')) { cLevel=9; argument++; continue; } // -c1 (high compression)
+ if ((argument[0]=='h') && (argument[1]=='c')) { cLevel=9; argument++; continue; } // -hc (high compression)
+ if (*argument=='y') { overwrite=1; continue; } // -y (answer 'yes' to overwrite permission)
+ if (*argument=='s') { displayLevel=1; continue; } // -s (silent mode)
+#endif // DISABLE_LZ4C_LEGACY_OPTIONS
+
+ switch(argument[0])
+ {
+ // Display help
+ case 'V': DISPLAY(WELCOME_MESSAGE); return 0; // Version
+ case 'h': usage_advanced(); return 0;
+ case 'H': usage_advanced(); usage_longhelp(); return 0;
+
+ // Compression (default)
+ case 'z': forceCompress = 1; break;
+
+ // Compression level
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': cLevel=*argument -'0'; break;
+
+ // Use Legacy format (for Linux kernel compression)
+ case 'l': legacy_format=1; break;
+
+ // Decoding
+ case 'd': decode=1; break;
+
+ // Force stdout, even if stdout==console
+ case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break;
+
+ // Test
+ case 't': decode=1; output_filename=nulmark; break;
+
+ // Overwrite
+ case 'f': overwrite=1; break;
+
+ // Verbose mode
+ case 'v': displayLevel=4; break;
+
+ // Quiet mode
+ case 'q': displayLevel--; break;
+
+ // keep source file (default anyway, so useless) (for xz/lzma compatibility)
+ case 'k': break;
+
+ // Modify Block Properties
+ case 'B':
+ while (argument[1]!=0)
+ {
+ int exitBlockProperties=0;
+ switch(argument[1])
+ {
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ {
+ int B = argument[1] - '0';
+ int S = 1 << (8 + 2*B);
+ BMK_SetBlocksize(S);
+ blockSizeId = B;
+ argument++;
+ break;
+ }
+ case 'D': blockIndependence = 0, argument++; break;
+ case 'X': blockChecksum = 1, argument ++; break;
+ default : exitBlockProperties=1;
+ }
+ if (exitBlockProperties) break;
+ }
+ break;
+
+ // Modify Stream properties
+ case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); }
+
+ // Benchmark
+ case 'b': bench=1; break;
+
+ // Modify Nb Iterations (benchmark only)
+ case 'i':
+ if ((argument[1] >='1') && (argument[1] <='9'))
+ {
+ int iters = argument[1] - '0';
+ BMK_SetNbIterations(iters);
+ argument++;
+ }
+ break;
+
+ // Pause at the end (hidden option)
+ case 'p': pause=1; BMK_SetPause(); break;
+
+ EXTENDED_ARGUMENTS;
+
+ // Unrecognised command
+ default : badusage();
+ }
+ }
+ continue;
+ }
+
+ // first provided filename is input
+ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
+
+ // second provided filename is output
+ if (!output_filename)
+ {
+ output_filename=argument;
+ if (!strcmp (output_filename, nullOutput)) output_filename = nulmark;
+ continue;
+ }
+ }
+
+ DISPLAYLEVEL(3, WELCOME_MESSAGE);
+ DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2)));
+
+ // No input filename ==> use stdin
+ if(!input_filename) { input_filename=stdinmark; }
+
+ // Check if input or output are defined as console; trigger an error in this case
+ if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
+
+ // Check if benchmark is selected
+ if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);
+
+ // No output filename ==> try to select one automatically (when possible)
+ while (!output_filename)
+ {
+ if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } // Default to stdout whenever possible (i.e. not a console)
+ if ((!decode) && !(forceCompress)) // auto-determine compression or decompression, based on file extension
+ {
+ size_t l = strlen(input_filename);
+ if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1;
+ }
+ if (!decode) // compression to file
+ {
+ size_t l = strlen(input_filename);
+ dynNameSpace = (char*)calloc(1,l+5);
+ output_filename = dynNameSpace;
+ strcpy(output_filename, input_filename);
+ strcpy(output_filename+l, LZ4_EXTENSION);
+ DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename);
+ break;
+ }
+ // decompression to file (automatic name will work only if input filename has correct format extension)
+ {
+ size_t outl;
+ size_t inl = strlen(input_filename);
+ dynNameSpace = (char*)calloc(1,inl+1);
+ output_filename = dynNameSpace;
+ strcpy(output_filename, input_filename);
+ outl = inl;
+ if (inl>4)
+ while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0;
+ if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); }
+ DISPLAYLEVEL(2, "Decoding file %s \n", output_filename);
+ }
+ }
+
+ // No warning message in pure pipe mode (stdin + stdout)
+ if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1;
+
+ // Check if input or output are defined as console; trigger an error in this case
+ if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage();
+ if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage();
+
+ // Decompress input if selected
+ if (decode) decodeFile(input_filename, output_filename);
+ else
+ // compression is default action
+ {
+ if (legacy_format)
+ {
+ DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n");
+ legacy_compress_file(input_filename, output_filename, cLevel);
+ }
+ else
+ {
+ DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel);
+ }
+ }
+
+ if (pause) waitEnter();
+ free(dynNameSpace);
+ return 0;
+}
-/*\r
- LZ4 HC - High Compression Mode of LZ4\r
- Copyright (C) 2011-2013, Yann Collet.\r
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)\r
-\r
- Redistribution and use in source and binary forms, with or without\r
- modification, are permitted provided that the following conditions are\r
- met:\r
-\r
- * Redistributions of source code must retain the above copyright\r
- notice, this list of conditions and the following disclaimer.\r
- * Redistributions in binary form must reproduce the above\r
- copyright notice, this list of conditions and the following disclaimer\r
- in the documentation and/or other materials provided with the\r
- distribution.\r
-\r
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\r
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\r
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\r
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\r
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-\r
-//**************************************\r
-// Memory routines\r
-//**************************************\r
-#include <stdlib.h> // calloc, free\r
-#define ALLOCATOR(s) calloc(1,s)\r
-#define FREEMEM free\r
-#include <string.h> // memset, memcpy\r
-#define MEM_INIT memset\r
-\r
-\r
-//**************************************\r
-// CPU Feature Detection\r
-//**************************************\r
-// 32 or 64 bits ?\r
-#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \\r
- || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \\r
- || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \\r
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode\r
-# define LZ4_ARCH64 1\r
-#else\r
-# define LZ4_ARCH64 0\r
-#endif\r
-\r
-// Little Endian or Big Endian ?\r
-// Overwrite the #define below if you know your architecture endianess\r
-#if defined (__GLIBC__)\r
-# include <endian.h>\r
-# if (__BYTE_ORDER == __BIG_ENDIAN)\r
-# define LZ4_BIG_ENDIAN 1\r
-# endif\r
-#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))\r
-# define LZ4_BIG_ENDIAN 1\r
-#elif defined(__sparc) || defined(__sparc__) \\r
- || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \\r
- || defined(__hpux) || defined(__hppa) \\r
- || defined(_MIPSEB) || defined(__s390__)\r
-# define LZ4_BIG_ENDIAN 1\r
-#else\r
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.\r
-#endif\r
-\r
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.\r
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected\r
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance\r
-#if defined(__ARM_FEATURE_UNALIGNED)\r
-# define LZ4_FORCE_UNALIGNED_ACCESS 1\r
-#endif\r
-\r
-// Define this parameter if your target system or compiler does not support hardware bit count\r
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count\r
-# define LZ4_FORCE_SW_BITCOUNT\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Compiler Options\r
-//**************************************\r
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99\r
- /* "restrict" is a known keyword */\r
-#else\r
-# define restrict // Disable restrict\r
-#endif\r
-\r
-#ifdef _MSC_VER // Visual Studio\r
-# define FORCE_INLINE static __forceinline\r
-# include <intrin.h> // For Visual 2005\r
-# if LZ4_ARCH64 // 64-bits\r
-# pragma intrinsic(_BitScanForward64) // For Visual 2005\r
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005\r
-# else // 32-bits\r
-# pragma intrinsic(_BitScanForward) // For Visual 2005\r
-# pragma intrinsic(_BitScanReverse) // For Visual 2005\r
-# endif\r
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant\r
-# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used\r
-#else \r
-# ifdef __GNUC__\r
-# define FORCE_INLINE static inline __attribute__((always_inline))\r
-# else\r
-# define FORCE_INLINE static inline\r
-# endif\r
-#endif\r
-\r
-#ifdef _MSC_VER // Visual Studio\r
-# define lz4_bswap16(x) _byteswap_ushort(x)\r
-#else\r
-# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Includes\r
-//**************************************\r
-#include "lz4hc.h"\r
-#include "lz4.h"\r
-\r
-\r
-//**************************************\r
-// Basic Types\r
-//**************************************\r
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99\r
-# include <stdint.h>\r
- typedef uint8_t BYTE;\r
- typedef uint16_t U16;\r
- typedef uint32_t U32;\r
- typedef int32_t S32;\r
- typedef uint64_t U64;\r
-#else\r
- typedef unsigned char BYTE;\r
- typedef unsigned short U16;\r
- typedef unsigned int U32;\r
- typedef signed int S32;\r
- typedef unsigned long long U64;\r
-#endif\r
-\r
-#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)\r
-# define _PACKED __attribute__ ((packed))\r
-#else\r
-# define _PACKED\r
-#endif\r
-\r
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)\r
-# ifdef __IBMC__\r
-# pragma pack(1)\r
-# else\r
-# pragma pack(push, 1)\r
-# endif\r
-#endif\r
-\r
-typedef struct _U16_S { U16 v; } _PACKED U16_S;\r
-typedef struct _U32_S { U32 v; } _PACKED U32_S;\r
-typedef struct _U64_S { U64 v; } _PACKED U64_S;\r
-\r
-#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)\r
-# pragma pack(pop)\r
-#endif\r
-\r
-#define A64(x) (((U64_S *)(x))->v)\r
-#define A32(x) (((U32_S *)(x))->v)\r
-#define A16(x) (((U16_S *)(x))->v)\r
-\r
-\r
-//**************************************\r
-// Constants\r
-//**************************************\r
-#define MINMATCH 4\r
-\r
-#define DICTIONARY_LOGSIZE 16\r
-#define MAXD (1<<DICTIONARY_LOGSIZE)\r
-#define MAXD_MASK ((U32)(MAXD - 1))\r
-#define MAX_DISTANCE (MAXD - 1)\r
-\r
-#define HASH_LOG (DICTIONARY_LOGSIZE-1)\r
-#define HASHTABLESIZE (1 << HASH_LOG)\r
-#define HASH_MASK (HASHTABLESIZE - 1)\r
-\r
-#define MAX_NB_ATTEMPTS 256\r
-\r
-#define ML_BITS 4\r
-#define ML_MASK (size_t)((1U<<ML_BITS)-1)\r
-#define RUN_BITS (8-ML_BITS)\r
-#define RUN_MASK ((1U<<RUN_BITS)-1)\r
-\r
-#define COPYLENGTH 8\r
-#define LASTLITERALS 5\r
-#define MFLIMIT (COPYLENGTH+MINMATCH)\r
-#define MINLENGTH (MFLIMIT+1)\r
-#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)\r
-\r
-#define KB *(1U<<10)\r
-#define MB *(1U<<20)\r
-#define GB *(1U<<30)\r
-\r
-\r
-//**************************************\r
-// Architecture-specific macros\r
-//**************************************\r
-#if LZ4_ARCH64 // 64-bit\r
-# define STEPSIZE 8\r
-# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;\r
-# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)\r
-# define UARCH U64\r
-# define AARCH A64\r
-# define HTYPE U32\r
-# define INITBASE(b,s) const BYTE* const b = s\r
-#else // 32-bit\r
-# define STEPSIZE 4\r
-# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;\r
-# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);\r
-# define UARCH U32\r
-# define AARCH A32\r
-//# define HTYPE const BYTE*\r
-//# define INITBASE(b,s) const int b = 0\r
-# define HTYPE U32\r
-# define INITBASE(b,s) const BYTE* const b = s\r
-#endif\r
-\r
-#if defined(LZ4_BIG_ENDIAN)\r
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }\r
-# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }\r
-#else // Little Endian\r
-# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }\r
-# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }\r
-#endif\r
-\r
-\r
-//************************************************************\r
-// Local Types\r
-//************************************************************\r
-typedef struct \r
-{\r
- const BYTE* inputBuffer;\r
- const BYTE* base;\r
- const BYTE* end;\r
- HTYPE hashTable[HASHTABLESIZE];\r
- U16 chainTable[MAXD];\r
- const BYTE* nextToUpdate;\r
-} LZ4HC_Data_Structure;\r
-\r
-\r
-//**************************************\r
-// Macros\r
-//**************************************\r
-#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);\r
-#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }\r
-#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))\r
-#define HASH_VALUE(p) HASH_FUNCTION(A32(p))\r
-#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base)\r
-#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] \r
-#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))\r
-\r
-\r
-//**************************************\r
-// Private functions\r
-//**************************************\r
-#if LZ4_ARCH64\r
-\r
-FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)\r
-{\r
-#if defined(LZ4_BIG_ENDIAN)\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r = 0;\r
- _BitScanReverse64( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_clzll(val) >> 3); \r
-# else\r
- int r;\r
- if (!(val>>32)) { r=4; } else { r=0; val>>=32; }\r
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }\r
- r += (!val);\r
- return r;\r
-# endif\r
-#else\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r = 0;\r
- _BitScanForward64( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_ctzll(val) >> 3); \r
-# else\r
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };\r
- return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58];\r
-# endif\r
-#endif\r
-}\r
-\r
-#else\r
-\r
-FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)\r
-{\r
-#if defined(LZ4_BIG_ENDIAN)\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r;\r
- _BitScanReverse( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_clz(val) >> 3); \r
-# else\r
- int r;\r
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }\r
- r += (!val);\r
- return r;\r
-# endif\r
-#else\r
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- unsigned long r;\r
- _BitScanForward( &r, val );\r
- return (int)(r>>3);\r
-# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)\r
- return (__builtin_ctz(val) >> 3); \r
-# else\r
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };\r
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];\r
-# endif\r
-#endif\r
-}\r
-\r
-#endif\r
-\r
-\r
-FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)\r
-{\r
- MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));\r
- MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));\r
- hc4->nextToUpdate = base + 1;\r
- hc4->base = base;\r
- hc4->inputBuffer = base;\r
- hc4->end = base;\r
-}\r
-\r
-\r
-void* LZ4_createHC (const char* inputBuffer)\r
-{\r
- void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));\r
- LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);\r
- return hc4;\r
-}\r
-\r
-\r
-int LZ4_freeHC (void* LZ4HC_Data)\r
-{\r
- FREEMEM(LZ4HC_Data);\r
- return (0);\r
-}\r
-\r
-\r
-// Update chains up to ip (excluded)\r
-FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)\r
-{\r
- U16* chainTable = hc4->chainTable;\r
- HTYPE* HashTable = hc4->hashTable;\r
- INITBASE(base,hc4->base);\r
-\r
- while(hc4->nextToUpdate < ip)\r
- {\r
- const BYTE* const p = hc4->nextToUpdate;\r
- size_t delta = (p) - HASH_POINTER(p); \r
- if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; \r
- DELTANEXT(p) = (U16)delta; \r
- HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base);\r
- hc4->nextToUpdate++;\r
- }\r
-}\r
-\r
-\r
-char* LZ4_slideInputBufferHC(void* LZ4HC_Data)\r
-{\r
- LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;\r
- U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB;\r
- distance = (distance >> 16) << 16; // Must be a multiple of 64 KB\r
- LZ4HC_Insert(hc4, hc4->end - MINMATCH);\r
- memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);\r
- hc4->nextToUpdate -= distance;\r
- hc4->base -= distance;\r
- if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow\r
- {\r
- int i;\r
- hc4->base += 1 GB;\r
- for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;\r
- }\r
- hc4->end -= distance;\r
- return (char*)(hc4->end);\r
-}\r
-\r
-\r
-FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)\r
-{\r
- const BYTE* p1t = p1;\r
-\r
- while (p1t<matchlimit-(STEPSIZE-1))\r
- {\r
- UARCH diff = AARCH(p2) ^ AARCH(p1t);\r
- if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; }\r
- p1t += LZ4_NbCommonBytes(diff);\r
- return (p1t - p1);\r
- }\r
- if (LZ4_ARCH64) if ((p1t<(matchlimit-3)) && (A32(p2) == A32(p1t))) { p1t+=4; p2+=4; }\r
- if ((p1t<(matchlimit-1)) && (A16(p2) == A16(p1t))) { p1t+=2; p2+=2; }\r
- if ((p1t<matchlimit) && (*p2 == *p1t)) p1t++;\r
- return (p1t - p1);\r
-}\r
-\r
-\r
-FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)\r
-{\r
- U16* const chainTable = hc4->chainTable;\r
- HTYPE* const HashTable = hc4->hashTable;\r
- const BYTE* ref;\r
- INITBASE(base,hc4->base);\r
- int nbAttempts=MAX_NB_ATTEMPTS;\r
- size_t repl=0, ml=0;\r
- U16 delta=0; // useless assignment, to remove an uninitialization warning\r
-\r
- // HC4 match finder\r
- LZ4HC_Insert(hc4, ip);\r
- ref = HASH_POINTER(ip);\r
-\r
-#define REPEAT_OPTIMIZATION\r
-#ifdef REPEAT_OPTIMIZATION\r
- // Detect repetitive sequences of length <= 4\r
- if ((U32)(ip-ref) <= 4) // potential repetition\r
- {\r
- if (A32(ref) == A32(ip)) // confirmed\r
- {\r
- delta = (U16)(ip-ref);\r
- repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;\r
- *matchpos = ref;\r
- }\r
- ref = GETNEXT(ref);\r
- }\r
-#endif\r
-\r
- while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))\r
- {\r
- nbAttempts--;\r
- if (*(ref+ml) == *(ip+ml))\r
- if (A32(ref) == A32(ip))\r
- {\r
- size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;\r
- if (mlt > ml) { ml = mlt; *matchpos = ref; }\r
- }\r
- ref = GETNEXT(ref);\r
- }\r
-\r
-#ifdef REPEAT_OPTIMIZATION\r
- // Complete table\r
- if (repl)\r
- {\r
- const BYTE* ptr = ip;\r
- const BYTE* end;\r
-\r
- end = ip + repl - (MINMATCH-1);\r
- while(ptr < end-delta)\r
- {\r
- DELTANEXT(ptr) = delta; // Pre-Load\r
- ptr++;\r
- }\r
- do\r
- {\r
- DELTANEXT(ptr) = delta; \r
- HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain\r
- ptr++;\r
- } while(ptr < end);\r
- hc4->nextToUpdate = end;\r
- }\r
-#endif \r
-\r
- return (int)ml;\r
-}\r
-\r
-\r
-FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)\r
-{\r
- U16* const chainTable = hc4->chainTable;\r
- HTYPE* const HashTable = hc4->hashTable;\r
- INITBASE(base,hc4->base);\r
- const BYTE* ref;\r
- int nbAttempts = MAX_NB_ATTEMPTS;\r
- int delta = (int)(ip-startLimit);\r
-\r
- // First Match\r
- LZ4HC_Insert(hc4, ip);\r
- ref = HASH_POINTER(ip);\r
-\r
- while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))\r
- {\r
- nbAttempts--;\r
- if (*(startLimit + longest) == *(ref - delta + longest))\r
- if (A32(ref) == A32(ip))\r
- {\r
-#if 1\r
- const BYTE* reft = ref+MINMATCH;\r
- const BYTE* ipt = ip+MINMATCH;\r
- const BYTE* startt = ip;\r
-\r
- while (ipt<matchlimit-(STEPSIZE-1))\r
- {\r
- UARCH diff = AARCH(reft) ^ AARCH(ipt);\r
- if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }\r
- ipt += LZ4_NbCommonBytes(diff);\r
- goto _endCount;\r
- }\r
- if (LZ4_ARCH64) if ((ipt<(matchlimit-3)) && (A32(reft) == A32(ipt))) { ipt+=4; reft+=4; }\r
- if ((ipt<(matchlimit-1)) && (A16(reft) == A16(ipt))) { ipt+=2; reft+=2; }\r
- if ((ipt<matchlimit) && (*reft == *ipt)) ipt++;\r
-_endCount:\r
- reft = ref;\r
-#else\r
- // Easier for code maintenance, but unfortunately slower too\r
- const BYTE* startt = ip;\r
- const BYTE* reft = ref;\r
- const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);\r
-#endif\r
-\r
- while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;}\r
-\r
- if ((ipt-startt) > longest)\r
- {\r
- longest = (int)(ipt-startt);\r
- *matchpos = reft;\r
- *startpos = startt;\r
- }\r
- }\r
- ref = GETNEXT(ref);\r
- }\r
-\r
- return longest;\r
-}\r
-\r
-\r
-typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;\r
-\r
-FORCE_INLINE int LZ4HC_encodeSequence (\r
- const BYTE** ip,\r
- BYTE** op,\r
- const BYTE** anchor,\r
- int matchLength,\r
- const BYTE* ref,\r
- limitedOutput_directive limitedOutputBuffer,\r
- BYTE* oend)\r
-{\r
- int length;\r
- BYTE* token;\r
-\r
- // Encode Literal length\r
- length = (int)(*ip - *anchor);\r
- token = (*op)++;\r
- if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; // Check output limit\r
- if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } \r
- else *token = (BYTE)(length<<ML_BITS);\r
-\r
- // Copy Literals\r
- LZ4_BLINDCOPY(*anchor, *op, length);\r
-\r
- // Encode Offset\r
- LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));\r
-\r
- // Encode MatchLength\r
- length = (int)(matchLength-MINMATCH);\r
- if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; // Check output limit\r
- if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } \r
- else *token += (BYTE)(length); \r
-\r
- // Prepare next loop\r
- *ip += matchLength;\r
- *anchor = *ip; \r
-\r
- return 0;\r
-}\r
-\r
-\r
-static int LZ4HC_compress_generic (\r
- void* ctxvoid,\r
- const char* source, \r
- char* dest,\r
- int inputSize,\r
- int maxOutputSize,\r
- limitedOutput_directive limit\r
- )\r
-{\r
- LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;\r
- const BYTE* ip = (const BYTE*) source;\r
- const BYTE* anchor = ip;\r
- const BYTE* const iend = ip + inputSize;\r
- const BYTE* const mflimit = iend - MFLIMIT;\r
- const BYTE* const matchlimit = (iend - LASTLITERALS);\r
-\r
- BYTE* op = (BYTE*) dest;\r
- BYTE* const oend = op + maxOutputSize;\r
-\r
- int ml, ml2, ml3, ml0;\r
- const BYTE* ref=NULL;\r
- const BYTE* start2=NULL;\r
- const BYTE* ref2=NULL;\r
- const BYTE* start3=NULL;\r
- const BYTE* ref3=NULL;\r
- const BYTE* start0;\r
- const BYTE* ref0;\r
-\r
-\r
- // Ensure blocks follow each other\r
- if (ip != ctx->end) return 0;\r
- ctx->end += inputSize;\r
-\r
- ip++;\r
-\r
- // Main Loop\r
- while (ip < mflimit)\r
- {\r
- ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref));\r
- if (!ml) { ip++; continue; }\r
-\r
- // saved, in case we would skip too much\r
- start0 = ip;\r
- ref0 = ref;\r
- ml0 = ml;\r
-\r
-_Search2:\r
- if (ip+ml < mflimit)\r
- ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);\r
- else ml2 = ml;\r
-\r
- if (ml2 == ml) // No better match\r
- {\r
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;\r
- continue;\r
- }\r
-\r
- if (start0 < ip)\r
- {\r
- if (start2 < ip + ml0) // empirical\r
- {\r
- ip = start0;\r
- ref = ref0;\r
- ml = ml0;\r
- }\r
- }\r
-\r
- // Here, start0==ip\r
- if ((start2 - ip) < 3) // First Match too small : removed\r
- {\r
- ml = ml2;\r
- ip = start2;\r
- ref =ref2;\r
- goto _Search2;\r
- }\r
-\r
-_Search3:\r
- // Currently we have :\r
- // ml2 > ml1, and\r
- // ip1+3 <= ip2 (usually < ip1+ml1)\r
- if ((start2 - ip) < OPTIMAL_ML)\r
- {\r
- int correction;\r
- int new_ml = ml;\r
- if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;\r
- if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;\r
- correction = new_ml - (int)(start2 - ip);\r
- if (correction > 0)\r
- {\r
- start2 += correction;\r
- ref2 += correction;\r
- ml2 -= correction;\r
- }\r
- }\r
- // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18)\r
-\r
- if (start2 + ml2 < mflimit)\r
- ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);\r
- else ml3 = ml2;\r
-\r
- if (ml3 == ml2) // No better match : 2 sequences to encode\r
- {\r
- // ip & ref are known; Now for ml\r
- if (start2 < ip+ml) ml = (int)(start2 - ip);\r
- // Now, encode 2 sequences\r
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;\r
- ip = start2;\r
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;\r
- continue;\r
- }\r
-\r
- if (start3 < ip+ml+3) // Not enough space for match 2 : remove it\r
- {\r
- if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1\r
- {\r
- if (start2 < ip+ml)\r
- {\r
- int correction = (int)(ip+ml - start2);\r
- start2 += correction;\r
- ref2 += correction;\r
- ml2 -= correction;\r
- if (ml2 < MINMATCH)\r
- {\r
- start2 = start3;\r
- ref2 = ref3;\r
- ml2 = ml3;\r
- }\r
- }\r
-\r
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;\r
- ip = start3;\r
- ref = ref3;\r
- ml = ml3;\r
-\r
- start0 = start2;\r
- ref0 = ref2;\r
- ml0 = ml2;\r
- goto _Search2;\r
- }\r
-\r
- start2 = start3;\r
- ref2 = ref3;\r
- ml2 = ml3;\r
- goto _Search3;\r
- }\r
-\r
- // OK, now we have 3 ascending matches; let's write at least the first one\r
- // ip & ref are known; Now for ml\r
- if (start2 < ip+ml)\r
- {\r
- if ((start2 - ip) < (int)ML_MASK)\r
- {\r
- int correction;\r
- if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;\r
- if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;\r
- correction = ml - (int)(start2 - ip);\r
- if (correction > 0)\r
- {\r
- start2 += correction;\r
- ref2 += correction;\r
- ml2 -= correction;\r
- }\r
- }\r
- else\r
- {\r
- ml = (int)(start2 - ip);\r
- }\r
- }\r
- if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;\r
-\r
- ip = start2;\r
- ref = ref2;\r
- ml = ml2;\r
-\r
- start2 = start3;\r
- ref2 = ref3;\r
- ml2 = ml3;\r
-\r
- goto _Search3;\r
-\r
- }\r
-\r
- // Encode Last Literals\r
- {\r
- int lastRun = (int)(iend - anchor);\r
- if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit\r
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } \r
- else *op++ = (BYTE)(lastRun<<ML_BITS);\r
- memcpy(op, anchor, iend - anchor);\r
- op += iend-anchor;\r
- } \r
-\r
- // End\r
- return (int) (((char*)op)-dest);\r
-}\r
-\r
-\r
-int LZ4_compressHC(const char* source, char* dest, int inputSize)\r
-{\r
- void* ctx = LZ4_createHC(source);\r
- int result;\r
- if (ctx==NULL) return 0;\r
-\r
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, noLimit);\r
-\r
- LZ4_freeHC(ctx);\r
- return result;\r
-}\r
-\r
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)\r
-{\r
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, noLimit);\r
-}\r
-\r
-\r
-int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)\r
-{\r
- void* ctx = LZ4_createHC(source);\r
- int result;\r
- if (ctx==NULL) return 0;\r
-\r
- result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, limitedOutput);\r
-\r
- LZ4_freeHC(ctx);\r
- return result;\r
-}\r
-\r
-int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)\r
-{\r
- return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, limitedOutput);\r
-}\r
-\r
+/*
+ LZ4 HC - High Compression Mode of LZ4
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+
+//**************************************
+// Memory routines
+//**************************************
+#include <stdlib.h> // calloc, free
+#define ALLOCATOR(s) calloc(1,s)
+#define FREEMEM free
+#include <string.h> // memset, memcpy
+#define MEM_INIT memset
+
+
+//**************************************
+// CPU Feature Detection
+//**************************************
+// 32 or 64 bits ?
+#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
+ || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
+ || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
+ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
+# define LZ4_ARCH64 1
+#else
+# define LZ4_ARCH64 0
+#endif
+
+// Little Endian or Big Endian ?
+// Overwrite the #define below if you know your architecture endianess
+#if defined (__GLIBC__)
+# include <endian.h>
+# if (__BYTE_ORDER == __BIG_ENDIAN)
+# define LZ4_BIG_ENDIAN 1
+# endif
+#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN))
+# define LZ4_BIG_ENDIAN 1
+#elif defined(__sparc) || defined(__sparc__) \
+ || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \
+ || defined(__hpux) || defined(__hppa) \
+ || defined(_MIPSEB) || defined(__s390__)
+# define LZ4_BIG_ENDIAN 1
+#else
+// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
+#endif
+
+// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
+// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+#if defined(__ARM_FEATURE_UNALIGNED)
+# define LZ4_FORCE_UNALIGNED_ACCESS 1
+#endif
+
+// Define this parameter if your target system or compiler does not support hardware bit count
+#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
+# define LZ4_FORCE_SW_BITCOUNT
+#endif
+
+
+//**************************************
+// Compiler Options
+//**************************************
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+ /* "restrict" is a known keyword */
+#else
+# define restrict // Disable restrict
+#endif
+
+#ifdef _MSC_VER // Visual Studio
+# define FORCE_INLINE static __forceinline
+# include <intrin.h> // For Visual 2005
+# if LZ4_ARCH64 // 64-bits
+# pragma intrinsic(_BitScanForward64) // For Visual 2005
+# pragma intrinsic(_BitScanReverse64) // For Visual 2005
+# else // 32-bits
+# pragma intrinsic(_BitScanForward) // For Visual 2005
+# pragma intrinsic(_BitScanReverse) // For Visual 2005
+# endif
+# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used
+#else
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+#endif
+
+#ifdef _MSC_VER // Visual Studio
+# define lz4_bswap16(x) _byteswap_ushort(x)
+#else
+# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
+#endif
+
+
+//**************************************
+// Includes
+//**************************************
+#include "lz4hc.h"
+#include "lz4.h"
+
+
+//**************************************
+// Basic Types
+//**************************************
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS)
+# define _PACKED __attribute__ ((packed))
+#else
+# define _PACKED
+#endif
+
+#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# ifdef __IBMC__
+# pragma pack(1)
+# else
+# pragma pack(push, 1)
+# endif
+#endif
+
+typedef struct _U16_S { U16 v; } _PACKED U16_S;
+typedef struct _U32_S { U32 v; } _PACKED U32_S;
+typedef struct _U64_S { U64 v; } _PACKED U64_S;
+
+#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# pragma pack(pop)
+#endif
+
+#define A64(x) (((U64_S *)(x))->v)
+#define A32(x) (((U32_S *)(x))->v)
+#define A16(x) (((U16_S *)(x))->v)
+
+
+//**************************************
+// Constants
+//**************************************
+#define MINMATCH 4
+
+#define DICTIONARY_LOGSIZE 16
+#define MAXD (1<<DICTIONARY_LOGSIZE)
+#define MAXD_MASK ((U32)(MAXD - 1))
+#define MAX_DISTANCE (MAXD - 1)
+
+#define HASH_LOG (DICTIONARY_LOGSIZE-1)
+#define HASHTABLESIZE (1 << HASH_LOG)
+#define HASH_MASK (HASHTABLESIZE - 1)
+
+#define MAX_NB_ATTEMPTS 256
+
+#define ML_BITS 4
+#define ML_MASK (size_t)((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+#define COPYLENGTH 8
+#define LASTLITERALS 5
+#define MFLIMIT (COPYLENGTH+MINMATCH)
+#define MINLENGTH (MFLIMIT+1)
+#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
+
+
+//**************************************
+// Architecture-specific macros
+//**************************************
+#if LZ4_ARCH64 // 64-bit
+# define STEPSIZE 8
+# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
+# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
+# define UARCH U64
+# define AARCH A64
+# define HTYPE U32
+# define INITBASE(b,s) const BYTE* const b = s
+#else // 32-bit
+# define STEPSIZE 4
+# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
+# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
+# define UARCH U32
+# define AARCH A32
+//# define HTYPE const BYTE*
+//# define INITBASE(b,s) const int b = 0
+# define HTYPE U32
+# define INITBASE(b,s) const BYTE* const b = s
+#endif
+
+#if defined(LZ4_BIG_ENDIAN)
+# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
+# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
+#else // Little Endian
+# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
+# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
+#endif
+
+
+//************************************************************
+// Local Types
+//************************************************************
+typedef struct
+{
+ const BYTE* inputBuffer;
+ const BYTE* base;
+ const BYTE* end;
+ HTYPE hashTable[HASHTABLESIZE];
+ U16 chainTable[MAXD];
+ const BYTE* nextToUpdate;
+} LZ4HC_Data_Structure;
+
+
+//**************************************
+// Macros
+//**************************************
+#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
+#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }
+#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
+#define HASH_VALUE(p) HASH_FUNCTION(A32(p))
+#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base)
+#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
+#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
+
+
+//**************************************
+// Private functions
+//**************************************
+#if LZ4_ARCH64
+
+FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
+{
+#if defined(LZ4_BIG_ENDIAN)
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanReverse64( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_clzll(val) >> 3);
+# else
+ int r;
+ if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
+ if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+ r += (!val);
+ return r;
+# endif
+#else
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r = 0;
+ _BitScanForward64( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_ctzll(val) >> 3);
+# else
+ static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+ return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58];
+# endif
+#endif
+}
+
+#else
+
+FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
+{
+#if defined(LZ4_BIG_ENDIAN)
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r;
+ _BitScanReverse( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_clz(val) >> 3);
+# else
+ int r;
+ if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+ r += (!val);
+ return r;
+# endif
+#else
+# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ unsigned long r;
+ _BitScanForward( &r, val );
+ return (int)(r>>3);
+# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
+ return (__builtin_ctz(val) >> 3);
+# else
+ static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
+ return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+# endif
+#endif
+}
+
+#endif
+
+
+int LZ4_sizeofStreamStateHC()
+{
+ return sizeof(LZ4HC_Data_Structure);
+}
+
+FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
+{
+ MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+ MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
+ hc4->nextToUpdate = base + 1;
+ hc4->base = base;
+ hc4->inputBuffer = base;
+ hc4->end = base;
+}
+
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
+{
+ if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; // Error : pointer is not aligned for pointer (32 or 64 bits)
+ LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
+ return 0;
+}
+
+
+void* LZ4_createHC (const char* inputBuffer)
+{
+ void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
+ LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
+ return hc4;
+}
+
+
+int LZ4_freeHC (void* LZ4HC_Data)
+{
+ FREEMEM(LZ4HC_Data);
+ return (0);
+}
+
+
+// Update chains up to ip (excluded)
+FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
+{
+ U16* chainTable = hc4->chainTable;
+ HTYPE* HashTable = hc4->hashTable;
+ INITBASE(base,hc4->base);
+
+ while(hc4->nextToUpdate < ip)
+ {
+ const BYTE* const p = hc4->nextToUpdate;
+ size_t delta = (p) - HASH_POINTER(p);
+ if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
+ DELTANEXT(p) = (U16)delta;
+ HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base);
+ hc4->nextToUpdate++;
+ }
+}
+
+
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
+{
+ LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
+ U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB;
+ distance = (distance >> 16) << 16; // Must be a multiple of 64 KB
+ LZ4HC_Insert(hc4, hc4->end - MINMATCH);
+ memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
+ hc4->nextToUpdate -= distance;
+ hc4->base -= distance;
+ if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow
+ {
+ int i;
+ hc4->base += 1 GB;
+ for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;
+ }
+ hc4->end -= distance;
+ return (char*)(hc4->end);
+}
+
+
+FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
+{
+ const BYTE* p1t = p1;
+
+ while (p1t<matchlimit-(STEPSIZE-1))
+ {
+ UARCH diff = AARCH(p2) ^ AARCH(p1t);
+ if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; }
+ p1t += LZ4_NbCommonBytes(diff);
+ return (p1t - p1);
+ }
+ if (LZ4_ARCH64) if ((p1t<(matchlimit-3)) && (A32(p2) == A32(p1t))) { p1t+=4; p2+=4; }
+ if ((p1t<(matchlimit-1)) && (A16(p2) == A16(p1t))) { p1t+=2; p2+=2; }
+ if ((p1t<matchlimit) && (*p2 == *p1t)) p1t++;
+ return (p1t - p1);
+}
+
+
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)
+{
+ U16* const chainTable = hc4->chainTable;
+ HTYPE* const HashTable = hc4->hashTable;
+ const BYTE* ref;
+ INITBASE(base,hc4->base);
+ int nbAttempts=MAX_NB_ATTEMPTS;
+ size_t repl=0, ml=0;
+ U16 delta=0; // useless assignment, to remove an uninitialization warning
+
+ // HC4 match finder
+ LZ4HC_Insert(hc4, ip);
+ ref = HASH_POINTER(ip);
+
+#define REPEAT_OPTIMIZATION
+#ifdef REPEAT_OPTIMIZATION
+ // Detect repetitive sequences of length <= 4
+ if ((U32)(ip-ref) <= 4) // potential repetition
+ {
+ if (A32(ref) == A32(ip)) // confirmed
+ {
+ delta = (U16)(ip-ref);
+ repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
+ *matchpos = ref;
+ }
+ ref = GETNEXT(ref);
+ }
+#endif
+
+ while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
+ {
+ nbAttempts--;
+ if (*(ref+ml) == *(ip+ml))
+ if (A32(ref) == A32(ip))
+ {
+ size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
+ if (mlt > ml) { ml = mlt; *matchpos = ref; }
+ }
+ ref = GETNEXT(ref);
+ }
+
+#ifdef REPEAT_OPTIMIZATION
+ // Complete table
+ if (repl)
+ {
+ const BYTE* ptr = ip;
+ const BYTE* end;
+
+ end = ip + repl - (MINMATCH-1);
+ while(ptr < end-delta)
+ {
+ DELTANEXT(ptr) = delta; // Pre-Load
+ ptr++;
+ }
+ do
+ {
+ DELTANEXT(ptr) = delta;
+ HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain
+ ptr++;
+ } while(ptr < end);
+ hc4->nextToUpdate = end;
+ }
+#endif
+
+ return (int)ml;
+}
+
+
+FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)
+{
+ U16* const chainTable = hc4->chainTable;
+ HTYPE* const HashTable = hc4->hashTable;
+ INITBASE(base,hc4->base);
+ const BYTE* ref;
+ int nbAttempts = MAX_NB_ATTEMPTS;
+ int delta = (int)(ip-startLimit);
+
+ // First Match
+ LZ4HC_Insert(hc4, ip);
+ ref = HASH_POINTER(ip);
+
+ while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))
+ {
+ nbAttempts--;
+ if (*(startLimit + longest) == *(ref - delta + longest))
+ if (A32(ref) == A32(ip))
+ {
+#if 1
+ const BYTE* reft = ref+MINMATCH;
+ const BYTE* ipt = ip+MINMATCH;
+ const BYTE* startt = ip;
+
+ while (ipt<matchlimit-(STEPSIZE-1))
+ {
+ UARCH diff = AARCH(reft) ^ AARCH(ipt);
+ if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }
+ ipt += LZ4_NbCommonBytes(diff);
+ goto _endCount;
+ }
+ if (LZ4_ARCH64) if ((ipt<(matchlimit-3)) && (A32(reft) == A32(ipt))) { ipt+=4; reft+=4; }
+ if ((ipt<(matchlimit-1)) && (A16(reft) == A16(ipt))) { ipt+=2; reft+=2; }
+ if ((ipt<matchlimit) && (*reft == *ipt)) ipt++;
+_endCount:
+ reft = ref;
+#else
+ // Easier for code maintenance, but unfortunately slower too
+ const BYTE* startt = ip;
+ const BYTE* reft = ref;
+ const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);
+#endif
+
+ while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;}
+
+ if ((ipt-startt) > longest)
+ {
+ longest = (int)(ipt-startt);
+ *matchpos = reft;
+ *startpos = startt;
+ }
+ }
+ ref = GETNEXT(ref);
+ }
+
+ return longest;
+}
+
+
+typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
+
+FORCE_INLINE int LZ4HC_encodeSequence (
+ const BYTE** ip,
+ BYTE** op,
+ const BYTE** anchor,
+ int matchLength,
+ const BYTE* ref,
+ limitedOutput_directive limitedOutputBuffer,
+ BYTE* oend)
+{
+ int length;
+ BYTE* token;
+
+ // Encode Literal length
+ length = (int)(*ip - *anchor);
+ token = (*op)++;
+ if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; // Check output limit
+ if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
+ else *token = (BYTE)(length<<ML_BITS);
+
+ // Copy Literals
+ LZ4_BLINDCOPY(*anchor, *op, length);
+
+ // Encode Offset
+ LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
+
+ // Encode MatchLength
+ length = (int)(matchLength-MINMATCH);
+ if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; // Check output limit
+ if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
+ else *token += (BYTE)(length);
+
+ // Prepare next loop
+ *ip += matchLength;
+ *anchor = *ip;
+
+ return 0;
+}
+
+
+static int LZ4HC_compress_generic (
+ void* ctxvoid,
+ const char* source,
+ char* dest,
+ int inputSize,
+ int maxOutputSize,
+ limitedOutput_directive limit
+ )
+{
+ LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
+ const BYTE* ip = (const BYTE*) source;
+ const BYTE* anchor = ip;
+ const BYTE* const iend = ip + inputSize;
+ const BYTE* const mflimit = iend - MFLIMIT;
+ const BYTE* const matchlimit = (iend - LASTLITERALS);
+
+ BYTE* op = (BYTE*) dest;
+ BYTE* const oend = op + maxOutputSize;
+
+ int ml, ml2, ml3, ml0;
+ const BYTE* ref=NULL;
+ const BYTE* start2=NULL;
+ const BYTE* ref2=NULL;
+ const BYTE* start3=NULL;
+ const BYTE* ref3=NULL;
+ const BYTE* start0;
+ const BYTE* ref0;
+
+
+ // Ensure blocks follow each other
+ if (ip != ctx->end) return 0;
+ ctx->end += inputSize;
+
+ ip++;
+
+ // Main Loop
+ while (ip < mflimit)
+ {
+ ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref));
+ if (!ml) { ip++; continue; }
+
+ // saved, in case we would skip too much
+ start0 = ip;
+ ref0 = ref;
+ ml0 = ml;
+
+_Search2:
+ if (ip+ml < mflimit)
+ ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);
+ else ml2 = ml;
+
+ if (ml2 == ml) // No better match
+ {
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ continue;
+ }
+
+ if (start0 < ip)
+ {
+ if (start2 < ip + ml0) // empirical
+ {
+ ip = start0;
+ ref = ref0;
+ ml = ml0;
+ }
+ }
+
+ // Here, start0==ip
+ if ((start2 - ip) < 3) // First Match too small : removed
+ {
+ ml = ml2;
+ ip = start2;
+ ref =ref2;
+ goto _Search2;
+ }
+
+_Search3:
+ // Currently we have :
+ // ml2 > ml1, and
+ // ip1+3 <= ip2 (usually < ip1+ml1)
+ if ((start2 - ip) < OPTIMAL_ML)
+ {
+ int correction;
+ int new_ml = ml;
+ if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
+ if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
+ correction = new_ml - (int)(start2 - ip);
+ if (correction > 0)
+ {
+ start2 += correction;
+ ref2 += correction;
+ ml2 -= correction;
+ }
+ }
+ // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18)
+
+ if (start2 + ml2 < mflimit)
+ ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);
+ else ml3 = ml2;
+
+ if (ml3 == ml2) // No better match : 2 sequences to encode
+ {
+ // ip & ref are known; Now for ml
+ if (start2 < ip+ml) ml = (int)(start2 - ip);
+ // Now, encode 2 sequences
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ ip = start2;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
+ continue;
+ }
+
+ if (start3 < ip+ml+3) // Not enough space for match 2 : remove it
+ {
+ if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
+ {
+ if (start2 < ip+ml)
+ {
+ int correction = (int)(ip+ml - start2);
+ start2 += correction;
+ ref2 += correction;
+ ml2 -= correction;
+ if (ml2 < MINMATCH)
+ {
+ start2 = start3;
+ ref2 = ref3;
+ ml2 = ml3;
+ }
+ }
+
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ ip = start3;
+ ref = ref3;
+ ml = ml3;
+
+ start0 = start2;
+ ref0 = ref2;
+ ml0 = ml2;
+ goto _Search2;
+ }
+
+ start2 = start3;
+ ref2 = ref3;
+ ml2 = ml3;
+ goto _Search3;
+ }
+
+ // OK, now we have 3 ascending matches; let's write at least the first one
+ // ip & ref are known; Now for ml
+ if (start2 < ip+ml)
+ {
+ if ((start2 - ip) < (int)ML_MASK)
+ {
+ int correction;
+ if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
+ if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
+ correction = ml - (int)(start2 - ip);
+ if (correction > 0)
+ {
+ start2 += correction;
+ ref2 += correction;
+ ml2 -= correction;
+ }
+ }
+ else
+ {
+ ml = (int)(start2 - ip);
+ }
+ }
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+
+ ip = start2;
+ ref = ref2;
+ ml = ml2;
+
+ start2 = start3;
+ ref2 = ref3;
+ ml2 = ml3;
+
+ goto _Search3;
+
+ }
+
+ // Encode Last Literals
+ {
+ int lastRun = (int)(iend - anchor);
+ if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; // Check output limit
+ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
+ else *op++ = (BYTE)(lastRun<<ML_BITS);
+ memcpy(op, anchor, iend - anchor);
+ op += iend-anchor;
+ }
+
+ // End
+ return (int) (((char*)op)-dest);
+}
+
+
+int LZ4_compressHC(const char* source, char* dest, int inputSize)
+{
+ void* ctx = LZ4_createHC(source);
+ int result;
+ if (ctx==NULL) return 0;
+
+ result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, noLimit);
+
+ LZ4_freeHC(ctx);
+ return result;
+}
+
+int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ void* ctx = LZ4_createHC(source);
+ int result;
+ if (ctx==NULL) return 0;
+
+ result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, limitedOutput);
+
+ LZ4_freeHC(ctx);
+ return result;
+}
+
+
+//*****************************
+// Using an external allocation
+//*****************************
+
+int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); }
+
+
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize)
+{
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits)
+ LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
+ return LZ4HC_compress_generic (state, source, dest, inputSize, 0, noLimit);
+}
+
+
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; // Error : state is not aligned for pointers (32 or 64 bits)
+ LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
+ return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, limitedOutput);
+}
+
+
+//****************************
+// Stream functions
+//****************************
+
+int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, noLimit);
+}
+
+int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, limitedOutput);
+}
+
-/*\r
- LZ4 HC - High Compression Mode of LZ4\r
- Header File\r
- Copyright (C) 2011-2013, Yann Collet.\r
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)\r
-\r
- Redistribution and use in source and binary forms, with or without\r
- modification, are permitted provided that the following conditions are\r
- met:\r
-\r
- * Redistributions of source code must retain the above copyright\r
- notice, this list of conditions and the following disclaimer.\r
- * Redistributions in binary form must reproduce the above\r
- copyright notice, this list of conditions and the following disclaimer\r
- in the documentation and/or other materials provided with the\r
- distribution.\r
-\r
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\r
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\r
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\r
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\r
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
- You can contact the author at :\r
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html\r
- - LZ4 source repository : http://code.google.com/p/lz4/\r
-*/\r
-#pragma once\r
-\r
-\r
-#if defined (__cplusplus)\r
-extern "C" {\r
-#endif\r
-\r
-\r
-int LZ4_compressHC (const char* source, char* dest, int inputSize);\r
-/*\r
-LZ4_compressHC :\r
- return : the number of bytes in compressed buffer dest\r
- or 0 if compression fails.\r
- note : destination buffer must be already allocated. \r
- To avoid any problem, size it to handle worst cases situations (input data not compressible)\r
- Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")\r
-*/\r
-\r
-int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);\r
-/*\r
-LZ4_compress_limitedOutput() :\r
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.\r
- If it cannot achieve it, compression will stop, and result of the function will be zero.\r
- This function never writes outside of provided output buffer.\r
-\r
- inputSize : Max supported value is 1 GB\r
- maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)\r
- return : the number of output bytes written in buffer 'dest'\r
- or 0 if compression fails.\r
-*/\r
-\r
-\r
-/* Note :\r
-Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)\r
-*/\r
-\r
-\r
-/* Advanced Functions */\r
-\r
-void* LZ4_createHC (const char* inputBuffer);\r
-int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);\r
-int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);\r
-char* LZ4_slideInputBufferHC (void* LZ4HC_Data);\r
-int LZ4_freeHC (void* LZ4HC_Data);\r
-\r
-/* \r
-These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.\r
-In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :\r
-\r
-void* LZ4_createHC (const char* inputBuffer);\r
-The result of the function is the (void*) pointer on the LZ4HC Data Structure.\r
-This pointer will be needed in all other functions.\r
-If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.\r
-The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.\r
-The input buffer must be already allocated, and size at least 192KB.\r
-'inputBuffer' will also be the 'const char* source' of the first block.\r
-\r
-All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.\r
-To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().\r
-Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(), \r
-but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.\r
-If next block does not begin immediately after the previous one, the compression will fail (return 0).\r
-\r
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to : \r
-char* LZ4_slideInputBufferHC(void* LZ4HC_Data);\r
-must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.\r
-Note that, for this function to work properly, minimum size of an input buffer must be 192KB.\r
-==> The memory position where the next input data block must start is provided as the result of the function.\r
-\r
-Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.\r
-\r
-When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.\r
-*/\r
-\r
-\r
-#if defined (__cplusplus)\r
-}\r
-#endif\r
+/*
+ LZ4 HC - High Compression Mode of LZ4
+ Header File
+ Copyright (C) 2011-2013, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+int LZ4_compressHC (const char* source, char* dest, int inputSize);
+/*
+LZ4_compressHC :
+ return : the number of bytes in compressed buffer dest
+ or 0 if compression fails.
+ note : destination buffer must be already allocated.
+ To avoid any problem, size it to handle worst cases situations (input data not compressible)
+ Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
+*/
+
+int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
+/*
+LZ4_compress_limitedOutput() :
+ Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This function never writes outside of provided output buffer.
+
+ inputSize : Max supported value is 1 GB
+ maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
+ return : the number of output bytes written in buffer 'dest'
+ or 0 if compression fails.
+*/
+
+
+/* Note :
+Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
+*/
+
+
+//*****************************
+// Using an external allocation
+//*****************************
+int LZ4_sizeofStateHC();
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofStateHC();
+
+Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+//****************************
+// Streaming Functions
+//****************************
+
+void* LZ4_createHC (const char* inputBuffer);
+int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
+char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+int LZ4_freeHC (void* LZ4HC_Data);
+
+/*
+These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
+In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
+
+void* LZ4_createHC (const char* inputBuffer);
+The result of the function is the (void*) pointer on the LZ4HC Data Structure.
+This pointer will be needed in all other functions.
+If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
+To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
+Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
+but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
+If next block does not begin immediately after the previous one, the compression will fail (return 0).
+
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
+must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
+Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
+==> The memory position where the next input data block must start is provided as the result of the function.
+
+Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
+
+When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
+*/
+
+int LZ4_sizeofStreamStateHC();
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_createHC (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
+Note also that space must be aligned for pointers (32 or 64 bits).
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned for pointers (32 or 64 bits), and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamStateHC() must be 0 is OK.
+Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
+*/
+
+
+#if defined (__cplusplus)
+}
+#endif
-/*\r
-xxHash - Fast Hash algorithm\r
-Copyright (C) 2012-2013, Yann Collet.\r
-BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)\r
-\r
-Redistribution and use in source and binary forms, with or without\r
-modification, are permitted provided that the following conditions are\r
-met:\r
-\r
-* Redistributions of source code must retain the above copyright\r
-notice, this list of conditions and the following disclaimer.\r
-* Redistributions in binary form must reproduce the above\r
-copyright notice, this list of conditions and the following disclaimer\r
-in the documentation and/or other materials provided with the\r
-distribution.\r
-\r
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\r
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\r
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\r
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\r
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
-You can contact the author at :\r
-- xxHash source repository : http://code.google.com/p/xxhash/\r
-*/\r
-\r
-\r
-//**************************************\r
-// Tuning parameters\r
-//**************************************\r
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.\r
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.\r
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.\r
-// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).\r
-#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)\r
-# define XXH_USE_UNALIGNED_ACCESS 1\r
-#endif\r
-\r
-// XXH_ACCEPT_NULL_INPUT_POINTER :\r
-// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.\r
-// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.\r
-// This option has a very small performance cost (only measurable on small inputs).\r
-// By default, this option is disabled. To enable it, uncomment below define :\r
-//#define XXH_ACCEPT_NULL_INPUT_POINTER 1\r
-\r
-// XXH_FORCE_NATIVE_FORMAT :\r
-// By default, xxHash library provides endian-independant Hash values, based on little-endian convention.\r
-// Results are therefore identical for little-endian and big-endian CPU.\r
-// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.\r
-// Should endian-independance be of no importance for your application, you may set the #define below to 1.\r
-// It will improve speed for Big-endian CPU.\r
-// This option has no impact on Little_Endian CPU.\r
-#define XXH_FORCE_NATIVE_FORMAT 0\r
-\r
-\r
-//**************************************\r
-// Compiler Specific Options\r
-//**************************************\r
-// Disable some Visual warning messages\r
-#ifdef _MSC_VER // Visual Studio\r
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant\r
-#endif\r
-\r
-#ifdef _MSC_VER // Visual Studio\r
-# define forceinline static __forceinline\r
-#else \r
-# ifdef __GNUC__\r
-# define forceinline static inline __attribute__((always_inline))\r
-# else\r
-# define forceinline static inline\r
-# endif\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Includes & Memory related functions\r
-//**************************************\r
-#include "xxhash.h"\r
-// Modify the local functions below should you wish to use some other memory related routines\r
-// for malloc(), free()\r
-#include <stdlib.h>\r
-forceinline void* XXH_malloc(size_t s) { return malloc(s); }\r
-forceinline void XXH_free (void* p) { free(p); }\r
-// for memcpy()\r
-#include <string.h>\r
-forceinline void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }\r
-\r
-\r
-//**************************************\r
-// Basic Types\r
-//**************************************\r
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99\r
-# include <stdint.h>\r
- typedef uint8_t BYTE;\r
- typedef uint16_t U16;\r
- typedef uint32_t U32;\r
- typedef int32_t S32;\r
- typedef uint64_t U64;\r
-#else\r
- typedef unsigned char BYTE;\r
- typedef unsigned short U16;\r
- typedef unsigned int U32;\r
- typedef signed int S32;\r
- typedef unsigned long long U64;\r
-#endif\r
-\r
-#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS)\r
-# define _PACKED __attribute__ ((packed))\r
-#else\r
-# define _PACKED\r
-#endif\r
-\r
-#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)\r
-# ifdef __IBMC__\r
-# pragma pack(1)\r
-# else\r
-# pragma pack(push, 1)\r
-# endif\r
-#endif\r
-\r
-typedef struct _U32_S { U32 v; } _PACKED U32_S;\r
-\r
-#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)\r
-# pragma pack(pop)\r
-#endif\r
-\r
-#define A32(x) (((U32_S *)(x))->v)\r
-\r
-\r
-//***************************************\r
-// Compiler-specific Functions and Macros\r
-//***************************************\r
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)\r
-\r
-// Note : although _rotl exists for minGW (GCC under windows), performance seems poor\r
-#if defined(_MSC_VER)\r
-# define XXH_rotl32(x,r) _rotl(x,r)\r
-#else\r
-# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))\r
-#endif\r
-\r
-#if defined(_MSC_VER) // Visual Studio\r
-# define XXH_swap32 _byteswap_ulong\r
-#elif GCC_VERSION >= 403\r
-# define XXH_swap32 __builtin_bswap32\r
-#else\r
-static inline U32 XXH_swap32 (U32 x) {\r
- return ((x << 24) & 0xff000000 ) |\r
- ((x << 8) & 0x00ff0000 ) |\r
- ((x >> 8) & 0x0000ff00 ) |\r
- ((x >> 24) & 0x000000ff );}\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Constants\r
-//**************************************\r
-#define PRIME32_1 2654435761U\r
-#define PRIME32_2 2246822519U\r
-#define PRIME32_3 3266489917U\r
-#define PRIME32_4 668265263U\r
-#define PRIME32_5 374761393U\r
-\r
-\r
-//**************************************\r
-// Architecture Macros\r
-//**************************************\r
-typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;\r
-#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch\r
- static const int one = 1;\r
-# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one))\r
-#endif\r
-\r
-\r
-//**************************************\r
-// Macros\r
-//**************************************\r
-#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations\r
-\r
-\r
-//****************************\r
-// Memory reads\r
-//****************************\r
-typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;\r
-\r
-forceinline U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align)\r
-{ \r
- if (align==XXH_unaligned)\r
- return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); \r
- else\r
- return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); \r
-}\r
-\r
-forceinline U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); }\r
-\r
-\r
-//****************************\r
-// Simple Hash Functions\r
-//****************************\r
-forceinline U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align)\r
-{\r
- const BYTE* p = (const BYTE*)input;\r
- const BYTE* const bEnd = p + len;\r
- U32 h32;\r
-\r
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER\r
- if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; }\r
-#endif\r
-\r
- if (len>=16)\r
- {\r
- const BYTE* const limit = bEnd - 16;\r
- U32 v1 = seed + PRIME32_1 + PRIME32_2;\r
- U32 v2 = seed + PRIME32_2;\r
- U32 v3 = seed + 0;\r
- U32 v4 = seed - PRIME32_1;\r
-\r
- do\r
- {\r
- v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;\r
- v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;\r
- v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;\r
- v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;\r
- } while (p<=limit);\r
-\r
- h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);\r
- }\r
- else\r
- {\r
- h32 = seed + PRIME32_5;\r
- }\r
-\r
- h32 += (U32) len;\r
-\r
- while (p<=bEnd-4)\r
- {\r
- h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3;\r
- h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;\r
- p+=4;\r
- }\r
-\r
- while (p<bEnd)\r
- {\r
- h32 += (*p) * PRIME32_5;\r
- h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;\r
- p++;\r
- }\r
-\r
- h32 ^= h32 >> 15;\r
- h32 *= PRIME32_2;\r
- h32 ^= h32 >> 13;\r
- h32 *= PRIME32_3;\r
- h32 ^= h32 >> 16;\r
-\r
- return h32;\r
-}\r
-\r
-\r
-U32 XXH32(const void* input, int len, U32 seed)\r
-{\r
-#if 0\r
- // Simple version, good for code maintenance, but unfortunately slow for small inputs\r
- void* state = XXH32_init(seed);\r
- XXH32_update(state, input, len);\r
- return XXH32_digest(state);\r
-#else\r
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;\r
-\r
-# if !defined(XXH_USE_UNALIGNED_ACCESS)\r
- if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage\r
- {\r
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)\r
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);\r
- else\r
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);\r
- }\r
-# endif\r
-\r
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)\r
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);\r
- else\r
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);\r
-#endif\r
-}\r
-\r
-\r
-//****************************\r
-// Advanced Hash Functions\r
-//****************************\r
-\r
-struct XXH_state32_t\r
-{\r
- U64 total_len;\r
- U32 seed;\r
- U32 v1;\r
- U32 v2;\r
- U32 v3;\r
- U32 v4;\r
- int memsize;\r
- char memory[16];\r
-};\r
-\r
-\r
-int XXH32_sizeofState() \r
-{\r
- XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough\r
- return sizeof(struct XXH_state32_t); \r
-}\r
-\r
-\r
-XXH_errorcode XXH32_resetState(void* state_in, U32 seed)\r
-{ \r
- struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;\r
- state->seed = seed;\r
- state->v1 = seed + PRIME32_1 + PRIME32_2;\r
- state->v2 = seed + PRIME32_2;\r
- state->v3 = seed + 0;\r
- state->v4 = seed - PRIME32_1;\r
- state->total_len = 0;\r
- state->memsize = 0;\r
- return XXH_OK;\r
-}\r
-\r
-\r
-void* XXH32_init (U32 seed)\r
-{\r
- void* state = XXH_malloc (sizeof(struct XXH_state32_t));\r
- XXH32_resetState(state, seed);\r
- return state;\r
-}\r
-\r
-\r
-forceinline XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian)\r
-{\r
- struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;\r
- const BYTE* p = (const BYTE*)input;\r
- const BYTE* const bEnd = p + len;\r
-\r
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER\r
- if (input==NULL) return XXH_ERROR;\r
-#endif\r
-\r
- state->total_len += len;\r
-\r
- if (state->memsize + len < 16) // fill in tmp buffer\r
- {\r
- XXH_memcpy(state->memory + state->memsize, input, len);\r
- state->memsize += len;\r
- return XXH_OK;\r
- }\r
-\r
- if (state->memsize) // some data left from previous update\r
- {\r
- XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize);\r
- {\r
- const U32* p32 = (const U32*)state->memory;\r
- state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++;\r
- state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; \r
- state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++;\r
- state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++;\r
- }\r
- p += 16-state->memsize;\r
- state->memsize = 0;\r
- }\r
-\r
- if (p <= bEnd-16)\r
- {\r
- const BYTE* const limit = bEnd - 16;\r
- U32 v1 = state->v1;\r
- U32 v2 = state->v2;\r
- U32 v3 = state->v3;\r
- U32 v4 = state->v4;\r
-\r
- do\r
- {\r
- v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;\r
- v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;\r
- v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;\r
- v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;\r
- } while (p<=limit);\r
-\r
- state->v1 = v1;\r
- state->v2 = v2;\r
- state->v3 = v3;\r
- state->v4 = v4;\r
- }\r
-\r
- if (p < bEnd)\r
- {\r
- XXH_memcpy(state->memory, p, bEnd-p);\r
- state->memsize = (int)(bEnd-p);\r
- }\r
-\r
- return XXH_OK;\r
-}\r
-\r
-XXH_errorcode XXH32_update (void* state_in, const void* input, int len)\r
-{\r
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;\r
- \r
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)\r
- return XXH32_update_endian(state_in, input, len, XXH_littleEndian);\r
- else\r
- return XXH32_update_endian(state_in, input, len, XXH_bigEndian);\r
-}\r
-\r
-\r
-\r
-forceinline U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian)\r
-{\r
- struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;\r
- const BYTE * p = (const BYTE*)state->memory;\r
- BYTE* bEnd = (BYTE*)state->memory + state->memsize;\r
- U32 h32;\r
-\r
- if (state->total_len >= 16)\r
- {\r
- h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);\r
- }\r
- else\r
- {\r
- h32 = state->seed + PRIME32_5;\r
- }\r
-\r
- h32 += (U32) state->total_len;\r
-\r
- while (p<=bEnd-4)\r
- {\r
- h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3;\r
- h32 = XXH_rotl32(h32, 17) * PRIME32_4;\r
- p+=4;\r
- }\r
-\r
- while (p<bEnd)\r
- {\r
- h32 += (*p) * PRIME32_5;\r
- h32 = XXH_rotl32(h32, 11) * PRIME32_1;\r
- p++;\r
- }\r
-\r
- h32 ^= h32 >> 15;\r
- h32 *= PRIME32_2;\r
- h32 ^= h32 >> 13;\r
- h32 *= PRIME32_3;\r
- h32 ^= h32 >> 16;\r
-\r
- return h32;\r
-}\r
-\r
-\r
-U32 XXH32_intermediateDigest (void* state_in)\r
-{\r
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;\r
- \r
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)\r
- return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian);\r
- else\r
- return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian);\r
-}\r
-\r
-\r
-U32 XXH32_digest (void* state_in)\r
-{\r
- U32 h32 = XXH32_intermediateDigest(state_in);\r
-\r
- XXH_free(state_in);\r
-\r
- return h32;\r
-}\r
+/*
+xxHash - Fast Hash algorithm
+Copyright (C) 2012-2014, Yann Collet.
+BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash source repository : http://code.google.com/p/xxhash/
+*/
+
+
+//**************************************
+// Tuning parameters
+//**************************************
+// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
+// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
+// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
+#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+# define XXH_USE_UNALIGNED_ACCESS 1
+#endif
+
+// XXH_ACCEPT_NULL_INPUT_POINTER :
+// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+// This option has a very small performance cost (only measurable on small inputs).
+// By default, this option is disabled. To enable it, uncomment below define :
+//#define XXH_ACCEPT_NULL_INPUT_POINTER 1
+
+// XXH_FORCE_NATIVE_FORMAT :
+// By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+// Results are therefore identical for little-endian and big-endian CPU.
+// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+// Should endian-independance be of no importance for your application, you may set the #define below to 1.
+// It will improve speed for Big-endian CPU.
+// This option has no impact on Little_Endian CPU.
+#define XXH_FORCE_NATIVE_FORMAT 0
+
+
+//**************************************
+// Compiler Specific Options
+//**************************************
+// Disable some Visual warning messages
+#ifdef _MSC_VER // Visual Studio
+# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+#endif
+
+#ifdef _MSC_VER // Visual Studio
+# define FORCE_INLINE static __forceinline
+#else
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
+#endif
+
+
+//**************************************
+// Includes & Memory related functions
+//**************************************
+#include "xxhash.h"
+// Modify the local functions below should you wish to use some other memory related routines
+// for malloc(), free()
+#include <stdlib.h>
+FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); }
+FORCE_INLINE void XXH_free (void* p) { free(p); }
+// for memcpy()
+#include <string.h>
+FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+
+//**************************************
+// Basic Types
+//**************************************
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS)
+# define _PACKED __attribute__ ((packed))
+#else
+# define _PACKED
+#endif
+
+#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# ifdef __IBMC__
+# pragma pack(1)
+# else
+# pragma pack(push, 1)
+# endif
+#endif
+
+typedef struct _U32_S { U32 v; } _PACKED U32_S;
+
+#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
+# pragma pack(pop)
+#endif
+
+#define A32(x) (((U32_S *)(x))->v)
+
+
+//***************************************
+// Compiler-specific Functions and Macros
+//***************************************
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+// Note : although _rotl exists for minGW (GCC under windows), performance seems poor
+#if defined(_MSC_VER)
+# define XXH_rotl32(x,r) _rotl(x,r)
+#else
+# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#endif
+
+#if defined(_MSC_VER) // Visual Studio
+# define XXH_swap32 _byteswap_ulong
+#elif GCC_VERSION >= 403
+# define XXH_swap32 __builtin_bswap32
+#else
+static inline U32 XXH_swap32 (U32 x) {
+ return ((x << 24) & 0xff000000 ) |
+ ((x << 8) & 0x00ff0000 ) |
+ ((x >> 8) & 0x0000ff00 ) |
+ ((x >> 24) & 0x000000ff );}
+#endif
+
+
+//**************************************
+// Constants
+//**************************************
+#define PRIME32_1 2654435761U
+#define PRIME32_2 2246822519U
+#define PRIME32_3 3266489917U
+#define PRIME32_4 668265263U
+#define PRIME32_5 374761393U
+
+
+//**************************************
+// Architecture Macros
+//**************************************
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch
+ static const int one = 1;
+# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one))
+#endif
+
+
+//**************************************
+// Macros
+//**************************************
+#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations
+
+
+//****************************
+// Memory reads
+//****************************
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr));
+ else
+ return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); }
+
+
+//****************************
+// Simple Hash Functions
+//****************************
+FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align)
+{
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
+ U32 h32;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; }
+#endif
+
+ if (len>=16)
+ {
+ const BYTE* const limit = bEnd - 16;
+ U32 v1 = seed + PRIME32_1 + PRIME32_2;
+ U32 v2 = seed + PRIME32_2;
+ U32 v3 = seed + 0;
+ U32 v4 = seed - PRIME32_1;
+
+ do
+ {
+ v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;
+ v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;
+ v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;
+ v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;
+ } while (p<=limit);
+
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ }
+ else
+ {
+ h32 = seed + PRIME32_5;
+ }
+
+ h32 += (U32) len;
+
+ while (p<=bEnd-4)
+ {
+ h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3;
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
+ p+=4;
+ }
+
+ while (p<bEnd)
+ {
+ h32 += (*p) * PRIME32_5;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+
+U32 XXH32(const void* input, int len, U32 seed)
+{
+#if 0
+ // Simple version, good for code maintenance, but unfortunately slow for small inputs
+ void* state = XXH32_init(seed);
+ XXH32_update(state, input, len);
+ return XXH32_digest(state);
+#else
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+# if !defined(XXH_USE_UNALIGNED_ACCESS)
+ if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage
+ {
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ }
+# endif
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+
+//****************************
+// Advanced Hash Functions
+//****************************
+
+struct XXH_state32_t
+{
+ U64 total_len;
+ U32 seed;
+ U32 v1;
+ U32 v2;
+ U32 v3;
+ U32 v4;
+ int memsize;
+ char memory[16];
+};
+
+
+int XXH32_sizeofState()
+{
+ XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough
+ return sizeof(struct XXH_state32_t);
+}
+
+
+XXH_errorcode XXH32_resetState(void* state_in, U32 seed)
+{
+ struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
+ state->seed = seed;
+ state->v1 = seed + PRIME32_1 + PRIME32_2;
+ state->v2 = seed + PRIME32_2;
+ state->v3 = seed + 0;
+ state->v4 = seed - PRIME32_1;
+ state->total_len = 0;
+ state->memsize = 0;
+ return XXH_OK;
+}
+
+
+void* XXH32_init (U32 seed)
+{
+ void* state = XXH_malloc (sizeof(struct XXH_state32_t));
+ XXH32_resetState(state, seed);
+ return state;
+}
+
+
+FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian)
+{
+ struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (input==NULL) return XXH_ERROR;
+#endif
+
+ state->total_len += len;
+
+ if (state->memsize + len < 16) // fill in tmp buffer
+ {
+ XXH_memcpy(state->memory + state->memsize, input, len);
+ state->memsize += len;
+ return XXH_OK;
+ }
+
+ if (state->memsize) // some data left from previous update
+ {
+ XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize);
+ {
+ const U32* p32 = (const U32*)state->memory;
+ state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++;
+ state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++;
+ state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++;
+ state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++;
+ }
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= bEnd-16)
+ {
+ const BYTE* const limit = bEnd - 16;
+ U32 v1 = state->v1;
+ U32 v2 = state->v2;
+ U32 v3 = state->v3;
+ U32 v4 = state->v4;
+
+ do
+ {
+ v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4;
+ v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4;
+ v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4;
+ v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4;
+ } while (p<=limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < bEnd)
+ {
+ XXH_memcpy(state->memory, p, bEnd-p);
+ state->memsize = (int)(bEnd-p);
+ }
+
+ return XXH_OK;
+}
+
+XXH_errorcode XXH32_update (void* state_in, const void* input, int len)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+ else
+ return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian)
+{
+ struct XXH_state32_t * state = (struct XXH_state32_t *) state_in;
+ const BYTE * p = (const BYTE*)state->memory;
+ BYTE* bEnd = (BYTE*)state->memory + state->memsize;
+ U32 h32;
+
+ if (state->total_len >= 16)
+ {
+ h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+ }
+ else
+ {
+ h32 = state->seed + PRIME32_5;
+ }
+
+ h32 += (U32) state->total_len;
+
+ while (p<=bEnd-4)
+ {
+ h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3;
+ h32 = XXH_rotl32(h32, 17) * PRIME32_4;
+ p+=4;
+ }
+
+ while (p<bEnd)
+ {
+ h32 += (*p) * PRIME32_5;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+
+U32 XXH32_intermediateDigest (void* state_in)
+{
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian);
+ else
+ return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian);
+}
+
+
+U32 XXH32_digest (void* state_in)
+{
+ U32 h32 = XXH32_intermediateDigest(state_in);
+
+ XXH_free(state_in);
+
+ return h32;
+}
-/*\r
- xxHash - Fast Hash algorithm\r
- Header File\r
- Copyright (C) 2012-2013, Yann Collet.\r
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)\r
-\r
- Redistribution and use in source and binary forms, with or without\r
- modification, are permitted provided that the following conditions are\r
- met:\r
- \r
- * Redistributions of source code must retain the above copyright\r
- notice, this list of conditions and the following disclaimer.\r
- * Redistributions in binary form must reproduce the above\r
- copyright notice, this list of conditions and the following disclaimer\r
- in the documentation and/or other materials provided with the\r
- distribution.\r
- \r
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\r
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\r
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\r
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\r
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\r
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\r
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\r
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\r
-\r
- You can contact the author at :\r
- - xxHash source repository : http://code.google.com/p/xxhash/\r
-*/\r
-\r
-/* Notice extracted from xxHash homepage :\r
-\r
-xxHash is an extremely fast Hash algorithm, running at RAM speed limits.\r
-It also successfully passes all tests from the SMHasher suite.\r
-\r
-Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)\r
-\r
-Name Speed Q.Score Author\r
-xxHash 5.4 GB/s 10\r
-CrapWow 3.2 GB/s 2 Andrew\r
-MumurHash 3a 2.7 GB/s 10 Austin Appleby\r
-SpookyHash 2.0 GB/s 10 Bob Jenkins\r
-SBox 1.4 GB/s 9 Bret Mulvey\r
-Lookup3 1.2 GB/s 9 Bob Jenkins\r
-SuperFastHash 1.2 GB/s 1 Paul Hsieh\r
-CityHash64 1.05 GB/s 10 Pike & Alakuijala\r
-FNV 0.55 GB/s 5 Fowler, Noll, Vo\r
-CRC32 0.43 GB/s 9\r
-MD5-32 0.33 GB/s 10 Ronald L. Rivest\r
-SHA1-32 0.28 GB/s 10\r
-\r
-Q.Score is a measure of quality of the hash function. \r
-It depends on successfully passing SMHasher test set. \r
-10 is a perfect score.\r
-*/\r
-\r
-#pragma once\r
-\r
-#if defined (__cplusplus)\r
-extern "C" {\r
-#endif\r
-\r
-\r
-//****************************\r
-// Type\r
-//****************************\r
-typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;\r
-\r
-\r
-\r
-//****************************\r
-// Simple Hash Functions\r
-//****************************\r
-\r
-unsigned int XXH32 (const void* input, int len, unsigned int seed);\r
-\r
-/*\r
-XXH32() :\r
- Calculate the 32-bits hash of sequence of length "len" stored at memory address "input".\r
- The memory between input & input+len must be valid (allocated and read-accessible).\r
- "seed" can be used to alter the result predictably.\r
- This function successfully passes all SMHasher tests.\r
- Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s\r
- Note that "len" is type "int", which means it is limited to 2^31-1.\r
- If your data is larger, use the advanced functions below.\r
-*/\r
-\r
-\r
-\r
-//****************************\r
-// Advanced Hash Functions\r
-//****************************\r
-\r
-void* XXH32_init (unsigned int seed);\r
-XXH_errorcode XXH32_update (void* state, const void* input, int len);\r
-unsigned int XXH32_digest (void* state);\r
-\r
-/*\r
-These functions calculate the xxhash of an input provided in several small packets,\r
-as opposed to an input provided as a single block.\r
-\r
-It must be started with :\r
-void* XXH32_init()\r
-The function returns a pointer which holds the state of calculation.\r
-\r
-This pointer must be provided as "void* state" parameter for XXH32_update().\r
-XXH32_update() can be called as many times as necessary.\r
-The user must provide a valid (allocated) input.\r
-The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.\r
-Note that "len" is type "int", which means it is limited to 2^31-1. \r
-If your data is larger, it is recommended to chunk your data into blocks \r
-of size for example 2^30 (1GB) to avoid any "int" overflow issue.\r
-\r
-Finally, you can end the calculation anytime, by using XXH32_digest().\r
-This function returns the final 32-bits hash.\r
-You must provide the same "void* state" parameter created by XXH32_init().\r
-Memory will be freed by XXH32_digest().\r
-*/\r
-\r
-\r
-int XXH32_sizeofState();\r
-XXH_errorcode XXH32_resetState(void* state, unsigned int seed);\r
-\r
-#define XXH32_SIZEOFSTATE 48\r
-typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t;\r
-/*\r
-These functions allow user application to make its own allocation for state.\r
-\r
-XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state.\r
-Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer.\r
-This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state.\r
-\r
-For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()),\r
-use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields.\r
-*/\r
-\r
-\r
-unsigned int XXH32_intermediateDigest (void* state);\r
-/*\r
-This function does the same as XXH32_digest(), generating a 32-bit hash,\r
-but preserve memory context.\r
-This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update().\r
-To free memory context, use XXH32_digest(), or free().\r
-*/\r
-\r
-\r
-\r
-//****************************\r
-// Deprecated function names\r
-//****************************\r
-// The following translations are provided to ease code transition\r
-// You are encouraged to no longer this function names\r
-#define XXH32_feed XXH32_update\r
-#define XXH32_result XXH32_digest\r
-#define XXH32_getIntermediateResult XXH32_intermediateDigest\r
-\r
-\r
-\r
-#if defined (__cplusplus)\r
-}\r
-#endif\r
+/*
+ xxHash - Fast Hash algorithm
+ Header File
+ Copyright (C) 2012-2014, Yann Collet.
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - xxHash source repository : http://code.google.com/p/xxhash/
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name Speed Q.Score Author
+xxHash 5.4 GB/s 10
+CrapWow 3.2 GB/s 2 Andrew
+MumurHash 3a 2.7 GB/s 10 Austin Appleby
+SpookyHash 2.0 GB/s 10 Bob Jenkins
+SBox 1.4 GB/s 9 Bret Mulvey
+Lookup3 1.2 GB/s 9 Bob Jenkins
+SuperFastHash 1.2 GB/s 1 Paul Hsieh
+CityHash64 1.05 GB/s 10 Pike & Alakuijala
+FNV 0.55 GB/s 5 Fowler, Noll, Vo
+CRC32 0.43 GB/s 9
+MD5-32 0.33 GB/s 10 Ronald L. Rivest
+SHA1-32 0.28 GB/s 10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+*/
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+//****************************
+// Type
+//****************************
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+
+//****************************
+// Simple Hash Functions
+//****************************
+
+unsigned int XXH32 (const void* input, int len, unsigned int seed);
+
+/*
+XXH32() :
+ Calculate the 32-bits hash of sequence of length "len" stored at memory address "input".
+ The memory between input & input+len must be valid (allocated and read-accessible).
+ "seed" can be used to alter the result predictably.
+ This function successfully passes all SMHasher tests.
+ Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
+ Note that "len" is type "int", which means it is limited to 2^31-1.
+ If your data is larger, use the advanced functions below.
+*/
+
+
+
+//****************************
+// Advanced Hash Functions
+//****************************
+
+void* XXH32_init (unsigned int seed);
+XXH_errorcode XXH32_update (void* state, const void* input, int len);
+unsigned int XXH32_digest (void* state);
+
+/*
+These functions calculate the xxhash of an input provided in several small packets,
+as opposed to an input provided as a single block.
+
+It must be started with :
+void* XXH32_init()
+The function returns a pointer which holds the state of calculation.
+
+This pointer must be provided as "void* state" parameter for XXH32_update().
+XXH32_update() can be called as many times as necessary.
+The user must provide a valid (allocated) input.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+Note that "len" is type "int", which means it is limited to 2^31-1.
+If your data is larger, it is recommended to chunk your data into blocks
+of size for example 2^30 (1GB) to avoid any "int" overflow issue.
+
+Finally, you can end the calculation anytime, by using XXH32_digest().
+This function returns the final 32-bits hash.
+You must provide the same "void* state" parameter created by XXH32_init().
+Memory will be freed by XXH32_digest().
+*/
+
+
+int XXH32_sizeofState();
+XXH_errorcode XXH32_resetState(void* state, unsigned int seed);
+
+#define XXH32_SIZEOFSTATE 48
+typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t;
+/*
+These functions allow user application to make its own allocation for state.
+
+XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state.
+Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer.
+This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state.
+
+For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()),
+use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields.
+*/
+
+
+unsigned int XXH32_intermediateDigest (void* state);
+/*
+This function does the same as XXH32_digest(), generating a 32-bit hash,
+but preserve memory context.
+This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update().
+To free memory context, use XXH32_digest(), or free().
+*/
+
+
+
+//****************************
+// Deprecated function names
+//****************************
+// The following translations are provided to ease code transition
+// You are encouraged to no longer this function names
+#define XXH32_feed XXH32_update
+#define XXH32_result XXH32_digest
+#define XXH32_getIntermediateResult XXH32_intermediateDigest
+
+
+
+#if defined (__cplusplus)
+}
+#endif
projects / platform / upstream / lz4.git / commitdiff