//****************************\r
// Includes\r
//****************************\r
-#include <stdio.h> // fprintf, fopen, fread, _fileno(?)\r
-#include <stdlib.h> // malloc\r
-#include <string.h> // strcmp\r
-#include <time.h> // clock\r
+#include <stdio.h> // fprintf, fopen, fread, _fileno(?)\r
+#include <stdlib.h> // malloc\r
+#include <string.h> // strcmp\r
+#include <time.h> // clock\r
#ifdef _WIN32\r
-#include <io.h> // _setmode\r
-#include <fcntl.h> // _O_BINARY\r
+#include <io.h> // _setmode\r
+#include <fcntl.h> // _O_BINARY\r
#endif\r
#include "lz4.h"\r
#include "lz4hc.h"\r
#define EXTENSION ".lz4"\r
#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED\r
\r
-#define CACHELINE 64\r
-#define MAGICNUMBER_SIZE 4\r
-#define LZ4S_MAGICNUMBER 0x184D2204\r
-#define LZ4S_BLOCKSIZEID_DEFAULT 7\r
-#define LZ4S_CHECKSUM_SEED 0\r
+#define KB *(1U<<10)\r
+#define MB *(1U<<20)\r
+#define GB *(1U<<30)\r
\r
-#define LZ4S_SKIPPABLE0 0x184D2A50\r
-#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0\r
-\r
-#define LEGACY_MAGICNUMBER 0x184C2102\r
-#define LEGACY_BLOCKSIZE (8<<20) // 8 MB\r
-\r
-#define _1BIT 1\r
+#define _1BIT 0x01\r
#define _2BITS 0x03\r
#define _3BITS 0x07\r
#define _4BITS 0x0F\r
#define _8BITS 0xFF\r
\r
+#define MAGICNUMBER_SIZE 4\r
+#define LZ4S_MAGICNUMBER 0x184D2204\r
+#define LZ4S_SKIPPABLE0 0x184D2A50\r
+#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0\r
+#define LEGACY_MAGICNUMBER 0x184C2102\r
+\r
+#define CACHELINE 64\r
+#define LEGACY_BLOCKSIZE (8 MB)\r
+#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)\r
+#define LZ4S_BLOCKSIZEID_DEFAULT 7\r
+#define LZ4S_CHECKSUM_SEED 0\r
+#define LZ4S_EOS 0\r
+#define LZ4S_MAXHEADERSIZE (4+2+8+4+1)\r
+\r
+\r
//**************************************\r
// Architecture Macros\r
//**************************************\r
static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;\r
static int blockChecksum = 0;\r
static int streamChecksum = 1;\r
+static int blockIndependence = 1;\r
\r
//**************************************\r
// Exceptions\r
DISPLAY( "\nAdvanced options :\n");\r
DISPLAY( " -t : test compressed file \n");\r
DISPLAY( " -B# : Block size [4-7](default : 7)\n");\r
- DISPLAY( " -x : enable block checksum (default:disabled)\n");\r
- DISPLAY( " -nx : disable stream checksum (default:enabled)\n");\r
+ DISPLAY( " -BD : Block dependency (improve compression ratio)\n");\r
+ DISPLAY( " -BX : enable block checksum (default:disabled)\n");\r
+ DISPLAY( " -Sx : disable stream checksum (default:enabled)\n");\r
DISPLAY( " -b# : benchmark files, using # [0-1] compression level\n");\r
DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n");\r
DISPLAY( "input : can be 'stdin' (pipe) or a filename\n");\r
DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n");\r
- DISPLAY( " example : lz4c -hc stdin compressedfile.lz4");\r
+ DISPLAY( " example 1 : lz4c -hc stdin compressedfile.lz4\n");\r
+ DISPLAY( " example 2 : lz4c -hcyB4D filename \n");\r
return 0;\r
}\r
\r
-\r
int badusage(char* exename)\r
{\r
DISPLAY("Wrong parameters\n");\r
}\r
\r
\r
-static int LZ4S_GetBlocksize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }\r
+static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); }\r
static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; }\r
+static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; }\r
\r
\r
int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput)\r
{\r
case 0 : compressionFunction = LZ4_compress; break;\r
case 1 : compressionFunction = LZ4_compressHC; break;\r
- default : compressionFunction = LZ4_compress;\r
+ default: compressionFunction = LZ4_compress;\r
}\r
start = clock();\r
get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
}\r
\r
\r
+int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel)\r
+{\r
+ void* (*initFunction) (const char*);\r
+ int (*compressionFunction)(void*, const char*, char*, int, int);\r
+ char* (*translateFunction) (void*);\r
+ int (*freeFunction) (void*);\r
+ void* ctx;\r
+ unsigned long long filesize = 0;\r
+ unsigned long long compressedfilesize = 0;\r
+ unsigned int checkbits;\r
+ char* in_buff, *in_start, *in_end;\r
+ char* out_buff;\r
+ FILE* finput;\r
+ FILE* foutput;\r
+ int errorcode;\r
+ int displayLevel = (compressionlevel>0);\r
+ clock_t start, end;\r
+ unsigned int blockSize, inputBufferSize;\r
+ size_t sizeCheck, header_size;\r
+ void* streamChecksumState=NULL;\r
+\r
+\r
+ // Init\r
+ start = clock();\r
+ switch (compressionlevel)\r
+ {\r
+ case 0 :\r
+ case 1 :\r
+ default:\r
+ initFunction = LZ4_createHC;\r
+ compressionFunction = LZ4_compressHC_limitedOutput_continue;\r
+ translateFunction = LZ4_slideInputBufferHC;\r
+ freeFunction = LZ4_freeHC;\r
+ }\r
+ errorcode = get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
+ if (errorcode) return errorcode;\r
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);\r
+\r
+ // Allocate Memory\r
+ inputBufferSize = blockSize + 64 KB;\r
+ if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE;\r
+ in_buff = (char*)malloc(inputBufferSize);\r
+ out_buff = (char*)malloc(blockSize+CACHELINE);\r
+ if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");\r
+ in_start = in_buff; in_end = in_buff + inputBufferSize;\r
+ if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
+ ctx = initFunction(in_buff);\r
+\r
+ // Write Archive Header\r
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention\r
+ *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')\r
+ *(out_buff+4) |= (blockIndependence & _1BIT) << 5;\r
+ *(out_buff+4) |= (blockChecksum & _1BIT) << 4;\r
+ *(out_buff+4) |= (streamChecksum & _1BIT) << 2;\r
+ *(out_buff+5) = (blockSizeId & _3BITS) << 4;\r
+ checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);\r
+ checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);\r
+ *(out_buff+6) = (unsigned char) checkbits;\r
+ header_size = 7;\r
+ sizeCheck = fwrite(out_buff, 1, header_size, foutput);\r
+ if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header");\r
+ compressedfilesize += header_size;\r
+\r
+ // Main Loop\r
+ while (1)\r
+ {\r
+ unsigned int outSize;\r
+ unsigned int inSize;\r
+ // Read Block\r
+ if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx);\r
+ inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput);\r
+ if( inSize<=0 ) break; // No more input : end of compression\r
+ filesize += inSize;\r
+ if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20));\r
+ if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize);\r
+\r
+ // Compress Block\r
+ outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1);\r
+ if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;\r
+ if (blockChecksum) compressedfilesize+=4;\r
+ if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100);\r
+\r
+ // Write Block\r
+ if (outSize > 0)\r
+ {\r
+ unsigned int checksum;\r
+ int sizeToWrite;\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize);\r
+ if (blockChecksum)\r
+ {\r
+ checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED);\r
+ * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum);\r
+ }\r
+ sizeToWrite = 4 + outSize + (4*blockChecksum);\r
+ sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput);\r
+ if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block");\r
+\r
+ }\r
+ else // Copy Original\r
+ {\r
+ unsigned int checksum;\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header");\r
+ sizeCheck = fwrite(in_start, 1, inSize, foutput);\r
+ if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block");\r
+ if (blockChecksum)\r
+ {\r
+ checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED);\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum");\r
+ }\r
+ }\r
+ in_start += inSize;\r
+ }\r
+\r
+ // End of Stream mark\r
+ * (unsigned int*) out_buff = LZ4S_EOS;\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");\r
+ compressedfilesize += 4;\r
+ if (streamChecksum)\r
+ {\r
+ unsigned int checksum = XXH32_digest(streamChecksumState);\r
+ * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);\r
+ sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
+ if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");\r
+ compressedfilesize += 4;\r
+ }\r
+\r
+ // Status\r
+ end = clock();\r
+ DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",\r
+ (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);\r
+ {\r
+ double seconds = (double)(end - start)/CLOCKS_PER_SEC;\r
+ DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);\r
+ }\r
+\r
+ // Close & Free\r
+ freeFunction(ctx);\r
+ free(in_buff);\r
+ free(out_buff);\r
+ fclose(finput);\r
+ fclose(foutput);\r
+\r
+ return 0;\r
+}\r
+\r
+\r
int compress_file(char* input_filename, char* output_filename, int compressionlevel)\r
{\r
int (*compressionFunction)(const char*, char*, int, int);\r
size_t sizeCheck, header_size;\r
void* streamChecksumState=NULL;\r
\r
+ // Branch out\r
+ if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel);\r
\r
// Init\r
start = clock();\r
{\r
case 0 : compressionFunction = LZ4_compress_limitedOutput; break;\r
case 1 : compressionFunction = LZ4_compressHC_limitedOutput; break;\r
- default : compressionFunction = LZ4_compress_limitedOutput;\r
+ default: compressionFunction = LZ4_compress_limitedOutput;\r
}\r
errorcode = get_fileHandle(input_filename, output_filename, &finput, &foutput);\r
if (errorcode) return errorcode;\r
- blockSize = LZ4S_GetBlocksize_FromBlockId (blockSizeId);\r
+ blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId);\r
\r
// Allocate Memory\r
in_buff = (char*)malloc(blockSize);\r
if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
\r
// Write Archive Header\r
- *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention \r
- *(out_buff+4) = 0x60; // Version("01"), Block independence\r
- *(out_buff+4) |= blockChecksum << 4; \r
- *(out_buff+4) |= streamChecksum << 2; \r
- *(out_buff+5) = (char)(blockSizeId<<4); // Block Size\r
+ *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention\r
+ *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01')\r
+ *(out_buff+4) |= (blockIndependence & _1BIT) << 5;\r
+ *(out_buff+4) |= (blockChecksum & _1BIT) << 4;\r
+ *(out_buff+4) |= (streamChecksum & _1BIT) << 2;\r
+ *(out_buff+5) = (blockSizeId & _3BITS) <<4;\r
checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);\r
checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);\r
*(out_buff+6) = (unsigned char) checkbits;\r
unsigned int outSize;\r
// Read Block\r
unsigned int inSize = (unsigned int) fread(in_buff, (size_t)1, (size_t)blockSize, finput);\r
- if( inSize<=0 ) break;\r
+ if( inSize<=0 ) break; // No more input : end of compression\r
filesize += inSize;\r
if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20));\r
if (streamChecksum) XXH32_update(streamChecksumState, in_buff, inSize);\r
}\r
\r
// End of Stream mark\r
- * (unsigned int*) out_buff = 0;\r
+ * (unsigned int*) out_buff = LZ4S_EOS;\r
sizeCheck = fwrite(out_buff, 1, 4, foutput);\r
if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");\r
compressedfilesize += 4;\r
{\r
// Block Size\r
uselessRet = fread(&blockSize, 1, 4, finput);\r
- if( uselessRet==0 ) break; // Nothing to read : file read is completed\r
- blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian\r
+ if( uselessRet==0 ) break; // Nothing to read : file read is completed\r
+ blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian\r
if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) \r
{ // Cannot read next block : maybe new stream ?\r
fseek(finput, -4, SEEK_CUR);\r
- break; \r
+ break;\r
}\r
\r
// Read Block\r
\r
// Write Block\r
sizeCheck = fwrite(out_buff, 1, sinkint, foutput);\r
- if (sizeCheck != (size_t)sinkint) EXM_THROW(53, "Write error : cannot write decoded block\n");\r
+ if (sizeCheck != (size_t)sinkint) EXM_THROW(53, "Write error : cannot write decoded block into output\n");\r
}\r
\r
// Free\r
{\r
unsigned long long filesize = 0;\r
char* in_buff;\r
- char* out_buff;\r
- unsigned char descriptor[3];\r
+ char* out_buff, *out_start, *out_end;\r
+ unsigned char descriptor[LZ4S_MAXHEADERSIZE];\r
size_t nbReadBytes;\r
- int decodedBytes;\r
+ int decodedBytes=0;\r
unsigned int maxBlockSize;\r
size_t sizeCheck;\r
- int blockChecksumFlag, streamChecksumFlag;\r
+ int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;\r
void* streamChecksumState=NULL;\r
+ int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;\r
+ unsigned int prefix64k = 0;\r
\r
// Decode stream descriptor\r
nbReadBytes = fread(descriptor, 1, 3, finput);\r
if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header");\r
{\r
int version = (descriptor[0] >> 6) & _2BITS;\r
- int independance = (descriptor[0] >> 5) & _1BIT;\r
int streamSize = (descriptor[0] >> 3) & _1BIT;\r
int reserved1 = (descriptor[0] >> 1) & _1BIT;\r
int dictionary = (descriptor[0] >> 0) & _1BIT;\r
int checkBits = (descriptor[2] >> 0) & _8BITS;\r
int checkBits_xxh32;\r
\r
+ blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT;\r
blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;\r
streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;\r
\r
if (version != 1) EXM_THROW(62, "Wrong version number");\r
- if (independance != 1) EXM_THROW(63, "Does not support block inter-dependence");\r
if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); \r
if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits");\r
if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); \r
if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits");\r
if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); \r
if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits");\r
- maxBlockSize = LZ4S_GetBlocksize_FromBlockId(blockSizeId);\r
+ maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId);\r
// Checkbits verification\r
descriptor[1] &= 0xF0;\r
checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED);\r
if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");\r
}\r
\r
+ if (!blockIndependenceFlag)\r
+ {\r
+ decompressionFunction = LZ4_decompress_safe_withPrefix64k;\r
+ prefix64k = 64 KB;\r
+ }\r
+\r
// Allocate Memory\r
- in_buff = (char*)malloc(maxBlockSize);\r
- out_buff = (char*)malloc(maxBlockSize);\r
- if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");\r
+ {\r
+ unsigned int outbuffSize = prefix64k+maxBlockSize;\r
+ in_buff = (char*)malloc(maxBlockSize);\r
+ if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;\r
+ out_buff = (char*)malloc(outbuffSize); \r
+ out_end = out_buff + outbuffSize;\r
+ out_start = out_buff + prefix64k;\r
+ if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");\r
+ }\r
if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);\r
\r
// Main Loop\r
// Block Size\r
nbReadBytes = fread(&blockSize, 1, 4, finput);\r
if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size");\r
- if (blockSize == 0) break; // End of Stream Mark : stream is completed\r
+ if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed\r
blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian\r
uncompressedFlag = blockSize >> 31;\r
blockSize &= 0x7FFFFFFF;\r
if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");\r
filesize += blockSize;\r
if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);\r
+ if (!blockIndependenceFlag)\r
+ {\r
+ if (blockSize >= prefix64k)\r
+ {\r
+ memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks\r
+ out_start = out_buff + prefix64k;\r
+ continue;\r
+ }\r
+ else\r
+ {\r
+ memcpy(out_start, in_buff, blockSize);\r
+ }\r
+ }\r
}\r
else\r
{\r
// Decode Block\r
- decodedBytes = LZ4_decompress_safe(in_buff, out_buff, blockSize, maxBlockSize);\r
+ decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);\r
if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");\r
filesize += decodedBytes;\r
- if (streamChecksumFlag) XXH32_update(streamChecksumState, out_buff, decodedBytes);\r
+ if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);\r
\r
// Write Block\r
- sizeCheck = fwrite(out_buff, 1, decodedBytes, foutput);\r
+ sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);\r
if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");\r
}\r
+\r
+ if (!blockIndependenceFlag)\r
+ {\r
+ out_start += decodedBytes;\r
+ if (out_start + maxBlockSize > out_end) \r
+ {\r
+ memcpy(out_buff, out_start - prefix64k, prefix64k); \r
+ out_start = out_buff + prefix64k; \r
+ }\r
+ }\r
}\r
\r
// Stream Checksum\r
\r
// Check Archive Header\r
nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput);\r
- if (nbReadBytes==0) return 0; // EOF\r
+ if (nbReadBytes==0) return 0; // EOF\r
if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");\r
- magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format\r
- if ((magicNumber & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers\r
+ magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format\r
+ if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers\r
\r
switch(magicNumber)\r
{\r
if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");\r
return selectDecoder(finput, foutput);\r
default:\r
- if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream\r
+ if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream\r
DISPLAY("Stream followed by unrecognized data\n");\r
- return 0; \r
+ return 0;\r
}\r
}\r
\r
case 'c': if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } break;\r
case 'h': if (argument[1]=='c') { cLevel=1; argument++; } break;\r
\r
- // Checksum management\r
- case 'x': blockChecksum=1; break;\r
- case 'n': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(exename); return 1; }\r
-\r
// Use Legacy format (hidden option)\r
case 'l': legacy_format=1; break;\r
\r
// Decoding\r
case 'd': decode=1; break;\r
\r
- // Bench\r
- case 'b': bench=1; \r
- if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } \r
- break;\r
+ // Test\r
+ case 't': decode=1; output_filename=nulmark; break;\r
\r
- // Modify Block Size \r
- case 'B': \r
- if ((argument[1] >='4') && (argument[1] <='7')) \r
+ // Modify Block Properties\r
+ case 'B':\r
+ while (argument[1]!=0)\r
+ switch(argument[1])\r
+ {\r
+ case '4':\r
+ case '5':\r
+ case '6':\r
+ case '7':\r
{ \r
int B = argument[1] - '0'; \r
int S = 1 << (8 + 2*B); \r
BMK_SetBlocksize(S); \r
blockSizeId = B;\r
argument++;\r
- } \r
+ break;\r
+ }\r
+ case 'D': blockIndependence = 0, argument++; break;\r
+ case 'X': blockChecksum = 1, argument ++; break;\r
+ default : goto _exit_blockProperties;\r
+ }\r
+_exit_blockProperties:\r
+ break;\r
+\r
+ // Modify Stream properties\r
+ case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(exename); return 1; }\r
+\r
+ // Bench\r
+ case 'b': bench=1; \r
+ if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } \r
break;\r
\r
// Modify Nb Iterations (benchmark only)\r
// Pause at the end (benchmark only) (hidden option)\r
case 'p': BMK_SetPause(); break;\r
\r
- // Test\r
- case 't': decode=1; output_filename=nulmark; break;\r
-\r
// Overwrite\r
case 'y': overwrite=1; break;\r
\r
\r
if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel);\r
\r
- // No output filename ==> build one automatically (for compression only)\r
+ // No output filename ==> build one automatically (when possible)\r
if (!output_filename) \r
{ \r
if (!decode) // compression\r
for (i=0;i<l;i++) output_filename[i] = input_filename[i];\r
for (i=l;i<l+4;i++) output_filename[i] = extension[i-l];\r
}\r
- else\r
+ else // decompression (input file must respect format extension ".lz4")\r
{\r
int inl=0,outl;\r
while (input_filename[inl]!=0) inl++;\r
#define MINLENGTH (MFLIMIT+1)\r
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)\r
\r
+#define KB *(1U<<10)\r
+#define MB *(1U<<20)\r
+#define GB *(1U<<30)\r
\r
//**************************************\r
// Architecture-specific macros\r
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);\r
# define UARCH U32\r
# define AARCH A32\r
-# define HTYPE const BYTE*\r
-# define INITBASE(b,s) const int b = 0\r
+//# define HTYPE const BYTE*\r
+//# define INITBASE(b,s) const int b = 0\r
+# define HTYPE U32\r
+# define INITBASE(b,s) const BYTE* const b = s\r
#endif\r
\r
#if defined(LZ4_BIG_ENDIAN)\r
//************************************************************\r
typedef struct \r
{\r
+ const BYTE* inputBuffer;\r
const BYTE* base;\r
+ const BYTE* end;\r
HTYPE hashTable[HASHTABLESIZE];\r
U16 chainTable[MAXD];\r
const BYTE* nextToUpdate;\r
#endif\r
\r
\r
-inline static int LZ4HC_Init (LZ4HC_Data_Structure* hc4, const BYTE* base)\r
+static inline int LZ4_InitHC (LZ4HC_Data_Structure* hc4, const BYTE* base)\r
{\r
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));\r
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));\r
- hc4->nextToUpdate = base + LZ4_ARCH64;\r
+ hc4->nextToUpdate = base + 1;\r
hc4->base = base;\r
+ hc4->inputBuffer = base;\r
+ hc4->end = base;\r
return 1;\r
}\r
\r
\r
-inline static void* LZ4HC_create (const BYTE* base)\r
+extern inline void* LZ4_createHC (const char* slidingInputBuffer)\r
{\r
void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));\r
-\r
- LZ4HC_Init ((LZ4HC_Data_Structure*)hc4, base);\r
+ LZ4_InitHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)slidingInputBuffer);\r
return hc4;\r
}\r
\r
\r
-inline static int LZ4HC_free (void** LZ4HC_Data)\r
+extern inline int LZ4_freeHC (void* LZ4HC_Data)\r
{\r
- FREEMEM(*LZ4HC_Data);\r
- *LZ4HC_Data = NULL;\r
- return (1);\r
+ FREEMEM(LZ4HC_Data);\r
+ return (0);\r
}\r
\r
\r
// Update chains up to ip (excluded)\r
-forceinline static void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)\r
+static forceinline void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)\r
{\r
U16* chainTable = hc4->chainTable;\r
HTYPE* HashTable = hc4->hashTable;\r
\r
while(hc4->nextToUpdate < ip)\r
{\r
- const BYTE* p = hc4->nextToUpdate;\r
+ const BYTE* const p = hc4->nextToUpdate;\r
size_t delta = (p) - HASH_POINTER(p); \r
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; \r
DELTANEXT(p) = (U16)delta; \r
}\r
\r
\r
-forceinline static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)\r
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data)\r
+{\r
+ LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;\r
+ U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB;\r
+ distance = (distance >> 16) << 16; // Must be a multiple of 64 KB\r
+ LZ4HC_Insert(hc4, hc4->end - MINMATCH);\r
+ memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);\r
+ hc4->nextToUpdate -= distance;\r
+ hc4->base -= distance;\r
+ if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow\r
+ {\r
+ int i;\r
+ hc4->base += 1 GB;\r
+ for (i=0; i<HASHTABLESIZE; i++) hc4->hashTable[i] -= 1 GB;\r
+ }\r
+ hc4->end -= distance;\r
+ return (char*)(hc4->end);\r
+}\r
+\r
+\r
+static forceinline size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)\r
{\r
const BYTE* p1t = p1;\r
\r
}\r
\r
\r
-forceinline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)\r
+static forceinline int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)\r
{\r
U16* const chainTable = hc4->chainTable;\r
HTYPE* const HashTable = hc4->hashTable;\r
INITBASE(base,hc4->base);\r
int nbAttempts=MAX_NB_ATTEMPTS;\r
size_t repl=0, ml=0;\r
- U16 delta;\r
+ U16 delta=0; // useless assignment, to remove an uninitialization warning\r
\r
// HC4 match finder\r
LZ4HC_Insert(hc4, ip);\r
#define REPEAT_OPTIMIZATION\r
#ifdef REPEAT_OPTIMIZATION\r
// Detect repetitive sequences of length <= 4\r
- if (ref >= ip-4) // potential repetition\r
+ if ((U32)(ip-ref) <= 4) // potential repetition\r
{\r
if (A32(ref) == A32(ip)) // confirmed\r
{\r
}\r
#endif\r
\r
- while ((ref >= ip-MAX_DISTANCE) && (nbAttempts))\r
+ while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))\r
{\r
nbAttempts--;\r
if (*(ref+ml) == *(ip+ml))\r
}\r
\r
\r
-forceinline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)\r
+static forceinline int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)\r
{\r
U16* const chainTable = hc4->chainTable;\r
HTYPE* const HashTable = hc4->hashTable;\r
LZ4HC_Insert(hc4, ip);\r
ref = HASH_POINTER(ip);\r
\r
- while ((ref >= ip-MAX_DISTANCE) && (nbAttempts))\r
+ while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts))\r
{\r
nbAttempts--;\r
if (*(startLimit + longest) == *(ref - delta + longest))\r
const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);\r
#endif\r
\r
- while ((startt>startLimit) && (reft > hc4->base) && (startt[-1] == reft[-1])) {startt--; reft--;}\r
+ while ((startt>startLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;}\r
\r
if ((ipt-startt) > longest)\r
{\r