int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
-LZ4_compress_limitedOutput() :
+LZ4_compress_safe() :
Compresses 'sourceSize' bytes from buffer 'source'
- into already allocated 'dest' of size 'maxOutputSize'.
- Compression runs faster when 'maxOutputSize' >= LZ4_compressBound(sourceSize).
- That's because in such case, it is guaranteed to compress within 'dest' budget, even in worst case scenario.
- Compressing into a more limited space budget requires additional checks.
- If the function cannot compress 'source' into a limited 'dest' budget,
- compression stops *immediately*, and result of the function is zero.
- It greatly accelerates behavior on non-compressible input, but as a consequence, 'dest' content is not valid either.
+ into already allocated 'dest' buffer of size 'maxDestSize'.
+ Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
+ It also runs faster, so it's a recommended setting.
+ If the function cannot compress 'source' into a more limited 'dest' budget,
+ compression stops *immediately*, and the function result is zero.
+ As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
LZ4_compress_fast() :
Same as LZ4_compress_safe(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
- It's a trade-off, which can be fine tuned, selecting whichever value you want.
+ It's a trade-off. It can be fine tuned, with each successive value providing an additional +2/3% to speed.
An acceleration value of "0" means "use Default value", which is typically 17 (see lz4.c source code).
An acceleration value of "1" is the same as regular LZ4_compress_safe()
Note : this function is "safe", even if its name does not explicitly contain the word. It's just faster and compress less.
*/
-int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxOutputSize, unsigned acceleration);
-int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, unsigned acceleration);
+int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, unsigned acceleration);
+int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, unsigned acceleration);
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- If this condition is not possible, save the relevant part of decoded data into a safe buffer,
- and indicate where is its new address using LZ4_setStreamDecode()
+ In the case of a ring buffers, decoding buffer must be either :
+ - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+ In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+ - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+ In which case, encoding and decoding buffers do not need to be synchronized,
+ and encoding ring buffer can have any size, including small ones ( < 64 KB).
+ - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ In which case, encoding and decoding buffers do not need to be synchronized,
+ and encoding ring buffer can have any size, including larger than decoding buffer.
+ Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+ and indicate where it is saved using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
- a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue()
- They are stand-alone and don't use nor update an LZ4_streamDecode_t structure.
+ a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
+ They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
*/
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
{
U32 rand32 = *src;
rand32 *= PRIME1;
- rand32 += PRIME2;
+ rand32 ^= PRIME2;
rand32 = FUZ_rotl32(rand32, 13);
*src = rand32;
- return rand32 >> 3;
+ return rand32;
}
size_t pos = 0;
U32 P32 = (U32)(32768 * proba);
- // First Byte
- BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
+ /* First Bytes */
+ while (pos < 20)
+ BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
while (pos < bufferSize)
{
- // Select : Literal (noise) or copy (within 64K)
+ /* Select : Literal (noise) or copy (within 64K) */
if (FUZ_RAND15BITS < P32)
{
- // Copy (within 64K)
+ /* Copy (within 64K) */
size_t match, d;
size_t length = FUZ_RANDLENGTH + 4;
size_t offset = FUZ_RAND15BITS + 1;
- if (offset > pos) offset = pos;
+ while (offset > pos) offset >>= 1;
d = pos + length;
- if (d > bufferSize) d = bufferSize;
+ while (d > bufferSize) d = bufferSize;
match = pos - offset;
while (pos < d) BBuffer[pos++] = BBuffer[match++];
}
else
{
- // Literal (noise)
+ /* Literal (noise) */
size_t d;
size_t length = FUZ_RANDLENGTH;
d = pos + length;
}
+static void FUZ_findDiff(const void* buff1, const void* buff2)
+{
+ const BYTE* b1 = (const BYTE*)buff1;
+ const BYTE* b2 = (const BYTE*)buff2;
+ size_t i=0;
+ while (b1[i]==b2[i]) i++;
+ DISPLAY("Wrong Byte at position %u\n", (unsigned)i);
+}
+
+
static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration)
{
unsigned long long bytes = 0;
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
HCcompressedSize = ret;
- // Test compression HC using external state
+ /* Test compression HC using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
- // Test compression using external state
+ /* Test compression using external state */
FUZ_DISPLAYTEST;
ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
- // Test compression
+ /* Test compression */
FUZ_DISPLAYTEST;
ret = LZ4_compress(block, compressedBuffer, blockSize);
FUZ_CHECKTEST(ret==0, "LZ4_compress() failed");
crcOrig = XXH32(block, blockSize, 0);
- // Test decoding with output size being exactly what's necessary => must work
+ /* Test decoding with output size being exactly what's necessary => must work */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
crcCheck = XXH32(decodedBuffer, blockSize, 0);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
- // Test decoding with one byte missing => must fail
+ /* Test decoding with one byte missing => must fail */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
- // Test decoding with one byte too much => must fail
+ /* Test decoding with one byte too much => must fail */
FUZ_DISPLAYTEST;
ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
- // Test decoding with output size exactly what's necessary => must work
+ /* Test decoding with output size exactly what's necessary => must work */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
FUZ_DISPLAYTEST;
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
- FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data");
+ FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
- FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data");
+ FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
/* Compress using External dictionary */
FUZ_DISPLAYTEST;
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
- // Decompress with dictionary as external
+ /* Decompress with dictionary as external */
FUZ_DISPLAYTEST;
decodedBuffer[blockSize] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
- {
- int i=0;
- while (block[i]==decodedBuffer[i]) i++;
- printf("Wrong Byte at position %i/%i\n", i, blockSize);
- }
+ FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
FUZ_DISPLAYTEST;
FUZ_DISPLAYTEST;
decodedBuffer[blockSize-1] = 0;
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
- FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)");
+ FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
FUZ_DISPLAYTEST;
}
}
- // Compress HC using External dictionary
+ /* Compress HC using External dictionary */
FUZ_DISPLAYTEST;
- dict -= (FUZ_rand(&randState) & 7); // even bigger separation
+ dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
crcCheck = XXH32(decodedBuffer, blockSize, 0);
if (crcCheck!=crcOrig)
- {
- int i=0;
- while (block[i]==decodedBuffer[i]) i++;
- printf("Wrong Byte at position %i/%i\n", i, blockSize);
- }
+ FUZ_findDiff(block, decodedBuffer);
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
-
/* ***** End of tests *** */
/* Fill stats */
bytes += blockSize;
char ringBuffer[ringBufferSize];
U32 randState = 1;
- // Init
+ /* Init */
FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
- // 32-bits address space overflow test
+ /* 32-bits address space overflow test */
FUZ_AddressOverflow();
- // LZ4 streaming tests
+ /* LZ4 streaming tests */
{
LZ4_stream_t* statePtr;
LZ4_stream_t streamingState;
U64 crcNew;
int result;
- // Allocation test
+ /* Allocation test */
statePtr = LZ4_createStream();
FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
LZ4_freeStream(statePtr);
- // simple compression test
+ /* simple compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStream(&streamingState);
result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
- // ring buffer test
+ /* ring buffer test */
{
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
- // prepare next message
+ /* prepare next message */
iNext += messageSize;
rNext += messageSize;
dNext += messageSize;
}
}
- // LZ4 HC streaming tests
+ /* LZ4 HC streaming tests */
{
LZ4_streamHC_t* sp;
LZ4_streamHC_t sHC;
- //XXH64_state_t xxh;
U64 crcOrig;
U64 crcNew;
int result;
- // Allocation test
+ /* Allocation test */
sp = LZ4_createStreamHC();
FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
LZ4_freeStreamHC(sp);
- // simple compression test
+ /* simple HC compression test */
crcOrig = XXH64(testInput, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
crcNew = XXH64(testVerify, testCompressedSize, 0);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
- // simple dictionary compression test
+ /* simple dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 64 KB);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption");
}
- // remote dictionary HC compression test
+ /* remote dictionary HC compression test */
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_loadDictHC(&sHC, testInput, 32 KB);
XXH64_state_t xxhOrig;
XXH64_state_t xxhNew;
LZ4_streamDecode_t decodeState;
- const U32 maxMessageSizeLog = 10;
+ const U32 maxMessageSizeLog = 12;
const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
U32 totalMessageSize = 0;
U32 iNext = 0;
U32 dNext = 0;
- const U32 dBufferSize = 64 KB + maxMessageSizeMask;
+ const U32 dBufferSize = 64 KB;
XXH64_reset(&xxhOrig, 0);
XXH64_reset(&xxhNew, 0);
LZ4_resetStreamHC(&sHC, 0);
LZ4_setStreamDecode(&decodeState, NULL, 0);
+#define BSIZE1 65537
+#define BSIZE2 16435
+
+ /* first block */
+
+ messageSize = BSIZE1;
+ XXH64_update(&xxhOrig, testInput + iNext, messageSize);
+ crcOrig = XXH64_digest(&xxhOrig);
+
+ result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
+ FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
+
+ result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
+ FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
+
+ XXH64_update(&xxhNew, testVerify + dNext, messageSize);
+ crcNew = XXH64_digest(&xxhNew);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+
+ /* prepare next message */
+ dNext += messageSize;
+ totalMessageSize += messageSize;
+ messageSize = BSIZE2;
+ iNext = 132000;
+ memcpy(testInput + iNext, testInput + 8, messageSize);
+ if (dNext > dBufferSize) dNext = 0;
+
while (totalMessageSize < 9 MB)
{
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
- FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
+ FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
crcNew = XXH64_digest(&xxhNew);
- FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption");
+ if (crcOrig != crcNew)
+ FUZ_findDiff(testInput + iNext, testVerify + dNext);
+ FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
iNext = (FUZ_rand(&randState) & 65535);
- if (dNext + messageSize > dBufferSize) dNext = 0;
+ if (dNext > dBufferSize) dNext = 0;
}
}
}
if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
}
-
+/*
+ * LZ4IO_compressFilename_extRess()
+ * result : 0 : compression completed correctly
+ * 1 : missing or pb opening srcFileName
+ */
static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel)
{
unsigned long long filesize = 0;
memset(&prefs, 0, sizeof(prefs));
/* File check */
- {
- int srcFileAbsent = LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile);
- if (srcFileAbsent) return srcFileAbsent;
- }
+ if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1;
/* Set compression parameters */
prefs.autoFlush = 1;
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
- {
- DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
- (unsigned long long) filesize, (unsigned long long) compressedfilesize,
- (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */
- }
+ DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+ filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */
return 0;
}
int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel)
{
int i;
- int missing_files = 0;
- char* outFileName = (char*)malloc(FNSPACE);
+ int missed_files = 0;
+ char* dstFileName = (char*)malloc(FNSPACE);
size_t ofnSize = FNSPACE;
const size_t suffixSize = strlen(suffix);
cRess_t ress;
for (i=0; i<ifntSize; i++)
{
size_t ifnSize = strlen(inFileNamesTable[i]);
- if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
- strcpy(outFileName, inFileNamesTable[i]);
- strcat(outFileName, suffix);
+ if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); }
+ strcpy(dstFileName, inFileNamesTable[i]);
+ strcat(dstFileName, suffix);
- missing_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], outFileName, compressionLevel);
+ missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
}
/* Close & Free */
LZ4IO_freeCResources(ress);
- free(outFileName);
+ free(dstFileName);
- return missing_files;
+ return missed_files;
}
projects / platform / upstream / lz4.git / commitdiff