typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
typedef enum { byPtr, byU32, byU16 } tableType_t;
-typedef enum { noDict = 0, withPrefix64k = 1, usingExtDict = 2 } dict_directive;
+typedef enum { noDict = 0, withPrefix64k, withPrefixSmall, usingExtDict, usingSmallDict } dict_directive;
typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
#endif
-/****************************
+/********************************
Compression functions
-****************************/
+********************************/
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
static int LZ4_hashSequence(U32 sequence, tableType_t tableType)
lowLimit = (const BYTE*)source;
break;
case withPrefix64k:
+ case withPrefixSmall:
base = (const BYTE*)source - dictPtr->currentOffset;
lowLimit = (const BYTE*)source - dictPtr->dictSize;
break;
case usingExtDict:
+ case usingSmallDict:
base = (const BYTE*)source - dictPtr->currentOffset;
lowLimit = (const BYTE*)source;
break;
if (unlikely(forwardIp > mflimit)) goto _last_literals;
ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
- if (dict==usingExtDict)
+ if ((dict==usingExtDict) || (dict==usingSmallDict))
{
if (ref<(const BYTE*)source)
{
forwardH = LZ4_hashPosition(forwardIp, tableType);
LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
- } while ( ((tableType==byU16)? 0 : (ref + MAX_DISTANCE < ip)) || (A32(ref+refDelta) != A32(ip)) );
+ } while ( ((dict==withPrefixSmall) ? (ref < lowLimit) : 0)
+ || ((dict==usingSmallDict) && (refDelta) ? (ref < lowLimit) : 0)
+ || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip))
+ || (A32(ref+refDelta) != A32(ip)) );
}
/* Catch up */
/* Test next position */
ref = LZ4_getPosition(ip, ctx, tableType, base);
- if (dict==usingExtDict)
+ if ((dict==usingExtDict) || (dict==usingSmallDict))
{
if (ref<(const BYTE*)source)
{
}
}
LZ4_putPosition(ip, ctx, tableType, base);
- if ( (ref+MAX_DISTANCE>=ip) && (A32(ref+refDelta)==A32(ip)) ) { token=op++; *token=0; goto _next_match; }
+ if ( ((dict==withPrefixSmall) ? (ref>=lowLimit) : 1)
+ && ((dict==usingSmallDict) && (refDelta) ? (ref>=lowLimit) : 1)
+ && (ref+MAX_DISTANCE>=ip)
+ && (A32(ref+refDelta)==A32(ip)) )
+ { token=op++; *token=0; goto _next_match; }
/* Prepare next loop */
forwardH = LZ4_hashPosition(++ip, tableType);
const BYTE* sourceEnd = (const BYTE*) source + inputSize;
if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
{
- streamPtr->dictSize = dictEnd - sourceEnd;
+ streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
streamPtr->dictionary = dictEnd - streamPtr->dictSize;
if (dictEnd == (const BYTE*)source)
{
- int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
+ int result;
+ if (streamPtr->dictSize >= 64 KB)
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
+ else
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefixSmall);
streamPtr->dictSize += (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
return result;
}
{
- int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
+ int result;
+ if (streamPtr->dictSize >= 64 KB)
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
+ else
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingSmallDict);
streamPtr->dictionary = (const BYTE*)source;
streamPtr->dictSize = (U32)inputSize;
streamPtr->currentOffset += (U32)inputSize;
const BYTE* sourceEnd = (const BYTE*) source + inputSize;
if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
{
- streamPtr->dictSize = dictEnd - sourceEnd;
+ streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
streamPtr->dictionary = dictEnd - streamPtr->dictSize;
}
-int LZ4_moveDict (void* LZ4_dict, char* safeBuffer, int dictSize)
+int LZ4_saveDict (void* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, noDict, NULL, 0);
}
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
+int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 0);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, partial, targetOutputSize, noDict, NULL, 0);
}
-int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
+int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
+ return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0);
}
-int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxOutputSize)
+/* streaming decompression functions */
+
+//#define LZ4_STREAMDECODESIZE_U32 4
+//#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
+//typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
+typedef struct
{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, partial, targetOutputSize, noDict, NULL, 0);
+ const char* dictionary;
+ int dictSize;
+} LZ4_streamDecode_t_internal;
+
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStreamDecode()
+ * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
+ */
+void* LZ4_createStreamDecode()
+{
+ void* lz4s = ALLOCATOR(sizeof(U32), LZ4_STREAMDECODESIZE_U32);
+ MEM_INIT(lz4s, 0, LZ4_STREAMDECODESIZE);
+ return lz4s;
}
-int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
+/*
+ * LZ4_setDictDecode
+ * Use this function to instruct where to find the dictionary
+ * This function is not necessary if previous data is still available where it was decoded.
+ * Loading a size of 0 is allowed (same effect as no dictionary).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0);
+ LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ lz4sd->dictionary = dictionary;
+ lz4sd->dictSize = dictSize;
+ return 1;
}
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
+/*
+*_continue() :
+ These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ Previously decoded blocks must still be available at the memory position where they were decoded.
+ If it's not possible, save the relevant part of decoded data before it disappears,
+ and indicate where it stands using LZ4_setDictDecode()
+*/
+int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
- return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0);
+ LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ int result;
+
+ result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
+ lz4sd->dictionary = dest;
+ lz4sd->dictSize = result;
+
+ return result;
+}
+
+int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, char* dest, int originalSize)
+{
+ LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ int result;
+
+ result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
+ lz4sd->dictionary = dest;
+ lz4sd->dictSize = originalSize;
+
+ return result;
+}
+
+
+/*
+Advanced decoding functions :
+*_usingDict() :
+ These decoding functions work the same as "_continue" ones,
+ the dictionary must be explicitly provided within parameters
+*/
+
+int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
}
int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
}
-/**************************************
- Obsolete Functions
-**************************************/
+/***************************************************
+ Obsolete Functions
+***************************************************/
/*
These function names are deprecated and should no longer be used.
They are only provided here for compatibility with older user programs.
{
LZ4_stream_t_internal* lz4ds = (LZ4_stream_t_internal*)LZ4_Data;
- LZ4_moveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB);
+ LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB);
return (char*)(lz4ds->bufferStart + 64 KB);
}
-/* User-allocated state */
+/* Obsolete compresson functions using User-allocated state */
int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict);
}
+/* Obsolete streaming decompression functions */
+
+int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 0);
+}
+
+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
+{
+ return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0);
+}
/*
* LZ4_compress_continue
- * Compress data block 'source', using blocks compressed before to improve compression ratio
+ * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
* Previous data blocks are assumed to still be present at their previous location.
*/
int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
/*
- * LZ4_moveDict
+ * LZ4_saveDict
* If previously compressed data block is not guaranteed to remain at its previous memory location
* save it into a safe place (char* safeBuffer)
- * before calling again LZ4_compress_continue()
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ * dictionary is immediately usable, you can therefore call again LZ4_compress_continue()
* Return : 1 if OK, 0 if error
* Note : any dictSize > 64 KB will be interpreted as 64KB.
*/
-int LZ4_moveDict (void* LZ4_stream, char* safeBuffer, int dictSize);
+int LZ4_saveDict (void* LZ4_stream, char* safeBuffer, int dictSize);
/************************************************
Experimental Streaming Decompression Functions
************************************************/
+#define LZ4_STREAMDECODESIZE_U32 4
+#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
/*
+ * LZ4_streamDecode_t
+ * information structure to track an LZ4 stream.
+ * important : set this structure content to zero before first use !
+ */
+typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
+
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStreamDecode()
+ * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
+ * LZ4_free just frees it.
+ */
+void* LZ4_createStreamDecode();
+int LZ4_free (void* LZ4_stream); /* yes, it's the same one as compression */
+
+/*
+*_continue() :
+ These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ Previously decoded blocks must still be available at the memory position where they were decoded.
+ If it's not possible, save the relevant part of decoded data before it disappears,
+ and indicate where it stands using LZ4_setDictDecode()
+*/
+int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize);
+int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, char* dest, int originalSize);
+
+/*
+ * LZ4_setDictDecode
+ * Use this function to instruct where to find the dictionary
+ * This function is not necessary if previous data is still available where it was decoded.
+ * Loading a size of 0 is allowed (same effect as no dictionary).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictSize);
+
+
+/*
+Advanced decoding functions :
*_usingDict() :
- These decoding functions work the same as their "normal" versions,
- but can also use up to 64KB of dictionary data (dictStart, dictSize)
- to decode chained blocks.
+ These decoding functions work the same as "_continue" ones,
+ the dictionary must be explicitly provided within parameters
*/
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
-/*
-*_withPrefix64k() :
- These decoding functions work the same as their "normal" versions,
- but can also use up to 64KB of data in front of 'char* dest'
- to decode chained blocks.
- The last 64KB of previous block must be present there.
-*/
-int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize);
-int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
int LZ4_resetStreamState(void* state, const char* inputBuffer);
char* LZ4_slideInputBuffer (void* state);
+/* Obsolete streaming decoding functions */
+int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize);
+int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
+
#if defined (__cplusplus)
}
#define CACHELINE 64
#define LEGACY_BLOCKSIZE (8 MB)
-#define MIN_STREAM_BUFSIZE (1 MB + 64 KB)
+#define MIN_STREAM_BUFSIZE (192 KB)
#define LZ4S_BLOCKSIZEID_DEFAULT 7
#define LZ4S_CHECKSUM_SEED 0
#define LZ4S_EOS 0
{
initFunction = LZ4IO_LZ4_createStream;
compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue;
- nextBlockFunction = LZ4_moveDict;
+ nextBlockFunction = LZ4_saveDict;
freeFunction = LZ4_free;
}
else
{
size_t sizeToMove = 64 KB;
if (inSize < 64 KB) sizeToMove = inSize;
- nextBlockFunction(ctx, in_blockStart - sizeToMove, sizeToMove);
+ nextBlockFunction(ctx, in_blockStart - sizeToMove, (int)sizeToMove);
if (compressionlevel>=3) in_blockStart = in_buff + 64 KB;
}
}
size_t sizeCheck;
int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag;
void* streamChecksumState=NULL;
- int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe;
- unsigned int prefix64k = 0;
+ int (*decompressionFunction)(void* ctx, const char* src, char* dst, int cSize, int maxOSize) = LZ4_decompress_safe_continue;
+ LZ4_streamDecode_t ctx;
+
+ // init
+ memset(&ctx, 0, sizeof(ctx));
+ (void)blockIndependenceFlag;
// Decode stream descriptor
nbReadBytes = fread(descriptor, 1, 3, finput);
if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected");
}
- if (!blockIndependenceFlag)
- {
- decompressionFunction = LZ4_decompress_safe_withPrefix64k;
- prefix64k = 64 KB;
- }
-
// Allocate Memory
{
- unsigned int outbuffSize = prefix64k+maxBlockSize;
+ size_t outBuffSize = maxBlockSize + 64 KB;
+ if (outBuffSize < MIN_STREAM_BUFSIZE) outBuffSize = MIN_STREAM_BUFSIZE;
in_buff = (char*)malloc(maxBlockSize);
- if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE;
- out_buff = (char*)malloc(outbuffSize);
- out_end = out_buff + outbuffSize;
- out_start = out_buff + prefix64k;
+ out_buff = (char*)malloc(outBuffSize);
+ out_start = out_buff;
+ out_end = out_start + outBuffSize;
if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");
+ if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
}
- if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);
// Main Loop
while (1)
if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");
filesize += blockSize;
if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);
- if (!blockIndependenceFlag)
+ if (!independentBlocks)
{
- if (blockSize >= prefix64k)
- {
- memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks
- out_start = out_buff + prefix64k;
- continue;
- }
- else
- {
- memcpy(out_start, in_buff, blockSize);
- decodedBytes = blockSize;
- }
+ // handle dictionary for streaming
+ memcpy(in_buff + blockSize - 64 KB, out_buff, 64 KB);
+ LZ4_setDictDecode(&ctx, out_buff, 64 KB);
+ out_start = out_buff + 64 KB;
}
}
else
{
// Decode Block
- decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize);
+ if (out_start + maxBlockSize > out_end) out_start = out_buff;
+ decodedBytes = decompressionFunction(&ctx, in_buff, out_start, blockSize, maxBlockSize);
if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");
filesize += decodedBytes;
if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes);
// Write Block
sizeCheck = fwrite(out_start, 1, decodedBytes, foutput);
if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n");
- }
-
- if (!blockIndependenceFlag)
- {
out_start += decodedBytes;
- if ((size_t)(out_end - out_start) < (size_t)maxBlockSize)
- {
- memcpy(out_buff, out_start - prefix64k, prefix64k);
- out_start = out_buff + prefix64k;
- }
}
+
}
// Stream Checksum
sizeCheck = fread(&readChecksum, 1, 4, finput);
if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");
readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian
- if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");
+ if (checksum != readChecksum) EXM_THROW(79, "Error : invalid stream checksum detected");
}
// Free
projects / platform / upstream / lz4.git / commitdiff