From: Yann Collet Date: Thu, 19 Jun 2014 21:54:16 +0000 (+0100) Subject: bugfix : streaming tiny messages from within very small ringbuffer (Takayuki's stream... X-Git-Tag: upstream/1.9.3~263^2~3 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=6e1179a1e3aa00d329983f85e4e13a9a9a4d3223;p=platform%2Fupstream%2Flz4.git bugfix : streaming tiny messages from within very small ringbuffer (Takayuki's streaming example 2) --- diff --git a/lz4.c b/lz4.c index 11d6b80..957b907 100755 --- a/lz4.c +++ b/lz4.c @@ -612,7 +612,7 @@ _next_match: } LZ4_putPosition(ip, ctx, tableType, base); if ( ((dict==withPrefixSmall) ? (ref>=lowLimit) : 1) - && ((dict==usingSmallDict) && (refDelta) ? (ref>=lowLimit) : 1) + && ((dict==usingSmallDict) && (refDelta) ? (ref>=lowLimit) : 1) && (ref+MAX_DISTANCE>=ip) && (A32(ref+refDelta)==A32(ip)) ) { token=op++; *token=0; goto _next_match; } @@ -749,14 +749,15 @@ void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize) +FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, + int maxOutputSize, limitedOutput_directive limit) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = (const BYTE*) source; if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest > dictEnd)) smallest = dictEnd; + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; LZ4_renormDictT(streamPtr, smallest); /* Check overlapping input/dictionary space */ @@ -771,24 +772,26 @@ int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int } } + /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { int result; - if (streamPtr->dictSize >= 64 KB) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k); + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefixSmall); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefixSmall); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; } + /* external dictionary mode */ { int result; - if (streamPtr->dictSize >= 64 KB) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingExtDict); + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingSmallDict); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingSmallDict); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; @@ -796,43 +799,15 @@ int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int } } -int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - - /* Check overlapping input/dictionary space */ - { - const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) - { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - if (dictEnd == (const BYTE*)source) - { - int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } +int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); +} - { - int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } +int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); } @@ -895,7 +870,7 @@ FORCE_INLINE int LZ4_decompress_generic( int targetOutputSize, /* only used if partialDecoding==partial */ int dict, /* noDict, withPrefix64k, usingExtDict */ const char* dictStart, /* only if dict==usingExtDict */ - int dictSize /* only if dict==usingExtDict */ + int dictSize /* note : = 0 if noDict */ ) { /* Local Variables */ @@ -907,6 +882,7 @@ FORCE_INLINE int LZ4_decompress_generic( BYTE* const oend = op + outputSize; BYTE* cpy; BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = (const BYTE*) dest - dictSize; const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; #define OLD @@ -966,7 +942,7 @@ FORCE_INLINE int LZ4_decompress_generic( /* get offset */ LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; - if ((dict==noDict) && (unlikely(ref < (BYTE* const)dest))) goto _output_error; /* Error : offset outside destination buffer */ + if ((endOnInput) && (unlikely(ref < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ /* get matchlength */ if ((length=(token&ML_MASK)) == ML_MASK) @@ -1068,7 +1044,7 @@ int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSi int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0); + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB); } /* streaming decompression functions */ @@ -1113,7 +1089,7 @@ int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictS *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data before it disappears, + If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setDictDecode() */ int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) @@ -1122,8 +1098,16 @@ int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, ch int result; result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize); - lz4sd->dictionary = dest; - lz4sd->dictSize = result; + if (result <= 0) return result; + if (lz4sd->dictionary + lz4sd->dictSize == dest) + { + lz4sd->dictSize += result; + } + else + { + lz4sd->dictionary = dest; + lz4sd->dictSize = result; + } return result; } @@ -1134,8 +1118,16 @@ int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, ch int result; result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize); - lz4sd->dictionary = dest; - lz4sd->dictSize = originalSize; + if (result <= 0) return result; + if (lz4sd->dictionary + lz4sd->dictSize == dest) + { + lz4sd->dictSize += result; + } + else + { + lz4sd->dictionary = dest; + lz4sd->dictSize = result; + } return result; } @@ -1235,10 +1227,10 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB); } int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0); + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB); } diff --git a/lz4.h b/lz4.h old mode 100644 new mode 100755 index 9d58a1a..1cf9eab --- a/lz4.h +++ b/lz4.h @@ -241,7 +241,7 @@ int LZ4_free (void* LZ4_stream); /* yes, it's the same one as compression */ *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data before it disappears, + If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setDictDecode() */ int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize); @@ -249,9 +249,9 @@ int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, ch /* * LZ4_setDictDecode - * Use this function to instruct where to find the dictionary - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was already decoded. + * Setting a size of 0 is allowed (same effect as no dictionary). * Return : 1 if OK, 0 if error */ int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictSize); diff --git a/programs/fuzzer.c b/programs/fuzzer.c index d209fd3..6035213 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -439,19 +439,19 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) { memset(&LZ4dict, 0, sizeof(LZ4_stream_t)); LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed"); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); FUZ_DISPLAYTEST; memset(&LZ4dict, 0, sizeof(LZ4_stream_t)); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_usingDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue should fail : one missing byte for output buffer"); FUZ_DISPLAYTEST; memset(&LZ4dict, 0, sizeof(LZ4_stream_t)); LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize); - FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_usingDict should work : enough size available within output buffer"); + FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); // Decompress with dictionary as external FUZ_DISPLAYTEST;