bugfix : streaming tiny messages from within very small ringbuffer (Takayuki's stream...

diff --git a/lz4.c b/lz4.c
index 11d6b80..957b907 100755 (executable)
--- a/lz4.c
+++ b/lz4.c
@@ -612,7 +612,7 @@ _next_match:
         }
         LZ4_putPosition(ip, ctx, tableType, base);
         if ( ((dict==withPrefixSmall) ? (ref>=lowLimit) : 1)
-            && ((dict==usingSmallDict) && (refDelta) ? (ref>=lowLimit) : 1) 
+            && ((dict==usingSmallDict) && (refDelta) ? (ref>=lowLimit) : 1)
             && (ref+MAX_DISTANCE>=ip)
             && (A32(ref+refDelta)==A32(ip)) )
         { token=op++; *token=0; goto _next_match; }
@@ -749,14 +749,15 @@ void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
 }
 
 
-int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize)
+FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize,
+                                                int maxOutputSize, limitedOutput_directive limit)
 {
     LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream;
     const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
     const BYTE* smallest = (const BYTE*) source;
     if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
-    if ((streamPtr->dictSize>0) && (smallest > dictEnd)) smallest = dictEnd;
+    if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
     LZ4_renormDictT(streamPtr, smallest);
 
     /* Check overlapping input/dictionary space */
@@ -771,24 +772,26 @@ int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int
         }
     }
 
+    /* prefix mode : source data follows dictionary */
     if (dictEnd == (const BYTE*)source)
     {
         int result;
-        if (streamPtr->dictSize >= 64 KB)
-            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefix64k);
+        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
+            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefixSmall);
         else
-            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, withPrefixSmall);
+            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k);
         streamPtr->dictSize += (U32)inputSize;
         streamPtr->currentOffset += (U32)inputSize;
         return result;
     }
 
+    /* external dictionary mode */
     {
         int result;
-        if (streamPtr->dictSize >= 64 KB)
-            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingExtDict);
+        if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
+            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingSmallDict);
         else
-            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, 0, notLimited, byU32, usingSmallDict);
+            result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict);
         streamPtr->dictionary = (const BYTE*)source;
         streamPtr->dictSize = (U32)inputSize;
         streamPtr->currentOffset += (U32)inputSize;
@@ -796,43 +799,15 @@ int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int
     }
 }
 
-int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize)
-{
-    LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream;
-    const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
 
-    const BYTE* smallest = (const BYTE*) source;
-    if (streamPtr->initCheck) return 0;   /* Uninitialized structure detected */
-    if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
-    LZ4_renormDictT(streamPtr, smallest);
-
-    /* Check overlapping input/dictionary space */
-    {
-        const BYTE* sourceEnd = (const BYTE*) source + inputSize;
-        if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
-        {
-            streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
-            if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
-            if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
-            streamPtr->dictionary = dictEnd - streamPtr->dictSize;
-        }
-    }
-
-    if (dictEnd == (const BYTE*)source)
-    {
-        int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k);
-        streamPtr->dictSize += (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
-    }
+int LZ4_compress_continue (void* LZ4_stream, const char* source, char* dest, int inputSize)
+{
+    return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited);
+}
 
-    {
-        int result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict);
-        streamPtr->dictionary = (const BYTE*)source;
-        streamPtr->dictSize = (U32)inputSize;
-        streamPtr->currentOffset += (U32)inputSize;
-        return result;
-    }
+int LZ4_compress_limitedOutput_continue (void* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput);
 }
 
 
@@ -895,7 +870,7 @@ FORCE_INLINE int LZ4_decompress_generic(
                  int targetOutputSize,   /* only used if partialDecoding==partial */
                  int dict,               /* noDict, withPrefix64k, usingExtDict */
                  const char* dictStart,  /* only if dict==usingExtDict */
-                 int dictSize            /* only if dict==usingExtDict */
+                 int dictSize            /* note : = 0 if noDict */
                  )
 {
     /* Local Variables */
@@ -907,6 +882,7 @@ FORCE_INLINE int LZ4_decompress_generic(
     BYTE* const oend = op + outputSize;
     BYTE* cpy;
     BYTE* oexit = op + targetOutputSize;
+    const BYTE* const lowLimit = (const BYTE*) dest - dictSize;
 
     const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
 #define OLD
@@ -966,7 +942,7 @@ FORCE_INLINE int LZ4_decompress_generic(
 
         /* get offset */
         LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
-        if ((dict==noDict) && (unlikely(ref < (BYTE* const)dest))) goto _output_error;   /* Error : offset outside destination buffer */
+        if ((endOnInput) && (unlikely(ref < lowLimit))) goto _output_error;   /* Error : offset outside destination buffer */
 
         /* get matchlength */
         if ((length=(token&ML_MASK)) == ML_MASK)
@@ -1068,7 +1044,7 @@ int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSi
 
 int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
 {
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0);
+    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
 }
 
 /* streaming decompression functions */
@@ -1113,7 +1089,7 @@ int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictS
 *_continue() :
     These decoding functions allow decompression of multiple blocks in "streaming" mode.
     Previously decoded blocks must still be available at the memory position where they were decoded.
-    If it's not possible, save the relevant part of decoded data before it disappears,
+    If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setDictDecode()
 */
 int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
@@ -1122,8 +1098,16 @@ int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, ch
     int result;
 
     result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
-    lz4sd->dictionary = dest;
-    lz4sd->dictSize = result;
+    if (result <= 0) return result;
+    if (lz4sd->dictionary + lz4sd->dictSize == dest)
+    {
+        lz4sd->dictSize += result;
+    }
+    else
+    {
+        lz4sd->dictionary = dest;
+        lz4sd->dictSize = result;
+    }
 
     return result;
 }
@@ -1134,8 +1118,16 @@ int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, ch
     int result;
 
     result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
-    lz4sd->dictionary = dest;
-    lz4sd->dictSize = originalSize;
+    if (result <= 0) return result;
+    if (lz4sd->dictionary + lz4sd->dictSize == dest)
+    {
+        lz4sd->dictSize += result;
+    }
+    else
+    {
+        lz4sd->dictionary = dest;
+        lz4sd->dictSize = result;
+    }
 
     return result;
 }
@@ -1235,10 +1227,10 @@ int LZ4_compress_limitedOutput_withState (void* state, const char* source, char*
 
 int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
 {
-    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 0);
+    return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB);
 }
 
 int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
 {
-    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 0);
+    return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
 }

diff --git a/lz4.h b/lz4.h
old mode 100644 (file)
new mode 100755 (executable)
index 9d58a1a..1cf9eab
--- a/lz4.h
+++ b/lz4.h
@@ -241,7 +241,7 @@ int   LZ4_free (void* LZ4_stream);   /* yes, it's the same one as compression */
 *_continue() :
     These decoding functions allow decompression of multiple blocks in "streaming" mode.
     Previously decoded blocks must still be available at the memory position where they were decoded.
-    If it's not possible, save the relevant part of decoded data before it disappears,
+    If it's not possible, save the relevant part of decoded data into a safe buffer,
     and indicate where it stands using LZ4_setDictDecode()
 */
 int LZ4_decompress_safe_continue (void* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize);
@@ -249,9 +249,9 @@ int LZ4_decompress_fast_continue (void* LZ4_streamDecode, const char* source, ch
 
 /*
  * LZ4_setDictDecode
- * Use this function to instruct where to find the dictionary
- * This function is not necessary if previous data is still available where it was decoded.
- * Loading a size of 0 is allowed (same effect as no dictionary).
+ * Use this function to instruct where to find the dictionary.
+ * This function is not necessary if previous data is still available where it was already decoded.
+ * Setting a size of 0 is allowed (same effect as no dictionary).
  * Return : 1 if OK, 0 if error
  */
 int LZ4_setDictDecode (void* LZ4_streamDecode, const char* dictionary, int dictSize);

diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index d209fd3..6035213 100644 (file)
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -439,19 +439,19 @@ int FUZ_test(U32 seed, int nbCycles, int startCycle, double compressibility) {
             memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
             LZ4_loadDict(&LZ4dict, dict, dictSize);
             blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize);
-            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_usingDict failed");
+            FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed");
 
             FUZ_DISPLAYTEST;
             memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
             LZ4_loadDict(&LZ4dict, dict, dictSize);
             ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
-            FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_usingDict should fail : one missing byte for output buffer");
+            FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue should fail : one missing byte for output buffer");
 
             FUZ_DISPLAYTEST;
             memset(&LZ4dict, 0, sizeof(LZ4_stream_t));
             LZ4_loadDict(&LZ4dict, dict, dictSize);
             ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize);
-            FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_usingDict should work : enough size available within output buffer");
+            FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer");
 
             // Decompress with dictionary as external
             FUZ_DISPLAYTEST;