Fixed unfinished frame (issue #75)

diff --git a/NEWS b/NEWS
index 462a23b..e797ff7 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,9 +1,13 @@
 r129:
+New    : LZ4 CLI improved performance when compressing/decompressing multiple files (#86, kind contribution from Kyle J. Harper & Takayuki Matsuoka)
 Added  : LZ4_compress_fast()
-Changed: New compression functions LZ4_compress_safe() - Suggested by Evan Nemerson & Takayuki Matsuoka
+Changed: Sparse file support enabled by default
+Changed: New function names LZ4_compress_safe() - Suggested by Evan Nemerson & Takayuki Matsuoka
+Fixed  : GCC 4.9+ optimization bug - Reported by Markus Trippelsdorf, Greg Slazinski & Evan Nemerson
+Changed: Enums converted to LZ4F_ namespace convention - by Takayuki Matsuoka
 Added  : AppVeyor CI environment, for Visual tests - Suggested by Takayuki Matsuoka
-Fixed  : GCC 4.9+ optimization bug - Reported by Markus Trippelsdorf, Greg Slazinski, Evan Nemerson & Takayuki Matsuoka
 Modified:Obsolete functions generate warnings - Suggested by Evan Nemerson & Takayuki Matsuoka
+Fixed  : Bug #75 (unfinished stream), reported by Yongwoon Cho
 Updated: Documentation converted to MarkDown
 
 r128:

diff --git a/programs/Makefile b/programs/Makefile
index 910516f..b0e30d8 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -207,7 +207,13 @@ test-lz4-multiple: lz4 datagen
        ./lz4 -f -m tmp1 notHere tmp2; echo $$?
        @rm tmp*
 
-test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
+# test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
+test-lz4: lz4 datagen 
+       @echo ---- test erroneous data ----
+       ./datagen -s1     | ./lz4 -vf - tmp
+       tmpSize= $(stat -c%s "tmp")
+       echo "Size of tmp = $tmpSize bytes."
+       rm tmp*
        @echo ---- test lz4 basic compression/decompression ----
        ./datagen -g0     | ./lz4 -v     | ./lz4 -t
        ./datagen -g16KB  | ./lz4 -9     | ./lz4 -t

diff --git a/programs/frametest.c b/programs/frametest.c
index ed131d2..237fd4a 100644 (file)
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -279,6 +279,20 @@ int basicTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(4, "Reusing decompression context \n");
         {
+            size_t iSize = compressedBufferSize - 4;
+            DISPLAYLEVEL(3, "Missing last 4 bytes : ");
+            errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &iSize, NULL);
+            if (LZ4F_isError(errorCode)) goto _output_error;
+            if (!errorCode) goto _output_error;
+            DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode);
+            iSize = errorCode;
+            errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &iSize, NULL);
+            if (errorCode != 0) goto _output_error;
+            crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+            if (crcDest != crcOrig) goto _output_error;
+        }
+
+        {
             size_t oSize = 0;
             size_t iSize = 0;
             LZ4F_frameInfo_t fi;

diff --git a/programs/lz4io.c b/programs/lz4io.c
index e0c69d8..cbf366b 100644 (file)
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -807,7 +807,7 @@ static void LZ4IO_freeDResources(dRess_t ress)
 static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile)
 {
     unsigned long long filesize = 0;
-    LZ4F_errorCode_t errorCode;
+    LZ4F_errorCode_t nextToLoad;
     unsigned storedSkips = 0;
 
     /* Init feed with magic number (already consumed from FILE*  sFile) */
@@ -815,8 +815,8 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
         size_t inSize = MAGICNUMBER_SIZE;
         size_t outSize= 0;
         LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
-        errorCode = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL);
-        if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode));
+        nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL);
+        if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
     }
 
     /* Main Loop */
@@ -824,18 +824,20 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
     {
         size_t readSize;
         size_t pos = 0;
+        size_t decodedBytes = ress.dstBufferSize;
 
         /* Read input */
         readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile);
-        if (!readSize) break;   /* empty file or stream */
+        if (!readSize)
+            break;   /* empty file or stream */
 
-        while (pos < readSize)
+        while (nextToLoad && ((pos < readSize) || (decodedBytes == ress.dstBufferSize)))   /* still to read, or still to flush */
         {
             /* Decode Input (at least partially) */
             size_t remaining = readSize - pos;
-            size_t decodedBytes = ress.dstBufferSize;
-            errorCode = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
-            if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode));
+            decodedBytes = ress.dstBufferSize;
+            nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
+            if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
             pos += remaining;
 
             if (decodedBytes)
@@ -850,6 +852,9 @@ static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE
 
     LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
 
+    if (nextToLoad!=0)
+        EXM_THROW(67, "Unfinished stream");
+
     return filesize;
 }