lz4frame : support stableDst option

Multiple bugfixes within lz4frame decompression
Small decompression speed improvements
Improved fuzzer test, with more thorough and complex tests

diff --git a/lz4.c b/lz4.c
index b39a91d..1c8e416 100644 (file)
--- a/lz4.c
+++ b/lz4.c
@@ -1155,7 +1155,9 @@ Advanced decoding functions :
 
 FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
 {
-    if ((dictStart+dictSize == source) && (dictSize >= (int)(64 KB - 1)))
+    if (dictSize==0)
+        return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, NULL, 64 KB);
+    if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1)))
         return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, NULL, 64 KB);
     return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, dictStart, dictSize);
 }

diff --git a/lz4frame.c b/lz4frame.c
index 77b96c9..7e06a4e 100644 (file)
--- a/lz4frame.c
+++ b/lz4frame.c
@@ -152,6 +152,37 @@ typedef struct {
 
 
 /**************************************
+   Error management
+**************************************/
+#define LZ4F_GENERATE_STRING(STRING) #STRING,
+static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
+
+/*
+typedef enum { OK_NoError=0, ERROR_GENERIC = 1,
+    ERROR_maxBlockSize_invalid, ERROR_blockMode_invalid, ERROR_contentChecksumFlag_invalid,
+    ERROR_compressionLevel_invalid,
+       ERROR_allocation_failed,
+       ERROR_srcSize_tooLarge, ERROR_dstMaxSize_tooSmall,
+       ERROR_decompressionFailed,
+    ERROR_checksum_invalid,
+       ERROR_maxCode
+       } LZ4F_errorCodes;    error codes are negative unsigned values.
+                                                       Compare function result to (-specificCode) */
+
+int LZ4F_isError(LZ4F_errorCode_t code)
+{
+       return (code > (LZ4F_errorCode_t)(-ERROR_maxCode));
+}
+
+const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
+{
+    static const char* codeError = "Unspecified error code";
+    if (LZ4F_isError(code)) return LZ4F_errorStrings[-code];
+    return codeError;
+}
+
+
+/**************************************
    Private functions
 **************************************/
 static size_t LZ4F_getBlockSize(unsigned blockSizeID)
@@ -191,16 +222,6 @@ static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length)
 }
 
 
-
-/**************************************
-   Error management
-**************************************/
-int LZ4F_isError(LZ4F_errorCode_t code)
-{
-       return (code > (LZ4F_errorCode_t)(-ERROR_maxCode));
-}
-
-
 /**************************************
    Simple compression functions
 **************************************/
@@ -693,9 +714,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
         dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize);
         if (dctxPtr->tmpOutBuffer== NULL) return -ERROR_GENERIC;
     }
+    dctxPtr->tmpInSize = 0;
+    dctxPtr->tmpInTarget = 0;
     dctxPtr->dict = dctxPtr->tmpOutBuffer;
     dctxPtr->dictSize = 0;
     dctxPtr->tmpOut = dctxPtr->tmpOutBuffer;
+    dctxPtr->tmpOutStart = 0;
+    dctxPtr->tmpOutSize = 0;
 
     return 7;
 }
@@ -704,7 +729,8 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt
 typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader,
                dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize,
                dstage_copyDirect,
-               dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, dstage_flushOut,
+               dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock,
+               dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut,
                dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix } dStage_t;
 
 
@@ -740,62 +766,74 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont
 }
 
 
-static void LZ4F_saveDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* decoded, size_t decodedSize)
+static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
 {
-    size_t newDictSize = decodedSize;
-    size_t preserveDictSize;
-    if (newDictSize > 64 KB) newDictSize = 64 KB;
-    preserveDictSize = 64 KB - newDictSize;
-    if (preserveDictSize > dctxPtr->dictSize) preserveDictSize = dctxPtr->dictSize;
-
-    memmove(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveDictSize, preserveDictSize);
-    memmove(dctxPtr->tmpOutBuffer + preserveDictSize, decoded + decodedSize - newDictSize, newDictSize);
-
-    dctxPtr->dict = dctxPtr->tmpOutBuffer;
-    dctxPtr->dictSize = preserveDictSize + newDictSize;
-    dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize;
+    (void)dictStart; (void)dictSize;
+    return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
 }
 
 
-static void LZ4F_pointDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* decoded, size_t decodedSize)
+
+static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
 {
-    /* decoded block in the continuity of dictionary */
-    if (dctxPtr->dict + dctxPtr->dictSize == decoded)
+    if (dctxPtr->dictSize==0)
+        dctxPtr->dict = (BYTE*)dstPtr;   /* priority to dictionary continuity */
+
+    if (dctxPtr->dict + dctxPtr->dictSize == dstPtr)   /* dictionary continuity */
     {
-        dctxPtr->dictSize += decodedSize;
-        if (dctxPtr->dict == dctxPtr->tmpOutBuffer)   /* extended tmp buffer, don't go beyond 128 KB == maxDictSize */
-        {
-            if (dctxPtr->dictSize > 128 KB)
-            {
-                memcpy(dctxPtr->tmpOutBuffer, dctxPtr->tmpOutBuffer + dctxPtr->dictSize - 64 KB, 64 KB);
-                dctxPtr->dictSize = 64 KB;
-            }
-            dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize;
-        }
+        dctxPtr->dictSize += dstSize;
         return;
     }
 
-    /* large decoded block */
-    if (decodedSize >= (64 KB - 1))
+    if (dstPtr - dstPtr0 + dstSize >= 64 KB)   /* dstBuffer large enough to become dictionary */
     {
-        dctxPtr->dict = (BYTE*)decoded;
-        dctxPtr->dictSize = decodedSize;
-        dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + 64 KB;
+        dctxPtr->dict = (BYTE*)dstPtr0;
+        dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize;
         return;
     }
 
-    /* small block, and not contiguous : let's save that */
-    LZ4F_saveDict(dctxPtr, decoded, decodedSize);
-}
+    if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer))
+    {
+        /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */
+        dctxPtr->dictSize += dstSize;
+        return;
+    }
 
+    if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
+    {
+        size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
+        memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize);
+        dctxPtr->dict = dctxPtr->tmpOutBuffer;
+        dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize;
+        return;
+    }
 
-static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
-{
-    (void)dictStart; (void)dictSize;
-    return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
+    if (dctxPtr->dict == dctxPtr->tmpOutBuffer)     /* copy dst into tmp to complete dict */
+    {
+        if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize)   /* tmp buffer not large enough */
+        {
+            size_t preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
+            memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
+            dctxPtr->dictSize = preserveSize;
+        }
+        memcpy(dctxPtr->dict + dctxPtr->dictSize, dstPtr, dstSize);
+        dctxPtr->dictSize += dstSize;
+        return;
+    }
+
+    /* join dict & dest into tmp */
+    {
+        size_t preserveSize = 64 KB - dstSize;   /* note : dstSize < 64 KB */
+        if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize;
+        memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize);
+        memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize);
+        dctxPtr->dict = dctxPtr->tmpOutBuffer;
+        dctxPtr->dictSize = preserveSize + dstSize;
+    }
 }
 
 
+
 /* LZ4F_decompress()
  * Call this function repetitively to regenerate data compressed within srcBuffer.
  * The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
@@ -952,7 +990,11 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                 if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr;
                 memcpy(dstPtr, srcPtr, sizeToCopy);
                 if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy);
-                if (dctxPtr->frameInfo.blockMode==blockLinked) LZ4F_pointDict(dctxPtr, srcPtr, sizeToCopy);
+
+                /* dictionary management */
+                if (dctxPtr->frameInfo.blockMode==blockLinked)
+                    LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0);
+
                 srcPtr += sizeToCopy;
                 dstPtr += sizeToCopy;
                 if (sizeToCopy == dctxPtr->tmpInTarget)   /* all copied */
@@ -1000,6 +1042,15 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
 
         case dstage_decodeCBlock:
             {
+                if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize)   /* not enough place into dst : decode into tmpOut */
+                    dctxPtr->dStage = dstage_decodeCBlock_intoTmp;
+                else
+                    dctxPtr->dStage = dstage_decodeCBlock_intoDst;
+                break;
+            }
+
+        case dstage_decodeCBlock_intoDst:
+            {
                 int (*decoder)(const char*, char*, int, int, const char*, int);
                 int decodedSize;
 
@@ -1008,35 +1059,76 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                 else
                     decoder = LZ4F_decompress_safe;
 
-                if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize)   /* not enough place into dst : decode into tmpOut */
-                {
-                    decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
-                    if (decodedSize < 0) return -ERROR_GENERIC;   /* decompression failed */
-                    if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
-                    dctxPtr->tmpOutSize = decodedSize;
-                    dctxPtr->tmpOutStart = 0;
-                    dctxPtr->dStage = dstage_flushOut;
-                    break;
-                }
                 decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
                 if (decodedSize < 0) return -ERROR_GENERIC;   /* decompression failed */
                 if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize);
-                if (dctxPtr->frameInfo.blockMode==blockLinked) LZ4F_pointDict(dctxPtr, dstPtr, decodedSize);
+
+                /* dictionary management */
+                if (dctxPtr->frameInfo.blockMode==blockLinked)
+                    LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0);
+
                 dstPtr += decodedSize;
                 dctxPtr->dStage = dstage_getCBlockSize;
                 break;
             }
 
+        case dstage_decodeCBlock_intoTmp:
+            {
+                /* not enough place into dst : decode into tmpOut */
+                int (*decoder)(const char*, char*, int, int, const char*, int);
+                int decodedSize;
+
+                if (dctxPtr->frameInfo.blockMode == blockLinked)
+                    decoder = LZ4_decompress_safe_usingDict;
+                else
+                    decoder = LZ4F_decompress_safe;
+
+                /* ensure enough place for tmpOut */
+                if (dctxPtr->frameInfo.blockMode == blockLinked)
+                {
+                    if (dctxPtr->dict == dctxPtr->tmpOutBuffer)
+                    {
+                        if (dctxPtr->dictSize > 128 KB)
+                        {
+                            memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB);
+                            dctxPtr->dictSize = 64 KB;
+                        }
+                        dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize;
+                    }
+                    else   /* dict not within tmp */
+                    {
+                        size_t reservedDictSpace = dctxPtr->dictSize;
+                        if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB;
+                        dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace;
+                    }
+                }
+
+                /* Decode */
+                decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize);
+                if (decodedSize < 0) return -ERROR_decompressionFailed;   /* decompression failed */
+                if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize);
+                dctxPtr->tmpOutSize = decodedSize;
+                dctxPtr->tmpOutStart = 0;
+                dctxPtr->dStage = dstage_flushOut;
+                break;
+            }
+
         case dstage_flushOut:  /* flush decoded data from tmpOut to dstBuffer */
             {
                 size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart;
                 if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr;
                 memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy);
+
+                /* dictionary management */
+                if (dctxPtr->frameInfo.blockMode==blockLinked)
+                    LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1);
+
                 dctxPtr->tmpOutStart += sizeToCopy;
                 dstPtr += sizeToCopy;
+
+                /* end of flush ? */
                 if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize)
                 {
-                    if (dctxPtr->frameInfo.blockMode==blockLinked) LZ4F_pointDict(dctxPtr, dctxPtr->tmpOut, dctxPtr->tmpOutSize);
                     dctxPtr->dStage = dstage_getCBlockSize;
                     break;
                 }
@@ -1045,7 +1137,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
                 break;
             }
 
-        case dstage_getSuffix:
+       case dstage_getSuffix:
             {
                 size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4;
                 if (suffixSize == 0)   /* frame completed */
@@ -1098,20 +1190,35 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
         }
     }
 
+    /* preserve dictionary within tmp if necessary */
     if ( (dctxPtr->frameInfo.blockMode==blockLinked)
        &&(dctxPtr->dict != dctxPtr->tmpOutBuffer)
-        )
-        LZ4F_saveDict(dctxPtr, NULL, 0);
-        //(!decompressOptionsPtr->stableDst + 1) )
-    /*{
-        size_t newDictSize = dctxPtr->dictSize;
-        BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
-        if ((newDictSize) > 64 KB) newDictSize = 64 KB;
-        memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
-        dctxPtr->dict = dctxPtr->tmpOutBuffer;
-        dctxPtr->dictSize = newDictSize;
-        dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
-    }*/
+       &&(!decompressOptionsPtr->stableDst)
+       )
+    {
+        if (dctxPtr->dStage == dstage_flushOut)
+        {
+            size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer;
+            BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart;
+
+            memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - preserveSize, preserveSize);
+
+            dctxPtr->dict = dctxPtr->tmpOutBuffer;
+            dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart;
+        }
+        else
+        {
+            size_t newDictSize = dctxPtr->dictSize;
+            BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize;
+            if ((newDictSize) > 64 KB) newDictSize = 64 KB;
+
+            memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
+
+            dctxPtr->dict = dctxPtr->tmpOutBuffer;
+            dctxPtr->dictSize = newDictSize;
+            dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize;
+        }
+    }
 
     if (srcPtr<srcEnd)   /* function must be called again with following source data */
         dctxPtr->srcExpect = srcPtr;

diff --git a/lz4frame.h b/lz4frame.h
index 4bc8c59..d24a824 100644 (file)
--- a/lz4frame.h
+++ b/lz4frame.h
@@ -59,18 +59,21 @@ extern "C" {
    Error management
 **************************************/
 typedef size_t LZ4F_errorCode_t;
-typedef enum { OK_FrameEnd = 1 } LZ4F_successCodes;
-typedef enum { OK_NoError = 0, ERROR_GENERIC = 1,
-    ERROR_maxBlockSize_invalid, ERROR_blockMode_invalid, ERROR_contentChecksumFlag_invalid,
-    ERROR_compressionLevel_invalid,
-       ERROR_allocation_failed,
-       ERROR_srcSize_tooLarge, ERROR_dstMaxSize_tooSmall,
-    ERROR_checksum_invalid,
-       ERROR_maxCode
-       } LZ4F_errorCodes;   /* error codes are negative unsigned values.
-                                                       Compare function result to (-specificCode) */
+#define LZ4F_LIST_ERRORS(ITEM) \
+        ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
+        ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
+        ITEM(ERROR_compressionLevel_invalid) \
+        ITEM(ERROR_allocation_failed) \
+        ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_decompressionFailed) \
+        ITEM(ERROR_checksum_invalid) \
+        ITEM(ERROR_maxCode)
+
+#define LZ4F_GENERATE_ENUM(ENUM) ENUM,
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;  /* enum is exposed, to let programmer detect & handle specific errors */
 
 int LZ4F_isError(LZ4F_errorCode_t code);   /* Basically : code > -ERROR_maxCode */
+const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /* return enum as string */
 
 
 /**************************************

diff --git a/programs/frametest.c b/programs/frametest.c
index 9f14bfb..03b47e8 100644 (file)
--- a/programs/frametest.c
+++ b/programs/frametest.c
@@ -105,6 +105,7 @@ static U32 g_time = 0;
 static U32 no_prompt = 0;
 static char* programName;
 static U32 displayLevel = 2;
+static U32 pause = 0;
 
 
 /*********************************************************
@@ -355,13 +356,14 @@ _output_error:
 }
 
 
-static void locateBuffDiff(const void* buff1, const void* buff2, size_t size)
+static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
 {
     int p=0;
     BYTE* b1=(BYTE*)buff1;
     BYTE* b2=(BYTE*)buff2;
+    if (nonContiguous) { DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); return; }
     while (b1[p]==b2[p]) p++;
-    printf("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
+    DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
  }
 
 
@@ -378,6 +380,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
        LZ4F_decompressionContext_t dCtx;
        LZ4F_compressionContext_t cCtx;
     size_t result;
+    XXH64_stateSpace_t xxh64;
 #   define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
                             DISPLAY(" (seed %u, test nb %i)  \n", seed, testNb); goto _output_error; }
 
@@ -414,6 +417,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         size_t cSize;
         U64 crcOrig, crcDecoded;
 
+        (void)FUZ_rand(&coreRand);   // update rand seed
         prefs.frameInfo.blockMode = BMId;
         prefs.frameInfo.blockSizeID = BSId;
         prefs.frameInfo.contentChecksumFlag = CCflag;
@@ -464,6 +468,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
             BYTE* op = decodedBuffer;
             BYTE* const oend = op + srcDataLength;
             unsigned maxBits = FUZ_highbit(cSize);
+            unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1;
+            nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst;   /* 0=>0; 1=>1,2 */
+            XXH64_resetState(&xxh64, 1);
             while (ip < iend)
             {
                 unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
@@ -473,19 +480,24 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
                 if (oSize > (size_t)(oend-op)) oSize = oend-op;
                 dOptions.stableDst = FUZ_rand(&randState) & 1;
+                if (nonContiguousDst==2) dOptions.stableDst = 0;
+                //if (ip == compressedBuffer+62073)                    DISPLAY("oSize : %i : pos %i \n", (int)oSize, (int)(op-(BYTE*)decodedBuffer));
                 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
-                if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
-                CHECK(LZ4F_isError(result), "Decompression failed (error %i)", (int)result);
+                //if (op+oSize >= (BYTE*)decodedBuffer+94727)                    DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
+                //if ((int)result<0)                    DISPLAY("iSize : %i : pos %i \n", (int)iSize, (int)(ip-(BYTE*)compressedBuffer));
+                if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
+                CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result));
+                XXH64_update(&xxh64, op, (U32)oSize);
                 op += oSize;
                 ip += iSize;
+                op += nonContiguousDst;
+                if (nonContiguousDst==2) op = decodedBuffer;   // overwritten destination
             }
             CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
-            crcDecoded = XXH64(decodedBuffer, op-(BYTE*)decodedBuffer, 1);
-            if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize);
+            crcDecoded = XXH64_intermediateDigest(&xxh64);
+            if (crcDecoded != crcOrig) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst);
             CHECK(crcDecoded != crcOrig, "Decompression corruption");
         }
-
-        (void)FUZ_rand(&coreRand);   // update rand seed
     }
 
        DISPLAYLEVEL(2, "\rAll tests completed   \n");
@@ -496,6 +508,12 @@ _end:
        free(srcBuffer);
        free(compressedBuffer);
        free(decodedBuffer);
+
+       if (pause)
+    {
+        DISPLAY("press enter to finish \n");
+        getchar();
+    }
        return testResult;
 
 _output_error:
@@ -542,10 +560,10 @@ int main(int argc, char** argv)
         if (argument[0]=='-')
         {
             if (!strcmp(argument, "--no-prompt")) { no_prompt=1; seedset=1; displayLevel=1; continue; }
+            argument++;
 
-            while (argument[1]!=0)
+            while (*argument!=0)
             {
-                argument++;
                 switch(*argument)
                 {
                 case 'h':
@@ -588,7 +606,7 @@ int main(int argc, char** argv)
                         argument++;
                     }
                     break;
-                case 'p':
+                case 'p':   /* compressibility % */
                     argument++;
                     proba=0;
                     while ((*argument>='0') && (*argument<='9'))
@@ -600,7 +618,12 @@ int main(int argc, char** argv)
                     if (proba<0) proba=0;
                     if (proba>100) proba=100;
                     break;
+                case 'P': /* pause at the end */
+                    argument++;
+                    pause = 1;
+                    break;
                 default: ;
+                    return FUZ_usage();
                 }
             }
         }

diff --git a/programs/fullbench.c b/programs/fullbench.c
index 9292f20..5ee1710 100644 (file)
--- a/programs/fullbench.c
+++ b/programs/fullbench.c
@@ -343,14 +343,14 @@ static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, in
 static int local_LZ4_decompress_fast_usingDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
-    LZ4_decompress_fast_usingDict(in, out, outSize, in - 65536, 65536);
+    LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536);
     return outSize;
 }
 
 static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
-    LZ4_decompress_safe_usingDict(in, out, inSize, outSize, in - 65536, 65536);
+    LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536);
     return outSize;
 }
 
@@ -359,7 +359,7 @@ extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSiz
 static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
 {
     (void)inSize;
-    LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, in - 65536, 65536);
+    LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
     return outSize;
 }