New function : LZ4_uncompress : secure version which is safe against buffer overflow...

New function : LZ4_uncompress_unknownOutputSize : secure but slower version which also guess the size of data to be decoded
The demo file compression program (main.c) is updated to use these new functions
LZ4_decode is still supported but in deprecated status, due to its vulnerability to malicious buffer overflow scenario

git-svn-id: https://lz4.googlecode.com/svn/trunk@9 650e7d94-2a16-8b24-b05c-7c0b3f6821cd

diff --git a/lz4.c b/lz4.c
index 38cdc79..fbf89c0 100644 (file)
--- a/lz4.c
+++ b/lz4.c
@@ -36,6 +36,16 @@
 \r
 \r
 //**************************************\r
+// Performance parameter                <---------------------------------------------------------\r
+//**************************************\r
+// Lowering this value reduce memory usage\r
+// It may also improve speed, especially if you reach L1 cache size (32KB for Intel, 64KB for AMD)\r
+// Expanding memory usage typically improves compression ratio\r
+// Memory usage formula : N->2^(N+2) Bytes (examples : 17 -> 512KB ; 12 -> 16KB)\r
+#define HASH_LOG 17                        \r
+\r
+\r
+//**************************************\r
 // Basic Types\r
 //**************************************\r
 #if defined(_MSC_VER) || defined(_WIN32) || defined(__WIN32__)\r
@@ -61,7 +71,6 @@
 #define MAXD_LOG 16\r
 #define MAX_DISTANCE ((1 << MAXD_LOG) - 1)\r
 \r
-#define HASH_LOG 17                        // <--- Lower this value to lower memory usage. N->2^(N+2) Bytes (ex : 17 -> 512KB)\r
 #define HASHTABLESIZE (1 << HASH_LOG)\r
 #define HASH_MASK (HASHTABLESIZE - 1)\r
 \r
@@ -145,27 +154,21 @@ int LZ4_compressCtx(void** ctx,
                ref = HashTable[h];\r
                HashTable[h] = ip;\r
 \r
-               // Min Match\r
+               // Check Min Match\r
                if (( ((ip-ref) >> MAXD_LOG) != 0) || (*(U32*)ref != sequence))\r
                { \r
-                       if (ip-anchor>limit) { limit<<=1; step += 1 + (step>>2); }\r
-                       ip+=step; \r
+                       if (ip-anchor>limit) { limit <<= 1; step += 1 + (step>>2); }\r
+                       ip += step; \r
                        continue; \r
                }       \r
 \r
                // catch up\r
-               if (step>1) \r
-               { \r
-                       HashTable[h] = ref;\r
-                       ip -= (step-1);\r
-                       step=1; \r
-                       continue;\r
-               }\r
-               limit=INCOMPRESSIBLE; \r
+               if (step>1) { HashTable[h] = ref; ip -= (step-1); step=1; continue; }\r
+               limit = INCOMPRESSIBLE; \r
 \r
                // Encode Literal length\r
                len = length = ip - anchor;\r
-               orun=op++;\r
+               orun = op++;\r
                if (len>(RUN_MASK-1)) { *orun=(RUN_MASK<<ML_BITS); len-=RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } \r
                else *orun = (len<<ML_BITS);\r
 \r
@@ -191,14 +194,16 @@ int LZ4_compressCtx(void** ctx,
                anchor = ip; \r
        }\r
 \r
-\r
        // Encode Last Literals\r
        len = length = iend - anchor;\r
-    orun=op++;\r
-    if (len>(RUN_MASK-1)) { *orun=(RUN_MASK<<ML_BITS); len-=RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE) len; } \r
-       else *orun = (len<<ML_BITS);\r
-       for(;length>0;length-=4) { *(U32*)op = *(U32*)anchor; op+=4; anchor+=4; }\r
-       op += length;    // correction\r
+       if (length)\r
+       {\r
+               orun=op++;\r
+               if (len>(RUN_MASK-1)) { *orun=(RUN_MASK<<ML_BITS); len-=RUN_MASK; for(; len > 254 ; len-=255) *op++ = 255; *op++ = (BYTE) len; } \r
+               else *orun = (len<<ML_BITS);\r
+               for(;length>0;length-=4) { *(U32*)op = *(U32*)anchor; op+=4; anchor+=4; }\r
+               op += length;    // correction\r
+       }\r
 \r
        // End\r
        return (int) (((char*)op)-dest);\r
@@ -209,15 +214,15 @@ int LZ4_compressCtx(void** ctx,
 //****************************\r
 // Decompression CODE\r
 //****************************\r
-int LZ4_decode ( char* source, \r
+int LZ4_uncompress(char* source, \r
                                 char* dest,\r
-                                int isize)\r
+                                int osize)\r
 {      \r
        // Local Variables\r
-       BYTE    *ip = (BYTE*) source,      \r
-                       *iend = (BYTE*) source + isize;\r
+       BYTE    *ip = (BYTE*) source;\r
 \r
        BYTE    *op = (BYTE*) dest, \r
+                       *oend=(BYTE*) dest + osize,\r
                        *ref, *cpy,\r
                        runcode;\r
        \r
@@ -226,7 +231,7 @@ int LZ4_decode ( char* source,
 \r
 \r
        // Main Loop\r
-       while (ip < iend)\r
+       while (1)\r
        {\r
                // get runlength\r
                runcode = *ip++;\r
@@ -234,19 +239,21 @@ int LZ4_decode ( char* source,
 \r
                // copy literals\r
                ref = op+length;\r
-#ifdef SAFEWRITEBUFFER\r
-               if (ref>iend-4) { while(op<iend-3) { *(U32*)op=*(U32*)ip; op+=4; ip+=4; } while(op<ref) *op++=*ip++; } \r
-               else\r
-#endif\r
+               if (ref>oend-4) \r
+               { \r
+                       if (ref > oend) goto _output_error;\r
+                       while(op<oend-3) { *(U32*)op=*(U32*)ip; op+=4; ip+=4; } \r
+                       while(op<ref) *op++=*ip++; \r
+                       break;    // Necessarily EOF\r
+               }\r
                while (op<ref) { *(U32*)op = *(U32*)ip; op+=4; ip+=4; }\r
                ip-=(op-ref); op=ref;   // correction\r
-               if (ip>=iend) break;    // Check EOF\r
 \r
                // get offset\r
                ref -= *(U16*)ip; ip+=2;\r
 \r
                // get matchlength\r
-               if ((length=(runcode&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } \r
+               if ((length=(runcode&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }\r
                length += MINMATCH;\r
 \r
                // copy repeated sequence\r
@@ -259,17 +266,157 @@ int LZ4_decode ( char* source,
                        *op++ = *ref++;\r
                        ref -= dec[op-ref];\r
                }\r
-#ifdef SAFEWRITEBUFFER\r
-               if (cpy>iend-4) { while(op<iend-3) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; } while(op<cpy) *op++=*ref++; } \r
-               else\r
-#endif\r
+               if (cpy>oend-4)\r
+               {\r
+                       if (cpy > oend) goto _output_error;\r
+                       while(op<cpy-3) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }\r
+                       while(op<cpy) *op++=*ref++;\r
+                       if (op>=oend) break;    // Check EOF\r
+                       continue;\r
+               }\r
+               while(op<cpy) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }\r
+               op=cpy;         // correction\r
+       }\r
+\r
+       // end of decoding\r
+       return (int) (((char*)ip)-source);\r
+\r
+       // write overflow error detected\r
+_output_error:\r
+       return (int) (-(((char*)ip)-source));\r
+}\r
+\r
+\r
+int LZ4_uncompress_unknownOutputSize(\r
+                               char* source, \r
+                               char* dest,\r
+                               int isize,\r
+                               int maxOutputSize)\r
+{      \r
+       // Local Variables\r
+       BYTE    *ip = (BYTE*) source,\r
+                       *iend = ip + isize;\r
+\r
+       BYTE    *op = (BYTE*) dest, \r
+                       *oend = op + maxOutputSize,\r
+                       *ref, *cpy,\r
+                       runcode;\r
+       \r
+       U32             dec[4]={0, 3, 2, 3};\r
+       int             len, length;\r
+\r
+\r
+       // Main Loop\r
+       while (ip<iend)\r
+       {\r
+               // get runlength\r
+               runcode = *ip++;\r
+               if ((length=(runcode>>ML_BITS)) == RUN_MASK)  { for (;(len=*ip++)==255;length+=255){} length += len; } \r
+\r
+               // copy literals\r
+               ref = op+length;\r
+               if (ref>oend-4) \r
+               { \r
+                       if (ref > oend) goto _output_error;\r
+                       while(op<oend-3) { *(U32*)op=*(U32*)ip; op+=4; ip+=4; } \r
+                       while(op<ref) *op++=*ip++; \r
+                       break;    // Necessarily EOF\r
+               }\r
+               while (op<ref) { *(U32*)op = *(U32*)ip; op+=4; ip+=4; }\r
+               ip-=(op-ref); op=ref;   // correction\r
+               if (ip>=iend) break;    // check EOF\r
+\r
+               // get offset\r
+               ref -= *(U16*)ip; ip+=2;\r
+\r
+               // get matchlength\r
+               if ((length=(runcode&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; }\r
+               length += MINMATCH;\r
+\r
+               // copy repeated sequence\r
+               cpy = op + length;\r
+               if (op-ref<4)\r
+               {\r
+                       *op++ = *ref++;\r
+                       *op++ = *ref++;\r
+                       *op++ = *ref++;\r
+                       *op++ = *ref++;\r
+                       ref -= dec[op-ref];\r
+               }\r
+               if (cpy>oend-4)\r
+               {\r
+                       if (cpy > oend) goto _output_error;\r
+                       while(op<cpy-3) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }\r
+                       while(op<cpy) *op++=*ref++;\r
+                       if (op>=oend) break;    // Check EOF\r
+                       continue;\r
+               }\r
                while(op<cpy) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }\r
                op=cpy;         // correction\r
        }\r
 \r
        // end of decoding\r
        return (int) (((char*)op)-dest);\r
+\r
+       // write overflow error detected\r
+_output_error:\r
+       return (int) (-(((char*)ip)-source));\r
 }\r
 \r
 \r
+//****************************\r
+// Deprecated functions\r
+//****************************\r
+int LZ4_decode ( char* source, \r
+                                char* dest,\r
+                                int isize)\r
+{      \r
+       // Local Variables\r
+       BYTE    *ip = (BYTE*)source,      \r
+                       *iend = ip + isize;\r
+\r
+       BYTE    *op = (BYTE*)dest, \r
+                       *ref, *cpy,\r
+                       runcode;\r
+       \r
+       U32             dec[4]={0, 3, 2, 3};\r
+       int             len, length;\r
+\r
+\r
+       // Main Loop\r
+       while (ip < iend)\r
+       {\r
+               // get runlength\r
+               runcode = *ip++;\r
+               if ((length=(runcode>>ML_BITS)) == RUN_MASK)  { for (;(len=*ip++)==255;length+=255){} length += len; } \r
+\r
+               // copy literals\r
+               ref=op+length;\r
+               while (op<ref) { *(U32*)op = *(U32*)ip; op+=4; ip+=4; }\r
+               ip-=(op-ref); op=ref;   // correction\r
+               if (ip>=iend) break;    // Check EOF\r
+\r
+               // get offset\r
+               ref -= *(U16*)ip; ip+=2;\r
+\r
+               // get matchlength\r
+               if ((length=(runcode&ML_MASK)) == ML_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } \r
+               length += MINMATCH;\r
+\r
+               // copy repeated sequence\r
+               cpy = op + length;\r
+               if (op-ref<4)\r
+               {\r
+                       *op++ = *ref++;\r
+                       *op++ = *ref++;\r
+                       *op++ = *ref++;\r
+                       *op++ = *ref++;\r
+                       ref -= dec[op-ref];\r
+               }\r
+               while(op<cpy) { *(U32*)op=*(U32*)ref; op+=4; ref+=4; }\r
+               op=cpy;         // correction\r
+       }\r
 \r
+       // end of decoding\r
+       return (int) (((char*)op)-dest);\r
+}\r

diff --git a/lz4.h b/lz4.h
index b932b39..031e8ee 100644 (file)
--- a/lz4.h
+++ b/lz4.h
@@ -34,21 +34,11 @@ extern "C" {
 \r
 \r
 //****************************\r
-// Instructions\r
-//****************************\r
-\r
-// Uncomment next line to ensure that LZ4_Decode will never write in destination buffer more than "decompressedSize" bytes\r
-// If commented, the decoder may write up to 3 bytes more than decompressedSize, so provide extra room in dest buffer for that\r
-// Recommendation : keep commented, for improved performance; ensure that destination buffer is at least decompressedSize + 3 Bytes\r
-// #define SAFEWRITEBUFFER\r
- \r
-\r
-//****************************\r
 // Simple Functions\r
 //****************************\r
 \r
-int LZ4_compress (char* source, char* dest, int isize);\r
-int LZ4_decode   (char* source, char* dest, int isize);\r
+int LZ4_compress   (char* source, char* dest, int isize);\r
+int LZ4_uncompress (char* source, char* dest, int osize);\r
 \r
 /*\r
 LZ4_compress :\r
@@ -57,12 +47,12 @@ LZ4_compress :
                To avoid any problem, size it to handle worst cases situations (input data not compressible)\r
                Worst case size is : "inputsize + 0.4%", with "0.4%" being at least 8 bytes.\r
 \r
-LZ4_decode :\r
-       return : the number of bytes in decoded buffer dest\r
-       note 1 : isize is the input size, therefore the compressed size\r
-       note 2 : destination buffer must be already allocated. \r
-                       The program calling the decoder must know in advance the size of decoded stream to properly allocate the destination buffer\r
-                       Note that, in fast mode, the destination buffer size must be at least "decompressedSize + 3 Bytes"\r
+LZ4_uncompress :\r
+       return : the number of bytes read in the source buffer\r
+                        If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction\r
+                        This version never writes beyond dest + osize, and is therefore protected against malicious data packets\r
+       note 1 : osize is the output size, therefore the original size\r
+       note 2 : destination buffer must be already allocated\r
 */\r
 \r
 \r
@@ -70,6 +60,19 @@ LZ4_decode :
 // Advanced Functions\r
 //****************************\r
 \r
+int LZ4_uncompress_unknownOutputSize (char* source, char* dest, int isize, int maxOutputSize);\r
+\r
+/*\r
+LZ4_uncompress :\r
+       return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)\r
+                        If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction\r
+                        This version never writes beyond dest + osize, and is therefore protected against malicious data packets\r
+       note 1 : isize is the input size, therefore the compressed size\r
+       note 2 : destination buffer must be already allocated\r
+       note 3 : this version is slower by up to 10%, and is therefore not recommended for general use\r
+*/\r
+\r
+\r
 int LZ4_compressCtx(void** ctx, char* source,  char* dest, int isize);\r
 \r
 /*\r
@@ -85,6 +88,24 @@ LZ4_compressCtx :
 */\r
 \r
 \r
+//****************************\r
+// Deprecated Functions\r
+//****************************\r
+\r
+int LZ4_decode   (char* source, char* dest, int isize);\r
+\r
+/*\r
+LZ4_decode :\r
+       return : the number of bytes in decoded buffer dest\r
+       note 1 : isize is the input size, therefore the compressed size\r
+       note 2 : destination buffer must be already allocated. \r
+                       The program calling the decoder must know in advance the size of decoded stream to properly allocate the destination buffer\r
+                       The destination buffer size must be at least "decompressedSize + 3 Bytes"\r
+                       This version is unprotected against malicious data packets designed to create buffer overflow errors.\r
+                       It is therefore deprecated, but still present in this version for compatibility.\r
+*/\r
+\r
+\r
 #if defined (__cplusplus)\r
 }\r
 #endif\r

diff --git a/main.c b/main.c
index cf94577..8e28e13 100644 (file)
--- a/main.c
+++ b/main.c
@@ -63,7 +63,7 @@
 \r
 #define CHUNKSIZE (8<<20)    // 8 MB\r
 #define CACHELINE 64\r
-#define OUT_CHUNKSIZE (CHUNKSIZE + CHUNKSIZE/256 + CACHELINE)\r
+#define OUT_CHUNKSIZE (CHUNKSIZE + (CHUNKSIZE>>8) + CACHELINE)\r
 #define ARCHIVE_MAGICNUMBER 0x184C2102\r
 #define ARCHIVE_MAGICNUMBER_SIZE 4\r
 \r
@@ -149,6 +149,8 @@ int decode_file(char* input_filename, char* output_filename)
        FILE* finput = fopen( input_filename, "rb" ); \r
        FILE* foutput = fopen( output_filename, "wb" ); \r
        size_t uselessRet;\r
+       int sinkint;\r
+       U32 nextSize;\r
        \r
        if (finput==0 ) { printf("Pb opening %s\n", input_filename);  return 4; }\r
        if (foutput==0) { printf("Pb opening %s\n", output_filename); return 5; }\r
@@ -160,25 +162,33 @@ int decode_file(char* input_filename, char* output_filename)
        // Check Archive Header\r
        uselessRet = fread(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, finput);\r
        if (*(U32*)out_buff != ARCHIVE_MAGICNUMBER) { printf("Wrong file : cannot be decoded\n"); return 6; }\r
+       uselessRet = fread(in_buff, 1, 4, finput);\r
+       nextSize = *(U32*)in_buff;\r
 \r
        // Main Loop\r
        while (1) \r
        {       \r
-               int outSize;\r
                // Read Block\r
-           U32 inSize = (U32) fread(in_buff, 1, 4, finput);\r
-               if( inSize<=0 ) break;\r
-               inSize = *(U32*)in_buff;\r
-           uselessRet = fread( in_buff, 1, inSize, finput);\r
+           uselessRet = fread(in_buff, 1, nextSize, finput);\r
+\r
+               // Check Next Block\r
+               uselessRet = (U32) fread(&nextSize, 1, 4, finput);\r
+               if( uselessRet==0 ) break;\r
 \r
                // Decode Block\r
-               outSize = LZ4_decode(in_buff, out_buff, inSize);\r
-               filesize += outSize;\r
+               sinkint = LZ4_uncompress(in_buff, out_buff, CHUNKSIZE);\r
+               filesize += CHUNKSIZE;\r
 \r
                // Write Block\r
-               fwrite(out_buff, 1, outSize, foutput);\r
+               fwrite(out_buff, 1, CHUNKSIZE, foutput);\r
        }\r
 \r
+       // Last Block\r
+    uselessRet = fread(in_buff, 1, nextSize, finput);\r
+       sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, nextSize, CHUNKSIZE);\r
+       filesize += sinkint;\r
+       fwrite(out_buff, 1, sinkint, foutput);\r
+\r
        // Status\r
        printf("Successfully decoded %llu bytes \n", (unsigned long long)filesize);\r
 \r