From: yann.collet.73@gmail.com Date: Fri, 30 Nov 2012 13:23:36 +0000 (+0000) Subject: LZ4 HC : extended detection window. Thanks to Adrien Grand. X-Git-Tag: upstream/1.9.3~297 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=ffb27d4eca6bd80c068e745388faa31a0982115c;p=platform%2Fupstream%2Flz4.git LZ4 HC : extended detection window. Thanks to Adrien Grand. Fuzzer : more tests cases lz4demo : detect write errors. Thanks to Dima Tisnek bench.c : compatibility with Solaris 64. Thanks to Thorbjørn Willoch LZ4_compressBound() : now both in inline function and macro format. Thanks to Jacob Gorm Hansen git-svn-id: https://lz4.googlecode.com/svn/trunk@84 650e7d94-2a16-8b24-b05c-7c0b3f6821cd --- diff --git a/bench.c b/bench.c index b0a0ecb..015c36e 100644 --- a/bench.c +++ b/bench.c @@ -33,7 +33,7 @@ #if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions # define _LARGEFILE_SOURCE # define FILE_OFFSET_BITS=64 -#else +#elif ! defined(__LP64__) // No point defining Large file for 64 bit # define _LARGEFILE64_SOURCE #endif diff --git a/fuzzer.c b/fuzzer.c index 9fea8fd..9ba79d4 100644 --- a/fuzzer.c +++ b/fuzzer.c @@ -33,7 +33,7 @@ // Includes //************************************** #include -#include // fgets +#include // fgets, sscanf #include // timeb #include "lz4.h" @@ -97,11 +97,12 @@ int main() { unsigned long long bytes = 0; unsigned long long cbytes = 0; unsigned char buf[LEN]; -# define FUZ_max LZ4_compressBound(LEN) + unsigned char testOut[LEN+1]; +# define FUZ_max LZ4_COMPRESSBOUND(LEN) # define FUZ_avail ROUND_PAGE(FUZ_max) const int off_full = FUZ_avail - FUZ_max; unsigned char cbuf[FUZ_avail + PAGE_SIZE]; - unsigned int seed, cur_seq, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); + unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); int i, j, k, ret, len; char userInput[30] = {0}; @@ -120,6 +121,7 @@ int main() { for (i = 0; i < NB_ATTEMPTS; i++) { printf("\r%7i /%7i\r", i, NB_ATTEMPTS); + FUZ_rand(&seed); for (j = 0; j < NUM_SEQ; j++) { seeds[j] = FUZ_rand(&seed) << 8; @@ -136,23 +138,64 @@ int main() { } buf[j] = FUZ_rand(&cur_seq) >> 16; } + + // Test compression ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } len = ret; - // Test compression with output size being exactly what's necessary + // Test decoding with output size being exactly what's necessary => must work + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with one byte missing => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with one byte too much => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with enough output size => must work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with output size being exactly what's necessary => should work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with output size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with input size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with input size being one byte too large => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test compression with output size being exactly what's necessary (should work) ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); - if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); return 1; } - if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); return 1; } + if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; } + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } // Test compression with just one missing byte into output buffer => must fail ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); - if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); return 1; } - if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); return 1; } + if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; } + if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; } bytes += LEN; cbytes += len; } + printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + getchar(); return 0; + +_output_error: + getchar(); + return 1; } diff --git a/lz4.c b/lz4.c index 75dc64a..a651748 100644 --- a/lz4.c +++ b/lz4.c @@ -91,7 +91,7 @@ //************************************** // Compiler Options //************************************** -#if __STDC_VERSION__ >= 199901L // C99 +#if __STDC_VERSION__ >= 199901L // C99 /* "restrict" is a known keyword */ #else # define restrict // Disable restrict @@ -100,7 +100,6 @@ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #ifdef _MSC_VER // Visual Studio -# define inline __forceinline // Visual is not C99, but supports some kind of inline # include // For Visual 2005 # if LZ4_ARCH64 // 64-bit # pragma intrinsic(_BitScanForward64) // For Visual 2005 diff --git a/lz4.h b/lz4.h index dee3984..3680121 100644 --- a/lz4.h +++ b/lz4.h @@ -38,6 +38,14 @@ extern "C" { #endif +//************************************** +// Compiler Options +//************************************** +#ifdef _MSC_VER // Visual Studio +# define inline __inline // Visual is not C99, but supports some kind of inline +#endif + + //**************************** // Simple Functions //**************************** @@ -50,7 +58,7 @@ LZ4_compress() : Compresses 'isize' bytes from 'source' into 'dest'. Destination buffer must be already allocated, and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by macro LZ4_compressBound() + Worst case size evaluation is provided by function LZ4_compressBound() isize : is the input size. Max supported value is ~1.9GB return : the number of bytes written in buffer dest @@ -70,12 +78,15 @@ LZ4_uncompress() : // Advanced Functions //**************************** -#define LZ4_compressBound(isize) (isize + (isize/255) + 16) +static inline int LZ4_compressBound(int isize) { return ((isize) + ((isize)/255) + 16); } +#define LZ4_COMPRESSBOUND( isize) ((isize) + ((isize)/255) + 16) /* LZ4_compressBound() : Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible) primarily useful for memory allocation of output buffer. + inline function is recommended for the general case, + but macro is also provided when results need to be evaluated at compile time (such as table size allocation). isize : is the input size. Max supported value is ~1.9GB return : maximum output size in a "worst case" scenario diff --git a/lz4demo.c b/lz4demo.c index 6367401..067ac2e 100644 --- a/lz4demo.c +++ b/lz4demo.c @@ -53,7 +53,7 @@ //************************************** -// Compiler functions +// Compiler-specific functions //************************************** #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) @@ -91,9 +91,9 @@ static inline unsigned int swap32(unsigned int x) { // Architecture Macros //************************************** static const int one = 1; -#define CPU_LITTLE_ENDIAN (*(char*)(&one)) -#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) -#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); } +#define CPU_LITTLE_ENDIAN (*(char*)(&one)) +#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) +#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); } //************************************** @@ -177,6 +177,7 @@ int compress_file(char* input_filename, char* output_filename, int compressionle int r; int displayLevel = (compressionlevel>0); clock_t start, end; + size_t sizeCheck; // Init @@ -199,7 +200,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle u32var = ARCHIVE_MAGICNUMBER; LITTLE_ENDIAN32(u32var); *(unsigned int*)out_buff = u32var; - fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); + sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; } // Main Loop while (1) @@ -220,7 +222,8 @@ int compress_file(char* input_filename, char* output_filename, int compressionle LITTLE_ENDIAN32(outSize); * (unsigned int*) out_buff = outSize; LITTLE_ENDIAN32(outSize); - fwrite(out_buff, 1, outSize+4, foutput); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; } } // Status @@ -254,6 +257,7 @@ int decode_file(char* input_filename, char* output_filename) FILE* foutput; clock_t start, end; int r; + size_t sizeCheck; // Init @@ -291,7 +295,8 @@ int decode_file(char* input_filename, char* output_filename) filesize += sinkint; // Write Block - fwrite(out_buff, 1, sinkint, foutput); + sizeCheck = fwrite(out_buff, 1, sinkint, foutput); + if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; } } // Status diff --git a/lz4hc.c b/lz4hc.c index 2ab507e..7cd8bb4 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -337,7 +337,7 @@ inline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const // HC4 match finder LZ4HC_Insert(hc4, ip); ref = HASH_POINTER(ip); - while ((ref > (ip-MAX_DISTANCE)) && (nbAttempts)) + while ((ref >= (ip-MAX_DISTANCE)) && (nbAttempts)) { nbAttempts--; if (*(ref+ml) == *(ip+ml)) @@ -380,7 +380,7 @@ inline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const LZ4HC_Insert(hc4, ip); ref = HASH_POINTER(ip); - while ((ref > ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts)) + while ((ref >= ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts)) { nbAttempts--; if (*(startLimit + longest) == *(ref - delta + longest))