From: Yann Collet Date: Thu, 16 Mar 2017 22:41:30 +0000 (-0700) Subject: LZ4_compress_HC_continue_destSize() works as intended up to level 10 X-Git-Tag: upstream/1.9.3~11^2~99 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=aae447fffd3c5afd95a94c56ea70c349edad1ebd;p=platform%2Fupstream%2Flz4.git LZ4_compress_HC_continue_destSize() works as intended up to level 10 It's incompatible with btopt though, so cLevel >= 11 feature much reduced performance (degraded mode) --- diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 5f7cfc1..f3c9946 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -543,6 +543,7 @@ static int LZ4HC_compress_generic ( { if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe to reconsider */ if (cLevel > 9) { + if (limit == limitedDestSize) cLevel = 10; switch (cLevel) { case 10: return LZ4HC_compress_hashChain(ctx, src, dst, srcSizePtr, dstCapacity, 1 << (16-1), limit); @@ -589,11 +590,10 @@ int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, in /* LZ4_compress_HC_destSize() : * currently, only compatible with Hash Chain implementation, - * limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/ + * hence limit compression level to LZ4HC_CLEVEL_OPT_MIN-1*/ int LZ4_compress_HC_destSize(void* LZ4HC_Data, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) { LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; - if (cLevel >= LZ4HC_CLEVEL_OPT_MIN) cLevel = LZ4HC_CLEVEL_OPT_MIN-1; /* this only works with hash chain */ LZ4HC_init(ctx, (const BYTE*) source); return LZ4HC_compress_generic(ctx, source, dest, sourceSizePtr, targetDestSize, cLevel, limitedDestSize); } @@ -654,12 +654,13 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl } static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, - const char* source, char* dest, - int inputSize, int maxOutputSize, limitedOutput_directive limit) + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; /* auto-init if forgotten */ - if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); + if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) src); /* Check overflow */ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { @@ -669,37 +670,35 @@ static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, } /* Check if blocks follow each other */ - if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); + if ((const BYTE*)src != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { + if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; } } - return LZ4HC_compress_generic (ctxPtr, source, dest, &inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); + return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); } int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) { if (dstCapacity < LZ4_compressBound(srcSize)) - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, srcSize, dstCapacity, limitedOutput); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); else - return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, srcSize, dstCapacity, noLimit); + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, noLimit); } -int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int* sourceSizePtr, int targetDestSize) +int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) { LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; - int const cLevel = (ctxPtr->compressionLevel < 1) ? 1 : ctxPtr->compressionLevel; /* guaranteed <= LZ4HC_CLEVEL_MAX */ - unsigned const maxNbAttempts = 1 << (cLevel - 1); - LZ4HC_init(ctxPtr, (const BYTE*) source); /* LZ4_compress_HC_continue_destSize: always auto-init - only compatible with hash chain - miss continuity check */ - return LZ4HC_compress_hashChain(ctxPtr, source, dest, sourceSizePtr, targetDestSize, maxNbAttempts, limitedDestSize); + if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) LZ4HC_init(ctxPtr, (const BYTE*)src); /* not compatible with btopt implementation */ + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, limitedDestSize); } diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 8535a9e..09a648c 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -258,6 +258,8 @@ LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data, * @return : the number of bytes written into 'dst' * or 0 if compression fails. * `sourceSizePtr` : value will be updated to indicate how much bytes were read from `source` + * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN + * beyond that level, compression performance will be much reduced due to internal incompatibilities */ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 28d2be2..a36d20b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -664,10 +664,10 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c LZ4_resetStreamHC (&LZ4dictHC, compressionLevel); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace); + DISPLAYLEVEL(3, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel); FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed"); FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow"); FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow"); - DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i \n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace); FUZ_DISPLAYTEST; decodedBuffer[consumedSize] = 0;