--- /dev/null
+// LZ4 HC streaming API example : ring buffer
+// Based on previous work from Takayuki Matsuoka
+
+
+/**************************************
+ Compiler Options
+**************************************/
+#ifdef _MSC_VER /* Visual Studio */
+# define _CRT_SECURE_NO_WARNINGS // for MSVC
+# define snprintf sprintf_s
+#endif
+
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#ifdef __GNUC__
+# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
+#endif
+
+
+
+/**************************************
+ Includes
+**************************************/
+#include "lz4hc.h"
+#include "lz4.h"
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+ MESSAGE_MAX_BYTES = 1024,
+ RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES,
+ DICT_BYTES = 65536,
+};
+
+
+size_t write_int32(FILE* fp, int32_t i) {
+ return fwrite(&i, sizeof(i), 1, fp);
+}
+
+size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
+ return fwrite(array, 1, arrayBytes, fp);
+}
+
+size_t read_int32(FILE* fp, int32_t* i) {
+ return fread(i, sizeof(*i), 1, fp);
+}
+
+size_t read_bin(FILE* fp, void* array, int arrayBytes) {
+ return fread(array, 1, arrayBytes, fp);
+}
+
+
+void test_compress(FILE* outFp, FILE* inpFp)
+{
+ LZ4_streamHC_t lz4Stream_body = { 0 };
+ LZ4_streamHC_t* lz4Stream = &lz4Stream_body;
+
+ static char inpBuf[RING_BUFFER_BYTES];
+ int inpOffset = 0;
+
+ for(;;)
+ {
+ // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
+ char* const inpPtr = &inpBuf[inpOffset];
+ const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
+ const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
+ if (0 == inpBytes) break;
+
+ {
+ char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
+ const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
+ if(cmpBytes <= 0) break;
+ write_int32(outFp, cmpBytes);
+ write_bin(outFp, cmpBuf, cmpBytes);
+
+ inpOffset += inpBytes;
+
+ // Wraparound the ringbuffer offset
+ if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
+ inpOffset = 0;
+ }
+ }
+
+ write_int32(outFp, 0);
+}
+
+
+void test_decompress(FILE* outFp, FILE* inpFp)
+{
+ static char decBuf[RING_BUFFER_BYTES];
+ int decOffset = 0;
+ LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
+ LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
+
+ for(;;) {
+ int cmpBytes = 0;
+ char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
+
+ {
+ const size_t r0 = read_int32(inpFp, &cmpBytes);
+ size_t r1;
+ if(r0 != 1 || cmpBytes <= 0) break;
+
+ r1 = read_bin(inpFp, cmpBuf, cmpBytes);
+ if(r1 != (size_t) cmpBytes) break;
+ }
+
+ {
+ char* const decPtr = &decBuf[decOffset];
+ const int decBytes = LZ4_decompress_safe_continue(
+ lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
+ if(decBytes <= 0)
+ break;
+ decOffset += decBytes;
+ write_bin(outFp, decPtr, decBytes);
+
+ // Wraparound the ringbuffer offset
+ if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0;
+ }
+ }
+}
+
+
+int compare(FILE* f0, FILE* f1)
+{
+ int result = 0;
+
+ while(0 == result) {
+ char b0[65536];
+ char b1[65536];
+ const size_t r0 = fread(b0, 1, sizeof(b0), f0);
+ const size_t r1 = fread(b1, 1, sizeof(b1), f1);
+
+ result = (int) r0 - (int) r1;
+
+ if(0 == r0 || 0 == r1) {
+ break;
+ }
+ if(0 == result) {
+ result = memcmp(b0, b1, r0);
+ }
+ }
+
+ return result;
+}
+
+
+int main(int argc, char** argv)
+{
+ char inpFilename[256] = { 0 };
+ char lz4Filename[256] = { 0 };
+ char decFilename[256] = { 0 };
+
+ if(argc < 2) {
+ printf("Please specify input filename\n");
+ return 0;
+ }
+
+ snprintf(inpFilename, 256, "%s", argv[1]);
+ snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0);
+ snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0);
+
+ printf("inp = [%s]\n", inpFilename);
+ printf("lz4 = [%s]\n", lz4Filename);
+ printf("dec = [%s]\n", decFilename);
+
+ // compress
+ {
+ FILE* inpFp = fopen(inpFilename, "rb");
+ FILE* outFp = fopen(lz4Filename, "wb");
+
+ test_compress(outFp, inpFp);
+
+ fclose(outFp);
+ fclose(inpFp);
+ }
+
+ // decompress
+ {
+ FILE* inpFp = fopen(lz4Filename, "rb");
+ FILE* outFp = fopen(decFilename, "wb");
+
+ test_decompress(outFp, inpFp);
+
+ fclose(outFp);
+ fclose(inpFp);
+ }
+
+ // verify
+ {
+ FILE* inpFp = fopen(inpFilename, "rb");
+ FILE* decFp = fopen(decFilename, "rb");
+
+ const int cmp = compare(inpFp, decFp);
+ if(0 == cmp) {
+ printf("Verify : OK\n");
+ } else {
+ printf("Verify : NG\n");
+ }
+
+ fclose(decFp);
+ fclose(inpFp);
+ }
+
+ return 0;
+}
default: all
-all: printVersion doubleBuffer ringBuffer lineCompress
+all: printVersion doubleBuffer ringBuffer ringBufferHC lineCompress
printVersion: $(LZ4DIR)/lz4.c printVersion.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
ringBuffer : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
./printVersion$(EXT)
./doubleBuffer$(EXT) $(TESTFILE)
./ringBuffer$(EXT) $(TESTFILE)
+ ./ringBufferHC$(EXT) $(TESTFILE)
./lineCompress$(EXT) $(TESTFILE)
clean:
@rm -f core *.o *.dec *-0 *-8192 *.lz4s \
- printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) lineCompress$(EXT)
+ printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT)
@echo Cleaning completed
}
+static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
+{
+ if (ctxPtr->end >= ctxPtr->base + 4)
+ LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content
+ // Note : need to handle risk of index overflow
+ // Use only one memory segment for dict, so any previous External Dict is lost at this stage
+ ctxPtr->lowLimit = ctxPtr->dictLimit;
+ ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
+ ctxPtr->dictBase = ctxPtr->base;
+ ctxPtr->base = newBlock - ctxPtr->dictLimit;
+ ctxPtr->end = newBlock;
+ ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block
+}
+
+
static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit)
{
const BYTE* const p1Start = p1;
}
-static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
-{
- if (ctxPtr->end >= ctxPtr->base + 4)
- LZ4HC_Insert (ctxPtr, ctxPtr->end-3); // finish referencing dictionary content
- // Note : need to handle risk of index overflow
- // Use only one memory segment for dict, so any previous External Dict is lost at this stage
- ctxPtr->lowLimit = ctxPtr->dictLimit;
- ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
- ctxPtr->dictBase = ctxPtr->base;
- ctxPtr->base = newBlock - ctxPtr->dictLimit;
- ctxPtr->end = newBlock;
- ctxPtr->nextToUpdate = ctxPtr->dictLimit; // reference table must skip to from beginning of block
-}
-
-
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, // Index table will be updated
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL;
if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL;
maxNbAttempts = 1 << compressionLevel;
- /* check if blocks follow each other */
- if (ip != ctx->end) LZ4HC_setExternalDict(ctx, ip);
ctx->end += inputSize;
ip++;
/* compression */
+static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr,
+ const char* source, char* dest,
+ int inputSize, int maxOutputSize, limitedOutput_directive limit)
+{
+ /* auto-init if forgotten */
+ if (dsPtr->base == NULL)
+ LZ4HC_init (dsPtr, (const BYTE*) source);
+
+ /* check if blocks follow each other */
+ if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source);
+
+ /* Check overlapping input/dictionary space */
+ {
+ const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+ const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit;
+ const BYTE* dictEnd = dsPtr->dictBase + dsPtr->dictLimit;
+ if ((sourceEnd > dictBegin) && (sourceEnd < dictEnd))
+ {
+ dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase);
+ if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit;
+ }
+ }
+
+ return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit);
+}
+
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
{
- if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
- return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit);
+ //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL)
+ // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
+ //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit);
+ return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit);
}
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
- if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
- return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput);
+ //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL)
+ // LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
+ //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput);
+ return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
}
projects / platform / upstream / lz4.git / commitdiff