From 3e65c1e0c620df19e8779b8c56dda769a320650f Mon Sep 17 00:00:00 2001 From: "yann.collet.73@gmail.com" Date: Mon, 9 Sep 2013 09:06:21 +0000 Subject: [PATCH] New command line utility, lz4 (notice the missing final 'c'), with gzip-style arguments (issue 83) lz4c still there, supports additional gzip arguments, but also keep compatibility with legacy commands lz4 (& lz4c) display version number Fix : Sun Studio : compatible #pragma directive (issue 81) Fix : compatible with Objective-C (iOS) (issue 79) Fix : minor warnings using Visual Studio x64 (issue 80) Changed : source file lz4c.c renamed lz4cli.c git-svn-id: https://lz4.googlecode.com/svn/trunk@103 650e7d94-2a16-8b24-b05c-7c0b3f6821cd --- Makefile | 49 +++++-- bench.c | 6 +- cmake/CMakeLists.txt | 2 +- cmake/pack/CMakeLists.txt | 2 +- fullbench.c | 4 +- lz4.c | 90 ++++++------- lz4.h | 27 ++-- lz4c.c => lz4cli.c | 334 +++++++++++++++++++++++++++++----------------- 8 files changed, 317 insertions(+), 197 deletions(-) rename lz4c.c => lz4cli.c (72%) diff --git a/Makefile b/Makefile index 995b387..e81ba77 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,38 @@ +# ################################################################ +# LZ4 Makefile +# Copyright (C) Yann Collet 2011-2013 +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - LZ4 source repository : http://code.google.com/p/lz4/ +# ################################################################ +# lz4 : Command Line Utility, supporting gzip-like arguments +# lz4c : CLU, supporting also legacy lz4demo arguments +# lz4c32: Same as lz4c, but forced to compile in 32-bits mode +# fuzzer : Test tool, to check lz4 integrity on target platform +# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# fullbench : Precisely measure speed for each LZ4 function variant +# fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# ################################################################ + CC=gcc CFLAGS=-I. -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration +# Define *.exe as extension for Windows systems OS := $(shell uname) ifeq ($(OS),Linux) EXT = @@ -10,20 +42,21 @@ endif default: lz4c -all: lz4c lz4cs lz4c32 fuzzer fuzzer32 fullbench fullbench32 +all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32 -lz4c: lz4.c lz4hc.c bench.c xxhash.c lz4c.c - $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) +lz4: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c + $(CC) -O3 $(CFLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) -lz4cs: lz4.c lz4hc.c bench.c xxhash.c lz4c.c - $(CC) -Os $(CFLAGS) $^ -o $@$(EXT) +lz4c: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c + $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) -lz4c32: lz4.c lz4hc.c bench.c xxhash.c lz4c.c +lz4c32: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) fuzzer : lz4.c lz4hc.c fuzzer.c + @echo fuzzer is a test tool to check lz4 integrity on target platform $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) - + fuzzer32 : lz4.c lz4hc.c fuzzer.c $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) @@ -34,4 +67,4 @@ fullbench32 : lz4.c lz4hc.c xxhash.c fullbench.c $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) clean: - rm -f core *.o lz4c$(EXT) lz4cs$(EXT) lz4c32$(EXT) fuzzer$(EXT) fullbench$(EXT) fullbench32$(EXT) + rm -f core *.o lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) diff --git a/bench.c b/bench.c index 016a46e..7191c2a 100644 --- a/bench.c +++ b/bench.c @@ -143,11 +143,7 @@ static int chunkSize = DEFAULT_CHUNKSIZE; static int nbIterations = NBLOOPS; static int BMK_pause = 0; -void BMK_SetBlocksize(int bsize) -{ - chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10); -} +void BMK_SetBlocksize(int bsize) { chunkSize = bsize; } void BMK_SetNbIterations(int nbLoops) { diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 1afe650..496c076 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -32,7 +32,7 @@ endif() set(SRC_DIR ../) set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ) -set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4c.c) +set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c) if(NOT BUILD_SHARED_LIBS) set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB}) diff --git a/cmake/pack/CMakeLists.txt b/cmake/pack/CMakeLists.txt index 8e59824..a9b0557 100644 --- a/cmake/pack/CMakeLists.txt +++ b/cmake/pack/CMakeLists.txt @@ -42,7 +42,7 @@ INCLUDE_DIRECTORIES (${SRC_DIR}) set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ${SRC_DIR}lz4_format_description.txt) -set(LZ4_SRCS ${LZ4_SRCS_LIB} ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4c.c ) +set(LZ4_SRCS ${LZ4_SRCS_LIB} ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c ) set(FUZZER_SRCS ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ${SRC_DIR}fuzzer.c) # EXECUTABLES FOR 32 Bit and 64 versions diff --git a/fullbench.c b/fullbench.c index abb4f92..cbb7483 100644 --- a/fullbench.c +++ b/fullbench.c @@ -103,11 +103,11 @@ //**************************** // Constants //**************************** -#define COMPRESSOR_NAME "Full LZ4 speed analyzer" +#define COMPRESSOR_NAME "LZ4 speed analyzer" #define COMPRESSOR_VERSION "" #define COMPILED __DATE__ #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, (int)(sizeof(void*)*8), AUTHOR, COMPILED #define NBLOOPS 6 #define TIMELOOP 2500 diff --git a/lz4.c b/lz4.c index 73783ba..9852bbf 100644 --- a/lz4.c +++ b/lz4.c @@ -177,8 +177,6 @@ typedef unsigned long long U64; #endif -typedef const BYTE* Ptr; - #if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) # define _PACKED __attribute__ ((packed)) #else @@ -186,7 +184,7 @@ typedef const BYTE* Ptr; #endif #if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ +# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) # pragma pack(1) # else # pragma pack(push, 1) @@ -199,7 +197,11 @@ typedef struct { U64 v; } _PACKED U64_S; typedef struct {size_t v;} _PACKED size_t_S; #if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) +# if defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(0) +# else +# pragma pack(pop) +# endif #endif #define A16(x) (((U16_S *)(x))->v) @@ -244,9 +246,9 @@ const int LZ4_minLength = (MFLIMIT+1); typedef struct { U32 hashTable[HASHNBCELLS4]; - Ptr bufferStart; - Ptr base; - Ptr nextBlock; + const BYTE* bufferStart; + const BYTE* base; + const BYTE* nextBlock; } LZ4_Data_Structure; typedef enum { notLimited = 0, limited = 1 } limitedOutput_directive; @@ -268,9 +270,9 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; #if LZ4_ARCH64 // 64-bit # define HTYPE U32 -# define INITBASE(base) Ptr const base = ip +# define INITBASE(base) const BYTE* const base = ip #else // 32-bit -# define HTYPE Ptr +# define HTYPE const BYTE* # define INITBASE(base) const int base = 0 #endif @@ -369,32 +371,32 @@ FORCE_INLINE int LZ4_hashSequence(U32 sequence, tableType_t tableType) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -FORCE_INLINE int LZ4_hashPosition(Ptr p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); } +FORCE_INLINE int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); } -FORCE_INLINE void LZ4_putPositionOnHash(Ptr p, U32 h, void* tableBase, tableType_t tableType, Ptr srcBase) +FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { switch (tableType) { - case byPtr: { Ptr* hashTable = (Ptr*) tableBase; hashTable[h] = p; break; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = p-srcBase; break; } + case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; } } } -FORCE_INLINE void LZ4_putPosition(Ptr p, void* tableBase, tableType_t tableType, Ptr srcBase) +FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -FORCE_INLINE Ptr LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, Ptr srcBase) +FORCE_INLINE const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { - if (tableType == byPtr) { Ptr* hashTable = (Ptr*) tableBase; return hashTable[h]; } + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } // default, to ensure a return } -FORCE_INLINE Ptr LZ4_getPosition(Ptr p, void* tableBase, tableType_t tableType, Ptr srcBase) +FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); @@ -412,13 +414,13 @@ FORCE_INLINE int LZ4_compress_generic( tableType_t tableType, prefix64k_directive prefix) { - Ptr ip = (Ptr) source; - Ptr const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (Ptr) source; - Ptr const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (Ptr)source); - Ptr anchor = (Ptr) source; - Ptr const iend = ip + inputSize; - Ptr const mflimit = iend - MFLIMIT; - Ptr const matchlimit = iend - LASTLITERALS; + const BYTE* ip = (const BYTE*) source; + const BYTE* const base = (prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->base : (const BYTE*) source; + const BYTE* const lowLimit = ((prefix==withPrefix) ? ((LZ4_Data_Structure*)ctx)->bufferStart : (const BYTE*)source); + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; @@ -441,8 +443,8 @@ FORCE_INLINE int LZ4_compress_generic( for ( ; ; ) { int findMatchAttempts = (1U << skipStrength) + 3; - Ptr forwardIp = ip; - Ptr ref; + const BYTE* forwardIp = ip; + const BYTE* ref; BYTE* token; // Find a match @@ -599,7 +601,7 @@ int LZ4_compress_limitedOutput_continue (void* LZ4_Data, const char* source, cha // Stream functions //**************************** -FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, Ptr base) +FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, const BYTE* base) { MEM_INIT(lz4ds->hashTable, 0, sizeof(lz4ds->hashTable)); lz4ds->bufferStart = base; @@ -611,7 +613,7 @@ FORCE_INLINE void LZ4_init(LZ4_Data_Structure* lz4ds, Ptr base) void* LZ4_create (const char* inputBuffer) { void* lz4ds = ALLOCATOR(1, sizeof(LZ4_Data_Structure)); - LZ4_init ((LZ4_Data_Structure*)lz4ds, (Ptr)inputBuffer); + LZ4_init ((LZ4_Data_Structure*)lz4ds, (const BYTE*)inputBuffer); return lz4ds; } @@ -636,7 +638,7 @@ char* LZ4_slideInputBuffer (void* LZ4_Data) for (nH=0; nH < HASHNBCELLS4; nH++) { if (lz4ds->hashTable[nH] < (U32)newBaseDelta) lz4ds->hashTable[nH] = 0; - else lz4ds->hashTable[nH] -= newBaseDelta; + else lz4ds->hashTable[nH] -= (U32)newBaseDelta; } lz4ds->base += newBaseDelta; } @@ -660,7 +662,7 @@ FORCE_INLINE int LZ4_decompress_generic( const char* source, char* dest, int inputSize, // - int outputSize, // OutputSize must be != 0; if endOnInput==endOnInputSize, this value is the max size of Output Buffer. + int outputSize, // If endOnInput==endOnInputSize, this value is the max size of Output Buffer. int endOnInput, // endOnOutputSize, endOnInputSize int prefix64k, // noPrefix, withPrefix @@ -669,19 +671,17 @@ FORCE_INLINE int LZ4_decompress_generic( ) { // Local Variables - Ptr restrict ip = (Ptr) source; - Ptr ref; - Ptr const iend = ip + inputSize; + const BYTE* restrict ip = (const BYTE*) source; + const BYTE* ref; + const BYTE* const iend = ip + inputSize; BYTE* op = (BYTE*) dest; BYTE* const oend = op + outputSize; BYTE* cpy; BYTE* oexit = op + targetOutputSize; - size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; -#if LZ4_ARCH64 - size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; -#endif + const size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; // static reduces speed for LZ4_decompress_safe() on GCC64 + static const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; // Special cases @@ -749,11 +749,7 @@ FORCE_INLINE int LZ4_decompress_generic( // copy repeated sequence if unlikely((op-ref)<(int)STEPSIZE) { -#if LZ4_ARCH64 - size_t dec64 = dec64table[op-ref]; -#else - const size_t dec64 = 0; -#endif + const size_t dec64 = dec64table[(sizeof(void*)==4) ? 0 : op-ref]; op[0] = ref[0]; op[1] = ref[1]; op[2] = ref[2]; @@ -803,6 +799,11 @@ int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, i return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize); } +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0); +} + int LZ4_decompress_fast(const char* source, char* dest, int outputSize) { #ifdef _MSC_VER // This version is faster with Visual @@ -812,8 +813,3 @@ int LZ4_decompress_fast(const char* source, char* dest, int outputSize) #endif } -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize) -{ - return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0); -} - diff --git a/lz4.h b/lz4.h index 8f87908..a897bc3 100644 --- a/lz4.h +++ b/lz4.h @@ -83,7 +83,7 @@ LZ4_compressBound() : Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible) primarily useful for memory allocation of output buffer. inline function is recommended for the general case, - macro is also provided when result needs to be evaluated at compilation (such as table size allocation). + macro is also provided when result needs to be evaluated at compilation (such as stack memory allocation). isize : is the input size. Max supported value is ~1.9GB return : maximum output size in a "worst case" scenario @@ -130,23 +130,11 @@ LZ4_decompress_safe_partial() : return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. Always control how many bytes were decoded. - If the source stream is malformed, the function will stop decoding and return a negative result. + If the source stream is detected malformed, the function will stop decoding and return a negative result. This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets */ -int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize); -int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize); - -/* -*_withPrefix64k() : - These decoding functions work the same as their "normal name" versions, - but will potentially use up to 64KB of data in front of 'char* dest'. - These functions are used for decoding inter-dependant blocks. -*/ - - - //**************************** // Stream Functions //**************************** @@ -187,6 +175,17 @@ When compression is completed, a call to LZ4_free() will release the memory used */ +int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize); + +/* +*_withPrefix64k() : + These decoding functions work the same as their "normal name" versions, + but can use up to 64KB of data in front of 'char* dest'. + These functions are necessary to decode inter-dependant blocks. +*/ + + //**************************** // Obsolete Functions //**************************** diff --git a/lz4c.c b/lz4cli.c similarity index 72% rename from lz4c.c rename to lz4cli.c index 65a7ea2..faa1764 100644 --- a/lz4c.c +++ b/lz4cli.c @@ -1,5 +1,5 @@ /* - LZ4c - LZ4 Compression CLI program + LZ4cli.c - LZ4 Command Line Interface Copyright (C) Yann Collet 2011-2013 GPL v2 License @@ -23,13 +23,22 @@ */ /* Note : this is stand-alone program. - It is not part of LZ4 compression library, it is a user program of LZ4 library. + It is not part of LZ4 compression library, it is a user program of the LZ4 library. The license of LZ4 library is BSD. The license of xxHash library is BSD. The license of this compression CLI program is GPLv2. */ //************************************** +// Tuning parameters +//************************************** +// DISABLE_LZ4C_LEGACY_OPTIONS : +// Control the availability of -c0, -c1 and -hc legacy arguments +// Default : Legacy options are enabled +// #define DISABLE_LZ4C_LEGACY_OPTIONS + + +//************************************** // Compiler Options //************************************** // Disable some Visual warning messages @@ -46,20 +55,31 @@ //**************************** // Includes //**************************** -#include // fprintf, fopen, fread, _fileno(?) +#include // fprintf, fopen, fread, _fileno, stdin, stdout #include // malloc #include // strcmp #include // clock -#ifdef _WIN32 -#include // _setmode -#include // _O_BINARY -#endif #include "lz4.h" #include "lz4hc.h" +#include "lz4stream.h" #include "bench.h" #include "xxhash.h" +//**************************** +// OS-specific Includes +//**************************** +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include // _O_BINARY +# include // _setmode, _isatty +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#endif + + //************************************** // Compiler-specific functions //************************************** @@ -84,14 +104,11 @@ // Constants //**************************** #define COMPRESSOR_NAME "LZ4 Compression CLI" -#define COMPRESSOR_VERSION "" +#define COMPRESSOR_VERSION "1.0.3" #define COMPILED __DATE__ #define AUTHOR "Yann Collet" #define EXTENSION ".lz4" -#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED - -#define UNLZ4 "unlz4" -#define LZ4CAT "lz4cat" +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, (int)(sizeof(void*)*8), AUTHOR, COMPILED #define KB *(1U<<10) #define MB *(1U<<20) @@ -130,7 +147,8 @@ static const int one = 1; //************************************** // Macros //************************************** -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } //************************************** @@ -150,8 +168,7 @@ char nulmark[] = "/dev/null"; // Local Parameters //************************************** static char* programName; -static int silence = 0; -static int verbose = 0; +static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information static int overwrite = 0; static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; static int blockChecksum = 0; @@ -167,9 +184,9 @@ static int blockIndependence = 1; #define EXM_THROW(error, ...) \ { \ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAY("Error %i : ", error); \ - DISPLAY(__VA_ARGS__); \ - DISPLAY("\n"); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ exit(error); \ } @@ -183,52 +200,101 @@ int usage() DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] [input] [output]\n", programName); DISPLAY( "\n"); - DISPLAY( "input : a filename, or \n"); - DISPLAY( " '%s' or '-' for pipe mode (default if empty)\n", stdinmark); + DISPLAY( "input : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); DISPLAY( "Arguments :\n"); - DISPLAY( " -c0/-c : Fast compression (default) \n"); - DISPLAY( " -c1/-hc: High compression \n"); + DISPLAY( " -1 : Fast compression (default) \n"); + DISPLAY( " -9 : High compression \n"); DISPLAY( " -d : decompression \n"); - DISPLAY( " -y : overwrite without prompting \n"); - DISPLAY( " -h/-H : Help (this text + advanced options)\n"); + DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); return 0; } int usage_advanced() { + DISPLAY(WELCOME_MESSAGE); usage(); DISPLAY( "\n"); - DISPLAY( "Advanced options :\n"); - DISPLAY( " -v : be verbose \n"); - DISPLAY( " -t : test compressed file \n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); + DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -t : test compressed file integrity\n"); + DISPLAY( " -z : force compression\n"); DISPLAY( " -B# : Block size [4-7](default : 7)\n"); DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); DISPLAY( " -BX : enable block checksum (default:disabled)\n"); DISPLAY( " -Sx : disable stream checksum (default:enabled)\n"); - DISPLAY( " -b# : benchmark files, using # [0-1] compression level\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b : benchmark file(s)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + DISPLAY( "Legacy arguments :\n"); + DISPLAY( " -c0 : fast compression\n"); + DISPLAY( " -c1 : high compression\n"); + DISPLAY( " -hc : high compression\n"); + DISPLAY( " -y : overwrite output without prompting \n"); + DISPLAY( " -s : suppress warnings \n"); +#endif // DISABLE_LZ4C_LEGACY_OPTIONS + return 0; +} + +int usage_longhelp() +{ + DISPLAY( "\n"); + DISPLAY( "Which values can get [output] ? \n"); + DISPLAY( "[output] : a filename\n"); + DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); + DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT); + DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n"); + DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); + DISPLAY( " - if stdout is console : \n"); + DISPLAY( " + if compression selected, output to filename%s \n", EXTENSION); + DISPLAY( " + if decompression selected, output to filename without '%s'\n", EXTENSION); + DISPLAY( " > if input filename has no '%s' extension : error\n", EXTENSION); DISPLAY( "\n"); - DISPLAY( "output : a filename, or \n"); - DISPLAY( " '%s', or '-' for pipe mode\n", stdoutmark); - DISPLAY( " or '%s'\n", NULL_OUTPUT); - DISPLAY( " default if empty : stdout if input is stdin\n"); - DISPLAY( " input.lz4 if compression selected\n"); - DISPLAY( " input without '.lz4' if decompression\n"); + DISPLAY( "Compression levels : \n"); + DISPLAY( "There are technically 2 accessible compression levels.\n"); + DISPLAY( "-0 ... -2 => Fast compression\n"); + DISPLAY( "-3 ... -9 => High compression\n"); DISPLAY( "\n"); - DISPLAY( "Examples :\n"); - DISPLAY( "1 : compress file 'filename', using default output name 'filename.lz4'\n"); + DISPLAY( "stdin, stdout and the console : \n"); + DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); + DISPLAY( "%s will refuse to read from console, or write to console \n", programName); + DISPLAY( "except if '-c' command is specified, to force output to console \n"); + DISPLAY( "\n"); + DISPLAY( "Simple example :\n"); + DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n"); DISPLAY( " %s filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments can be appended together, or provided independently. For example :\n"); DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); - DISPLAY( " %s -hcy filename \n", programName); + DISPLAY( " %s -f9 filename \n", programName); + DISPLAY( " is equivalent to :\n"); + DISPLAY( " %s -f -9 filename \n", programName); + DISPLAY( "\n"); + DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName); DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); DISPLAY( " generator | %s | consumer \n", programName); +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + DISPLAY( "\n"); + DISPLAY( "Warning :\n"); + DISPLAY( "Legacy arguments take precedence. Therefore : \n"); + DISPLAY( " %s -hc filename\n", programName); + DISPLAY( "means 'compress filename in high compression mode'\n"); + DISPLAY( "It is not equivalent to :\n"); + DISPLAY( " %s -h -c filename\n", programName); + DISPLAY( "which would display help text and exit\n"); +#endif // DISABLE_LZ4C_LEGACY_OPTIONS return 0; } int badusage() { - DISPLAY("Wrong parameters\n"); - usage(); + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (displayLevel >= 1) usage(); exit(1); } @@ -243,11 +309,9 @@ int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, if (!strcmp (input_filename, stdinmark)) { - if (verbose) DISPLAY( "Using stdin for input\n"); + DISPLAYLEVEL(4,"Using stdin for input\n"); *pfinput = stdin; -#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows - _setmode( _fileno( stdin ), _O_BINARY ); -#endif + SET_BINARY_MODE(stdin); } else { @@ -256,11 +320,9 @@ int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, if (!strcmp (output_filename, stdoutmark)) { - if (verbose) DISPLAY( "Using stdout for output\n"); + DISPLAYLEVEL(4,"Using stdout for output\n"); *pfoutput = stdout; -#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows - _setmode( _fileno( stdout ), _O_BINARY ); -#endif + SET_BINARY_MODE(stdout); } else { @@ -273,8 +335,9 @@ int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, if (!overwrite) { char ch; - DISPLAY( "Warning : %s already exists\n", output_filename); - DISPLAY( "Overwrite ? (Y/N) : "); + DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); + DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); + if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible ch = (char)getchar(); if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); } @@ -313,6 +376,7 @@ int legacy_compress_file(char* input_filename, char* output_filename, int compre } start = clock(); get_fileHandle(input_filename, output_filename, &finput, &foutput); + if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; // Allocate Memory in_buff = (char*)malloc(LEGACY_BLOCKSIZE); @@ -332,12 +396,12 @@ int legacy_compress_file(char* input_filename, char* output_filename, int compre int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); if( inSize<=0 ) break; filesize += inSize; - if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); + DISPLAYLEVEL(3, "Read : %i MB \r", (int)(filesize>>20)); // Compress Block outSize = compressionFunction(in_buff, out_buff+4, inSize); compressedfilesize += outSize+4; - if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYLEVEL(3, "Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); // Write Block * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); @@ -347,11 +411,11 @@ int legacy_compress_file(char* input_filename, char* output_filename, int compre // Status end = clock(); - if (!silence) DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); { double seconds = (double)(end - start)/CLOCKS_PER_SEC; - if (verbose) DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); } // Close & Free @@ -378,7 +442,6 @@ int compress_file_blockDependency(char* input_filename, char* output_filename, i char* out_buff; FILE* finput; FILE* foutput; - int displayLevel = ((compressionlevel>0) && (!silence)) || (verbose); clock_t start, end; unsigned int blockSize, inputBufferSize; size_t sizeCheck, header_size; @@ -387,6 +450,7 @@ int compress_file_blockDependency(char* input_filename, char* output_filename, i // Init start = clock(); + if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; switch (compressionlevel) { case 0 : @@ -440,14 +504,14 @@ int compress_file_blockDependency(char* input_filename, char* output_filename, i inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput); if( inSize==0 ) break; // No more input : end of compression filesize += inSize; - if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); + DISPLAYLEVEL(3, "Read : %i MB \r", (int)(filesize>>20)); if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); // Compress Block outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; if (blockChecksum) compressedfilesize+=4; - if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYLEVEL(3, "Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); // Write Block if (outSize > 0) @@ -498,11 +562,11 @@ int compress_file_blockDependency(char* input_filename, char* output_filename, i // Status end = clock(); - if (!silence) DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); { double seconds = (double)(end - start)/CLOCKS_PER_SEC; - if (verbose) DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); } // Close & Free @@ -527,7 +591,6 @@ int compress_file(char* input_filename, char* output_filename, int compressionle char* headerBuffer; FILE* finput; FILE* foutput; - int displayLevel = ((compressionlevel>0) && (!silence)) || (verbose); clock_t start, end; int blockSize; size_t sizeCheck, header_size, readSize; @@ -538,6 +601,7 @@ int compress_file(char* input_filename, char* output_filename, int compressionle // Init start = clock(); + if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; switch (compressionlevel) { case 0 : compressionFunction = LZ4_compress_limitedOutput; break; @@ -580,14 +644,14 @@ int compress_file(char* input_filename, char* output_filename, int compressionle unsigned int outSize; filesize += readSize; - if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); + DISPLAYLEVEL(3, "Read : %i MB \r", (int)(filesize>>20)); if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); // Compress Block outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1); if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; if (blockChecksum) compressedfilesize+=4; - if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYLEVEL(3, "Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); // Write Block if (outSize > 0) @@ -646,11 +710,11 @@ int compress_file(char* input_filename, char* output_filename, int compressionle // Final Status end = clock(); - if (!silence) DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); { double seconds = (double)(end - start)/CLOCKS_PER_SEC; - if (verbose) DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); } return 0; @@ -887,9 +951,10 @@ unsigned long long selectDecoder( FILE* finput, FILE* foutput) case LZ4S_MAGICNUMBER: return decodeLZ4S(finput, foutput); case LEGACY_MAGICNUMBER: - if (verbose) DISPLAY("Detected : Legacy format \n"); + DISPLAYLEVEL(4, "Detected : Legacy format \n"); return decodeLegacyStream(finput, foutput); case LZ4S_SKIPPABLE0: + DISPLAYLEVEL(4, "Skipping detected skippable area \n"); nbReadBytes = fread(&size, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format @@ -898,7 +963,7 @@ unsigned long long selectDecoder( FILE* finput, FILE* foutput) return selectDecoder(finput, foutput); default: if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream - DISPLAY("Stream followed by unrecognized data\n"); + DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); return 0; } } @@ -925,10 +990,10 @@ int decodeFile(char* input_filename, char* output_filename) // Final Status end = clock(); - if (!silence) DISPLAY( "Successfully decoded %llu bytes \n", filesize); + DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); { double seconds = (double)(end - start)/CLOCKS_PER_SEC; - if (verbose) DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); } // Close @@ -947,16 +1012,15 @@ int main(int argc, char** argv) decode=0, bench=0, filenamesStart=2, - legacy_format=0; + legacy_format=0, + forceStdout=0; char* input_filename=0; char* output_filename=0; char nullOutput[] = NULL_OUTPUT; char extension[] = EXTENSION; - // Select behavior + // Init programName = argv[0]; - if (strstr(programName, UNLZ4)) { decode=1; silence=1; } - else if (strstr(programName, LZ4CAT)) { decode=1; silence=1; output_filename=stdoutmark; } for(i=1; i='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } break; - case 'h': if (argument[1]=='c') { cLevel=1; argument++; } else { usage_advanced(); return 0; } break; + case 'z': break; + + // Compression level + case '0': + case '1': + case '2': cLevel=0; break; + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': cLevel=1; break; // Use Legacy format (hidden option) case 'l': legacy_format=1; break; @@ -992,40 +1081,56 @@ int main(int argc, char** argv) // Decoding case 'd': decode=1; break; + // Force stdout, even if stdout==console + case 'c': forceStdout=1; output_filename=stdoutmark; displayLevel=1; break; + // Test case 't': decode=1; output_filename=nulmark; break; + // Overwrite + case 'f': overwrite=1; break; + + // Verbose mode + case 'v': displayLevel=4; break; + + // Quiet mode + case 'q': displayLevel--; break; + + // keep source file (default anyway, so useless) (for xz/lzma compatibility) + case 'k': break; + // Modify Block Properties case 'B': while (argument[1]!=0) - switch(argument[1]) { - case '4': - case '5': - case '6': - case '7': - { - int B = argument[1] - '0'; - int S = 1 << (8 + 2*B); - BMK_SetBlocksize(S); - blockSizeId = B; - argument++; - break; - } - case 'D': blockIndependence = 0, argument++; break; - case 'X': blockChecksum = 1, argument ++; break; - default : goto _exit_blockProperties; + int exitBlockProperties=0; + switch(argument[1]) + { + case '4': + case '5': + case '6': + case '7': + { + int B = argument[1] - '0'; + int S = 1 << (8 + 2*B); + BMK_SetBlocksize(S); + blockSizeId = B; + argument++; + break; + } + case 'D': blockIndependence = 0, argument++; break; + case 'X': blockChecksum = 1, argument ++; break; + default : exitBlockProperties=1; + } + if (exitBlockProperties) break; } -_exit_blockProperties: break; // Modify Stream properties case 'S': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(); } - // Bench - case 'b': bench=1; - if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } - break; + // Benchmark + case 'b': bench=1; break; // Modify Nb Iterations (benchmark only) case 'i': @@ -1040,12 +1145,6 @@ _exit_blockProperties: // Pause at the end (benchmark only) (hidden option) case 'p': BMK_SetPause(); break; - // Overwrite - case 'y': overwrite=1; break; - - // Verbose mode - case 'v': verbose=1; break; - // Unrecognised command default : badusage(); } @@ -1065,18 +1164,19 @@ _exit_blockProperties: } } - if (verbose) DISPLAY( WELCOME_MESSAGE); + DISPLAYLEVEL(4, WELCOME_MESSAGE); + DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2))); // No input filename ==> use stdin if(!input_filename) { input_filename=stdinmark; } - // Check if benchmark was required + // Check if benchmark is selected if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); // No output filename ==> select one automatically (when possible) if (!output_filename) { - if (input_filename == stdinmark) { output_filename=stdoutmark; silence=1; } + if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; } else if (!decode) // compression { int i=0, l=0; @@ -1084,7 +1184,7 @@ _exit_blockProperties: output_filename = (char*)calloc(1,l+5); for (i=0;i4) while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0; - if (outl != inl-5) { DISPLAY("Cannot automatically decide an output filename\n"); badusage(); } + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot automatically decide an output filename\n"); badusage(); } + DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); } } - if ((decode ? input_filename==stdinmark : output_filename==stdoutmark) - && !overwrite -#ifdef _WIN32 - && _isatty (_fileno ((decode ? stdin : stdout)))) -#else - && isatty ( fileno ((decode ? stdin : stdout)))) -#endif - badusage(); - - if ((input_filename == stdinmark) && (output_filename == stdoutmark)) silence=1; + // No warning message in pure pipe mode (stdin + stdout) + if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel>1) && (displayLevel<4)) displayLevel=1; - if (verbose) silence=0; + // Check if input or output are defined as console; trigger an error in this case + if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); + if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); + // Decompress input if selected if (decode) return decodeFile(input_filename, output_filename); // compression is default action if (legacy_format) { - if (!silence) DISPLAY("! Generating compressed LZ4 using Legacy format (deprecated !) ! \n"); + DISPLAYLEVEL(2, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n"); return legacy_compress_file(input_filename, output_filename, cLevel); } else -- 2.7.4