Google Code Issue 155: lz4 cli should support sparse file

https://code.google.com/p/lz4/issues/detail?id=155

This is experimental implementation. Just a proof of concept.
It works Linux and Windows.

# Build

To build experimental version, define 'LZ4IO_ENABLE_SPARSE_FILE' like the following command :

    make lz4programs 'CFLAGS=-O3 -DLZ4IO_ENABLE_SPARSE_FILE=1'
    ./programs/lz4 -h

You will see "EXPERIMENTAL_SPARSE_FILE" as lz4 revision :

    "*** LZ4 command line interface 64-bits EXPERIMENTAL_SPARSE_FILE, by Yann Collet (...) ***"

# Experiment

This experimental version adds option "-x" for sparse file for decompression.
You can use this option like this :

    ./programs/lz4 -9 -f my-file
    ./programs/lz4 -d -f -x my-file.lz4 my-file.lz4.out
    cmp my-file my-file.lz4.out

diff --git a/programs/lz4cli.c b/programs/lz4cli.c
index 0da5dce..5b54143 100644 (file)
--- a/programs/lz4cli.c
+++ b/programs/lz4cli.c
@@ -83,6 +83,11 @@
 /*****************************
 *  Constants
 ******************************/
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+#  undef LZ4_VERSION
+#  define LZ4_VERSION "EXPERIMENTAL_SPARSE_FILE"
+#endif
+
 #define COMPRESSOR_NAME "LZ4 command line interface"
 #ifndef LZ4_VERSION
 #  define LZ4_VERSION "r126"
@@ -186,6 +191,10 @@ static int usage_advanced(void)
     DISPLAY( " -y     : overwrite output without prompting \n");
     DISPLAY( " -s     : suppress warnings \n");
 #endif /* ENABLE_LZ4C_LEGACY_OPTIONS */
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+    DISPLAY( "Experimental : Sparse file\n");
+    DISPLAY( " -x     : enable sparse file\n");
+#endif /* LZ4IO_ENABLE_SPARSE_FILE */
     EXTENDED_HELP;
     return 0;
 }
@@ -276,6 +285,9 @@ int main(int argc, char** argv)
     /* Init */
     programName = argv[0];
     LZ4IO_setOverwrite(0);
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+    LZ4IO_setSparseFile(0);
+#endif /* LZ4IO_ENABLE_SPARSE_FILE */
     blockSize = LZ4IO_setBlockSizeID(LZ4_BLOCKSIZEID_DEFAULT);
 
     /* lz4cat behavior */
@@ -403,6 +415,11 @@ int main(int argc, char** argv)
                     /* Pause at the end (hidden option) */
                 case 'p': main_pause=1; BMK_SetPause(); break;
 
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+                    /* Experimental : Enable sparse file */
+                case 'x': LZ4IO_setSparseFile(1); break;
+#endif /* LZ4IO_ENABLE_SPARSE_FILE */
+
                     /* Specific commands for customized versions */
                 EXTENDED_ARGUMENTS;
 

diff --git a/programs/lz4io.c b/programs/lz4io.c
index 023824e..4e583f6 100644 (file)
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -76,10 +76,18 @@
 #  endif
 #  define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
 #  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#  if defined(LZ4IO_ENABLE_SPARSE_FILE)
+#    include <windows.h>
+#    define SET_SPARSE_FILE_MODE(file) do { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } while(0)
+#    define fseek _fseeki64
+#  endif /* LZ4IO_ENABLE_SPARSE_FILE */
 #else
 #  include <unistd.h>  /* isatty */
 #  define SET_BINARY_MODE(file)
 #  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#  if defined(LZ4IO_ENABLE_SPARSE_FILE)
+#    define SET_SPARSE_FILE_MODE(file)
+#  endif /* LZ4IO_ENABLE_SPARSE_FILE */
 #endif
 
 
@@ -133,6 +141,9 @@ static int globalBlockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;
 static int blockChecksum = 0;
 static int streamChecksum = 1;
 static int blockIndependence = 1;
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+static int sparseFile = 0;
+#endif /* LZ4IO_ENABLE_SPARSE_FILE */
 
 static const int minBlockSizeID = 4;
 static const int maxBlockSizeID = 7;
@@ -174,6 +185,28 @@ int LZ4IO_setOverwrite(int yes)
    return overwrite;
 }
 
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+/* Default setting : sparseFile = 0; (Disable)
+   return : sparse file mode (0:Disable / 1:Enable) */
+int LZ4IO_setSparseFile(int yes)
+{
+    sparseFile = yes;
+    return sparseFile;
+}
+
+static int isSparse(const void* p, size_t size)
+{
+    const char* p8 = p;
+       for(; size; --size) {
+               if(*p8 != 0) {
+                       return 0;
+               }
+               ++p8;
+       }
+       return 1;
+}
+#endif /* LZ4IO_ENABLE_SPARSE_FILE */
+
 /* blockSizeID : valid values : 4-5-6-7 */
 int LZ4IO_setBlockSizeID(int bsid)
 {
@@ -539,6 +572,9 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
     LZ4F_decompressionContext_t ctx;
     LZ4F_errorCode_t errorCode;
     LZ4F_frameInfo_t frameInfo;
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+    size_t sparsePending = 0;
+#endif
 
     /* init */
     errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
@@ -580,9 +616,30 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
         filesize += decodedBytes;
 
         /* Write Block */
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+               if(sparseFile) {
+                       if(isSparse(outBuff, decodedBytes)) {
+                               sparsePending += decodedBytes;
+                               continue;
+                       }
+                       if(sparsePending > 0) {
+                               fseek(foutput, sparsePending, SEEK_CUR);
+                               sparsePending = 0;
+                       }
+               }
+#endif
         sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput);
         if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block\n");
     }
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+       if(sparseFile) {
+               if(sparsePending > 0) {
+                       fseek(foutput, sparsePending-1, SEEK_CUR);
+                       fputc(0, foutput);
+                       sparsePending = 0;
+               }
+       }
+#endif
 
     /* Free */
     free(inBuff);
@@ -645,6 +702,14 @@ int LZ4IO_decompressFilename(char* input_filename, char* output_filename)
     start = clock();
     get_fileHandle(input_filename, output_filename, &finput, &foutput);
 
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+    if (sparseFile!=0 && foutput!=0)
+    {
+        DISPLAY("Experimental : Using sparse file\n");
+        SET_SPARSE_FILE_MODE(foutput);
+    }
+#endif /* LZ4IO_ENABLE_SPARSE_FILE */
+
     /* Loop over multiple streams */
     do
     {

diff --git a/programs/lz4io.h b/programs/lz4io.h
index 7869a43..a0c6119 100644 (file)
--- a/programs/lz4io.h
+++ b/programs/lz4io.h
@@ -75,3 +75,8 @@ int LZ4IO_setStreamChecksumMode(int xxhash);
 
 /* Default setting : 0 (no notification) */
 int LZ4IO_setNotificationLevel(int level);
+
+#if defined(LZ4IO_ENABLE_SPARSE_FILE)
+/* Default setting : 0 (sparseFile = 0; disable sparse file) */
+int LZ4IO_setSparseFile(int yes);
+#endif