From 536bf52dbc4637141c8b0e391fcf988af673ac2c Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Mon, 26 Mar 2012 14:25:59 +0200 Subject: [PATCH] - implement support for xz/lzma compression --- CMakeLists.txt | 4 +- ext/solv_xfopen.c | 275 +++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 242 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 217361c..69497fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ OPTION (ENABLE_HELIXREPO "Build with helix repository support?" OFF) OPTION (ENABLE_DEBIAN "Build with debian database/repository support?" OFF) OPTION (ENABLE_MDKREPO "Build with mandriva/mageia repository support?" OFF) +OPTION (ENABLE_XZ_COMPRESSION "Build with xz/lzma compression support?" OFF) + #IF(${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERISION} GREATER 2.4) #ENDIF(${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERISION} GREATER 2.4) @@ -134,7 +136,7 @@ TEST_BIG_ENDIAN (WORDS_BIGENDIAN) # should create config.h with #cmakedefine instead... FOREACH (VAR HAVE_STRCHRNUL HAVE_FOPENCOOKIE HAVE_FUNOPEN WORDS_BIGENDIAN ENABLE_RPMDB ENABLE_RPMDB_PUBKEY ENABLE_RPMMD ENABLE_SUSEREPO ENABLE_COMPS - ENABLE_HELIXREPO ENABLE_MDKREPO ENABLE_DEBIAN) + ENABLE_HELIXREPO ENABLE_MDKREPO ENABLE_DEBIAN ENABLE_XZ_COMPRESSION) IF(${VAR}) ADD_DEFINITIONS (-D${VAR}=1) SET (SWIG_FLAGS ${SWIG_FLAGS} -D${VAR}) diff --git a/ext/solv_xfopen.c b/ext/solv_xfopen.c index df96556..9d94e01 100644 --- a/ext/solv_xfopen.c +++ b/ext/solv_xfopen.c @@ -8,12 +8,45 @@ #define _GNU_SOURCE #include +#include #include #include #include #include "solv_xfopen.h" + +static FILE *cookieopen(void *cookie, const char *mode, + ssize_t (*cread)(void *, char *, size_t), + ssize_t (*cwrite)(void *, const char *, size_t), + int (*cclose)(void *)) +{ + if (!cookie) + return 0; +#ifdef HAVE_FUNOPEN + return funopen(cookie, + (int (*)(void *, char *, int))(*mode == 'r' ? cread: NULL),/* readfn */ + (int (*)(void *, const char *, int))(*mode == 'w' ? cwrite : NULL), /* writefn */ + (fpos_t (*)(void *, fpos_t, int))NULL, /* seekfn */ + cclose + ); +#elif defined(HAVE_FOPENCOOKIE) + cookie_io_functions_t cio; + memset(&cio, 0, sizeof(cio)); + if (*mode == 'r') + cio.read = cread; + else if (*mode == 'w') + cio.write = cwrite; + cio.close = cclose; + return fopencookie(cookie, *mode == 'w' ? "w" : "r", cio); +#else +# error Need to implement custom I/O +#endif +} + + +/* gzip compression */ + static ssize_t cookie_gzread(void *cookie, char *buf, size_t nbytes) { return gzread((gzFile *)cookie, buf, nbytes); @@ -24,59 +57,226 @@ static ssize_t cookie_gzwrite(void *cookie, const char *buf, size_t nbytes) return gzwrite((gzFile *)cookie, buf, nbytes); } -static int -cookie_gzclose(void *cookie) +static int cookie_gzclose(void *cookie) { return gzclose((gzFile *)cookie); } -static FILE *mygzfopen(gzFile* gzf, const char *mode) +static inline FILE *mygzfopen(const char *fn, const char *mode) { -#ifdef HAVE_FUNOPEN - return funopen( - gzf, (int (*)(void *, char *, int))(*mode == 'r' ? cookie_gzread : NULL), /* readfn */ - (int (*)(void *, const char *, int))(*mode == 'w' ? cookie_gzwrite : NULL), /* writefn */ - (fpos_t (*)(void *, fpos_t, int))NULL, /* seekfn */ - cookie_gzclose - ); -#elif defined(HAVE_FOPENCOOKIE) - cookie_io_functions_t cio; - memset(&cio, 0, sizeof(cio)); - if (*mode == 'r') - cio.read = cookie_gzread; - else if (*mode == 'w') - cio.write = cookie_gzwrite; - cio.close = cookie_gzclose; - return fopencookie(gzf, *mode == 'w' ? "w" : "r", cio); -#else -# error Need to implement custom I/O -#endif + gzFile *gzf = gzopen(fn, mode); + return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose); +} + +static inline FILE *mygzfdopen(int fd, const char *mode) +{ + gzFile *gzf = gzdopen(fd, mode); + return cookieopen(gzf, mode, cookie_gzread, cookie_gzwrite, cookie_gzclose); } + +#ifdef ENABLE_XZ_COMPRESSION + +#include + +typedef struct lzfile { + unsigned char buf[1 << 15]; + lzma_stream strm; + FILE *file; + int encoding; + int eof; +} LZFILE; + +static LZFILE *lzopen(const char *path, const char *mode, int fd, int xz) +{ + int level = 7; /* Use XZ's default compression level if unspecified */ + int encoding = 0; + FILE *fp; + LZFILE *lzfile; + lzma_ret ret; + lzma_stream init_strm = LZMA_STREAM_INIT; + + if (!path && fd < 0) + return 0; + for (; *mode; mode++) { + if (*mode == 'w') + encoding = 1; + else if (*mode == 'r') + encoding = 0; + else if (*mode >= '1' && *mode <= '9') + level = *mode - '0'; + } + if (fd != -1) + fp = fdopen(fd, encoding ? "w" : "r"); + else + fp = fopen(path, encoding ? "w" : "r"); + if (!fp) + return 0; + lzfile = calloc(1, sizeof(*lzfile)); + if (!lzfile) + return 0; + lzfile->file = fp; + lzfile->encoding = encoding; + lzfile->eof = 0; + lzfile->strm = init_strm; + if (encoding) { + if (xz) { + ret = lzma_easy_encoder(&lzfile->strm, level, LZMA_CHECK_SHA256); + } else { + lzma_options_lzma options; + lzma_lzma_preset(&options, level); + ret = lzma_alone_encoder(&lzfile->strm, &options); + } + } else { /* lzma_easy_decoder_memusage(level) is not ready yet, use hardcoded limit for now */ + ret = lzma_auto_decoder(&lzfile->strm, 100<<20, 0); + } + if (ret != LZMA_OK) { + fclose(fp); + free(lzfile); + return 0; + } + return lzfile; +} + +static int lzclose(void *cookie) +{ + LZFILE *lzfile = cookie; + lzma_ret ret; + size_t n; + int rc; + + if (!lzfile) + return -1; + if (lzfile->encoding) { + for (;;) { + lzfile->strm.avail_out = sizeof(lzfile->buf); + lzfile->strm.next_out = lzfile->buf; + ret = lzma_code(&lzfile->strm, LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + return -1; + n = sizeof(lzfile->buf) - lzfile->strm.avail_out; + if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n) + return -1; + if (ret == LZMA_STREAM_END) + break; + } + } + lzma_end(&lzfile->strm); + rc = fclose(lzfile->file); + free(lzfile); + return rc; +} + +static ssize_t lzread(void *cookie, char *buf, size_t len) +{ + LZFILE *lzfile = cookie; + lzma_ret ret; + int eof = 0; + + if (!lzfile || lzfile->encoding) + return -1; + if (lzfile->eof) + return 0; + lzfile->strm.next_out = (unsigned char *)buf; + lzfile->strm.avail_out = len; + for (;;) { + if (!lzfile->strm.avail_in) { + lzfile->strm.next_in = lzfile->buf; + lzfile->strm.avail_in = fread(lzfile->buf, 1, sizeof(lzfile->buf), lzfile->file); + if (!lzfile->strm.avail_in) + eof = 1; + } + ret = lzma_code(&lzfile->strm, LZMA_RUN); + if (ret == LZMA_STREAM_END) { + lzfile->eof = 1; + return len - lzfile->strm.avail_out; + } + if (ret != LZMA_OK) + return -1; + if (!lzfile->strm.avail_out) + return len; + if (eof) + return -1; + } +} + +static ssize_t lzwrite(void *cookie, const char *buf, size_t len) +{ + LZFILE *lzfile = cookie; + lzma_ret ret; + size_t n; + if (!lzfile || !lzfile->encoding) + return -1; + if (!len) + return 0; + lzfile->strm.next_in = (unsigned char *)buf; + lzfile->strm.avail_in = len; + for (;;) { + lzfile->strm.next_out = lzfile->buf; + lzfile->strm.avail_out = sizeof(lzfile->buf); + ret = lzma_code(&lzfile->strm, LZMA_RUN); + if (ret != LZMA_OK) + return -1; + n = sizeof(lzfile->buf) - lzfile->strm.avail_out; + if (n && fwrite(lzfile->buf, 1, n, lzfile->file) != n) + return -1; + if (!lzfile->strm.avail_in) + return len; + } +} + +static inline FILE *myxzfopen(const char *fn, const char *mode) +{ + LZFILE *lzf = lzopen(fn, mode, -1, 1); + return cookieopen(lzf, mode, lzread, lzwrite, lzclose); +} + +static inline FILE *myxzfdopen(int fd, const char *mode) +{ + LZFILE *lzf = lzopen(0, mode, fd, 1); + return cookieopen(lzf, mode, lzread, lzwrite, lzclose); +} + +static inline FILE *mylzfopen(const char *fn, const char *mode) +{ + LZFILE *lzf = lzopen(fn, mode, -1, 0); + return cookieopen(lzf, mode, lzread, lzwrite, lzclose); +} + +static inline FILE *mylzfdopen(int fd, const char *mode) +{ + LZFILE *lzf = lzopen(0, mode, fd, 0); + return cookieopen(lzf, mode, lzread, lzwrite, lzclose); +} + +#endif /* ENABLE_XZ_COMPRESSION */ + + FILE * solv_xfopen(const char *fn, const char *mode) { char *suf; - gzFile *gzf; if (!fn) return 0; if (!mode) mode = "r"; suf = strrchr(fn, '.'); - if (!suf || strcmp(suf, ".gz") != 0) - return fopen(fn, mode); - gzf = gzopen(fn, mode); - if (!gzf) - return 0; - return mygzfopen(gzf, mode); + if (suf && !strcmp(suf, ".gz")) + return mygzfopen(fn, mode); +#ifdef ENABLE_XZ_COMPRESSION + if (suf && !strcmp(suf, ".xz")) + return myxzfopen(fn, mode); + if (suf && !strcmp(suf, ".lzma")) + return mylzfopen(fn, mode); +#endif + return fopen(fn, mode); } FILE * solv_xfopen_fd(const char *fn, int fd, const char *mode) { char *suf; - gzFile *gzf; suf = fn ? strrchr(fn, '.') : 0; if (!mode) @@ -97,11 +297,14 @@ solv_xfopen_fd(const char *fn, int fd, const char *mode) else mode = "r"; } - if (!suf || strcmp(suf, ".gz") != 0) - return fdopen(fd, mode); - gzf = gzdopen(fd, mode); - if (!gzf) - return 0; - return mygzfopen(gzf, mode); + if (suf && !strcmp(suf, ".gz")) + return mygzfdopen(fd, mode); +#ifdef ENABLE_XZ_COMPRESSION + if (suf && !strcmp(suf, ".xz")) + return myxzfdopen(fd, mode); + if (suf && !strcmp(suf, ".lzma")) + return mylzfdopen(fd, mode); +#endif + return fdopen(fd, mode); } -- 2.7.4