erofs-utils: support xz/lzma/lzip streams for tarerofs

Similar to commit e3dfe4b8db26 ("erofs-utils: mkfs: support tgz streams
for tarerofs"), let's add xz/lzma/lzip support by using liblzma.

Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20240303143530.4077607-1-hsiangkao@linux.alibaba.com

diff --git a/include/erofs/tar.h b/include/erofs/tar.h
index e45b89553f30ef6dd984cf750f22735a9a84ad30..b5c966bb9184ac76a4471f25c7ffa659ab10baad 100644 (file)
--- a/include/erofs/tar.h
+++ b/include/erofs/tar.h
@@ -26,11 +26,24 @@ struct erofs_pax_header {
 
 #define EROFS_IOS_DECODER_NONE         0
 #define EROFS_IOS_DECODER_GZIP         1
+#define EROFS_IOS_DECODER_LIBLZMA      2
+
+#ifdef HAVE_LIBLZMA
+#include <lzma.h>
+struct erofs_iostream_liblzma {
+       u8 inbuf[32768];
+       lzma_stream strm;
+       int fd;
+};
+#endif
 
 struct erofs_iostream {
        union {
                int fd;                 /* original fd */
                void *handler;
+#ifdef HAVE_LIBLZMA
+               struct erofs_iostream_liblzma *lzma;
+#endif
        };
        u64 sz;
        char *buffer;

diff --git a/lib/tar.c b/lib/tar.c
index 7c14c06d74b329e2aa26cfe21661e98ea3dc2b67..fe7fdd383a8872f9f2d9f08e47ef65f0d7d85d29 100644 (file)
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -70,6 +70,13 @@ void erofs_iostream_close(struct erofs_iostream *ios)
        if (ios->decoder == EROFS_IOS_DECODER_GZIP) {
 #if defined(HAVE_ZLIB)
                gzclose(ios->handler);
+#endif
+               return;
+       } else if (ios->decoder == EROFS_IOS_DECODER_LIBLZMA) {
+#if defined(HAVE_LIBLZMA)
+               lzma_end(&ios->lzma->strm);
+               close(ios->lzma->fd);
+               free(ios->lzma);
 #endif
                return;
        }
@@ -80,6 +87,7 @@ int erofs_iostream_open(struct erofs_iostream *ios, int fd, int decoder)
 {
        s64 fsz;
 
+       ios->feof = false;
        ios->tail = ios->head = 0;
        ios->decoder = decoder;
        ios->dumpfd = -1;
@@ -92,6 +100,24 @@ int erofs_iostream_open(struct erofs_iostream *ios, int fd, int decoder)
                ios->bufsize = 32768;
 #else
                return -EOPNOTSUPP;
+#endif
+       } else if (decoder == EROFS_IOS_DECODER_LIBLZMA) {
+#ifdef HAVE_LIBLZMA
+               lzma_ret ret;
+
+               ios->lzma = malloc(sizeof(*ios->lzma));
+               if (!ios->lzma)
+                       return -ENOMEM;
+               ios->lzma->fd = fd;
+               ios->lzma->strm = (lzma_stream)LZMA_STREAM_INIT;
+               ret = lzma_auto_decoder(&ios->lzma->strm,
+                                       UINT64_MAX, LZMA_CONCATENATED);
+               if (ret != LZMA_OK)
+                       return -EFAULT;
+               ios->sz = fsz = 0;
+               ios->bufsize = 32768;
+#else
+               return -EOPNOTSUPP;
 #endif
        } else {
                ios->fd = fd;
@@ -100,7 +126,6 @@ int erofs_iostream_open(struct erofs_iostream *ios, int fd, int decoder)
                        ios->feof = !fsz;
                        ios->sz = 0;
                } else {
-                       ios->feof = false;
                        ios->sz = fsz;
                        if (lseek(fd, 0, SEEK_SET))
                                return -EIO;
@@ -161,6 +186,37 @@ int erofs_iostream_read(struct erofs_iostream *ios, void **buf, u64 bytes)
                        ios->tail += ret;
 #else
                        return -EOPNOTSUPP;
+#endif
+               } else if (ios->decoder == EROFS_IOS_DECODER_LIBLZMA) {
+#ifdef HAVE_LIBLZMA
+                       struct erofs_iostream_liblzma *lzma = ios->lzma;
+                       lzma_action action = LZMA_RUN;
+                       lzma_ret ret2;
+
+                       if (!lzma->strm.avail_in) {
+                               lzma->strm.next_in = lzma->inbuf;
+                               ret = read(lzma->fd, lzma->inbuf,
+                                          sizeof(lzma->inbuf));
+                               if (ret < 0)
+                                       return -errno;
+                               lzma->strm.avail_in = ret;
+                               if (ret < sizeof(lzma->inbuf))
+                                       action = LZMA_FINISH;
+                       }
+                       lzma->strm.next_out = (u8 *)ios->buffer + rabytes;
+                       lzma->strm.avail_out = ios->bufsize - rabytes;
+
+                       ret2 = lzma_code(&lzma->strm, action);
+                       if (ret2 != LZMA_OK) {
+                               if (ret2 == LZMA_STREAM_END)
+                                       ios->feof = true;
+                               else
+                                       return -EIO;
+                       }
+                       ret = ios->bufsize - rabytes - lzma->strm.avail_out;
+                       ios->tail += ret;
+#else
+                       return -EOPNOTSUPP;
 #endif
                } else {
                        ret = erofs_read_from_fd(ios->fd, ios->buffer + rabytes,

diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 41eb5fb9658b56ce69041a8e57af4196865a3523..3bff41d3b4774b9b46cba31be2759c992b358270 100644 (file)
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -162,10 +162,6 @@ When this option is used together with
 the final file gids are
 set to \fIGID\fR + \fIGID-OFFSET\fR.
 .TP
-.BI \-\-ungzip\fR[\fP= file \fR]\fP
-Filter tarball streams through gzip. Optionally, raw streams can be dumped
-together.
-.TP
 \fB\-V\fR, \fB\-\-version\fR
 Print the version number and exit.
 .TP
@@ -210,6 +206,14 @@ When this option is used together with
 the final file uids are
 set to \fIUID\fR + \fIUIDOFFSET\fR.
 .TP
+.BI \-\-ungzip\fR[\fP= file \fR]\fP
+Filter tarball streams through gzip. Optionally, raw streams can be dumped
+together.
+.TP
+.BI \-\-unxz\fR[\fP= file \fR]\fP
+Filter tarball streams through xz, lzma, or lzip. Optionally, raw streams can
+be dumped together.
+.TP
 .BI "\-\-xattr-prefix=" PREFIX
 Specify a customized extended attribute namespace prefix for space saving,
 e.g. "trusted.overlay.".  You may give multiple

diff --git a/mkfs/main.c b/mkfs/main.c
index 258c1ce4da82dac7a89c5f3f6ed21ef68c4a6804..8a68a72dc777c1645399d186a9115f6a4c820c76 100644 (file)
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -69,11 +69,15 @@ static struct option long_options[] = {
        {"block-list-file", required_argument, NULL, 515},
 #endif
        {"ovlfs-strip", optional_argument, NULL, 516},
+       {"offset", required_argument, NULL, 517},
 #ifdef HAVE_ZLIB
-       {"gzip", no_argument, NULL, 517},
-       {"ungzip", optional_argument, NULL, 517},
+       {"gzip", no_argument, NULL, 518},
+       {"ungzip", optional_argument, NULL, 518},
+#endif
+#ifdef HAVE_LIBLZMA
+       {"unlzma", optional_argument, NULL, 519},
+       {"unxz", optional_argument, NULL, 519},
 #endif
-       {"offset", required_argument, NULL, 518},
        {0, 0, 0, 0},
 };
 
@@ -153,10 +157,6 @@ static void usage(int argc, char **argv)
                " --force-gid=#         set all file gids to # (# = GID)\n"
                " --uid-offset=#        add offset # to all file uids (# = id offset)\n"
                " --gid-offset=#        add offset # to all file gids (# = id offset)\n"
-#ifdef HAVE_ZLIB
-               " --ungzip[=X]          try to filter the tarball stream through gzip\n"
-               "                       (and optionally dump the raw stream to X together)\n"
-#endif
                " --ignore-mtime        use build time instead of strict per-file modification time\n"
                " --max-extent-bytes=#  set maximum decompressed extent size # in bytes\n"
                " --preserve-mtime      keep per-file modification time strictly\n"
@@ -170,6 +170,14 @@ static void usage(int argc, char **argv)
 #ifndef NDEBUG
                " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n"
                " --random-algorithms   randomize per-file algorithms (debugging only)\n"
+#endif
+#ifdef HAVE_ZLIB
+               " --ungzip[=X]          try to filter the tarball stream through gzip\n"
+               "                       (and optionally dump the raw stream to X together)\n"
+#endif
+#ifdef HAVE_LIBLZMA
+               " --unxz[=X]            try to filter the tarball stream through xz/lzma/lzip\n"
+               "                       (and optionally dump the raw stream to X together)\n"
 #endif
                " --xattr-prefix=X      X=extra xattr name prefix\n"
                " --mount-point=X       X=prefix of target fs path (default: /)\n"
@@ -194,7 +202,7 @@ static unsigned int pclustersize_packed, pclustersize_max;
 static struct erofs_tarfile erofstar = {
        .global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
 };
-static bool tar_mode, rebuild_mode, gzip_supported;
+static bool tar_mode, rebuild_mode;
 
 static unsigned int rebuild_src_count;
 static LIST_HEAD(rebuild_src_list);
@@ -413,6 +421,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
        char *endptr;
        int opt, i, err;
        bool quiet = false;
+       int tarerofs_decoder = 0;
 
        while ((opt = getopt_long(argc, argv, "C:E:L:T:U:b:d:x:z:Vh",
                                  long_options, NULL)) != -1) {
@@ -639,17 +648,18 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                                cfg.c_ovlfs_strip = false;
                        break;
                case 517:
-                       if (optarg)
-                               erofstar.dumpfile = strdup(optarg);
-                       gzip_supported = true;
-                       break;
-               case 518:
                        sbi.diskoffset = strtoull(optarg, &endptr, 0);
                        if (*endptr != '\0') {
                                erofs_err("invalid disk offset %s", optarg);
                                return -EINVAL;
                        }
                        break;
+               case 518:
+               case 519:
+                       if (optarg)
+                               erofstar.dumpfile = strdup(optarg);
+                       tarerofs_decoder = EROFS_IOS_DECODER_GZIP + (opt - 518);
+                       break;
                case 'V':
                        version();
                        exit(0);
@@ -696,7 +706,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                                          strerror(errno));
                                return -errno;
                        }
-                       err = erofs_iostream_open(&erofstar.ios, dupfd, gzip_supported);
+                       err = erofs_iostream_open(&erofstar.ios, dupfd,
+                                                 tarerofs_decoder);
                        if (err)
                                return err;
                }
@@ -717,7 +728,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                                erofs_err("failed to open file: %s", cfg.c_src_path);
                                return -errno;
                        }
-                       err = erofs_iostream_open(&erofstar.ios, fd, gzip_supported);
+                       err = erofs_iostream_open(&erofstar.ios, fd,
+                                                 tarerofs_decoder);
                        if (err)
                                return err;