erofs-utils: support dumping raw tar streams together

Since commit e3dfe4b8db26 ("erofs-utils: mkfs: support tgz streams for
tarerofs"), tgz streams can be converted to EROFS directly.

However, many use cases also require raw tar streams.  Let's add
support for dumping raw streams with `--ungzip=FILE` option.

Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20240222090145.709808-1-hsiangkao@linux.alibaba.com

diff --git a/include/erofs/tar.h b/include/erofs/tar.h
index a76f7402c44ed6a0af7401ab7f037b248ba185fd..be03d1bfe1d68430adea2e586b01c503016cf9e1 100644 (file)
--- a/include/erofs/tar.h
+++ b/include/erofs/tar.h
@@ -35,14 +35,14 @@ struct erofs_iostream {
        u64 sz;
        char *buffer;
        unsigned int head, tail, bufsize;
-       int decoder;
+       int decoder, dumpfd;
        bool feof;
 };
 
 struct erofs_tarfile {
        struct erofs_pax_header global;
        struct erofs_iostream ios;
-       char *mapfile;
+       char *mapfile, *dumpfile;
 
        int fd;
        u64 offset;

diff --git a/lib/tar.c b/lib/tar.c
index ead74ba40b22e20f190dfecbb192fbf55bf478fc..1d764b24cd176c9b1f7efe2be930f935c32ac319 100644 (file)
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -82,6 +82,7 @@ int erofs_iostream_open(struct erofs_iostream *ios, int fd, int decoder)
 
        ios->tail = ios->head = 0;
        ios->decoder = decoder;
+       ios->dumpfd = -1;
        if (decoder == EROFS_IOS_DECODER_GZIP) {
 #if defined(HAVE_ZLIB)
                ios->handler = gzdopen(fd, "r");
@@ -170,6 +171,10 @@ int erofs_iostream_read(struct erofs_iostream *ios, void **buf, u64 bytes)
                        if (ret < ios->bufsize - rabytes)
                                ios->feof = true;
                }
+               if (unlikely(ios->dumpfd >= 0))
+                       if (write(ios->dumpfd, ios->buffer + rabytes, ret) < ret)
+                               erofs_err("failed to dump %d bytes of the raw stream: %s",
+                                         ret, erofs_strerror(-errno));
        }
        *buf = ios->buffer;
        ret = min_t(int, ios->tail, bytes);
@@ -210,7 +215,7 @@ int erofs_iostream_lskip(struct erofs_iostream *ios, u64 sz)
        if (ios->feof)
                return sz;
 
-       if (ios->sz) {
+       if (ios->sz && likely(ios->dumpfd < 0)) {
                s64 cur = lseek(ios->fd, sz, SEEK_CUR);
 
                if (cur > ios->sz)

diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 45be11abb630faf3b50d7f2cbf86014fb32b5e18..f32dc26e788cb7447ea2af50a09be8bf1342accb 100644 (file)
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -162,8 +162,9 @@ When this option is used together with
 the final file gids are
 set to \fIGID\fR + \fIGID-OFFSET\fR.
 .TP
-.B \-\-gzip
-Filter tarball streams through gzip.
+.BI \-\-ungzip\fR[\fP= file \fR]\fP
+Filter tarball streams through gzip. Optionally, raw streams can be dumped
+together.
 .TP
 \fB\-V\fR, \fB\-\-version\fR
 Print the version number and exit.

diff --git a/mkfs/main.c b/mkfs/main.c
index 7aea64a973d27459eb4f3403f20a41cd265fa8d6..3191b89f347b236c3f72e9edd9ffd4fda2aff77d 100644 (file)
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -71,6 +71,7 @@ static struct option long_options[] = {
        {"ovlfs-strip", optional_argument, NULL, 516},
 #ifdef HAVE_ZLIB
        {"gzip", no_argument, NULL, 517},
+       {"ungzip", optional_argument, NULL, 517},
 #endif
        {"offset", required_argument, NULL, 518},
        {0, 0, 0, 0},
@@ -153,7 +154,8 @@ static void usage(int argc, char **argv)
                " --uid-offset=#        add offset # to all file uids (# = id offset)\n"
                " --gid-offset=#        add offset # to all file gids (# = id offset)\n"
 #ifdef HAVE_ZLIB
-               " --gzip                try to filter the tarball stream through gzip\n"
+               " --ungzip[=X]          try to filter the tarball stream through gzip\n"
+               "                       (and optionally dump the raw stream to X together)\n"
 #endif
                " --ignore-mtime        use build time instead of strict per-file modification time\n"
                " --max-extent-bytes=#  set maximum decompressed extent size # in bytes\n"
@@ -633,6 +635,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                                cfg.c_ovlfs_strip = false;
                        break;
                case 517:
+                       if (optarg)
+                               erofstar.dumpfile = strdup(optarg);
                        gzip_supported = true;
                        break;
                case 518:
@@ -712,6 +716,17 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                        err = erofs_iostream_open(&erofstar.ios, fd, gzip_supported);
                        if (err)
                                return err;
+
+                       if (erofstar.dumpfile) {
+                               fd = open(erofstar.dumpfile,
+                                         O_WRONLY | O_CREAT | O_TRUNC, 0644);
+                               if (fd < 0) {
+                                       erofs_err("failed to open dumpfile: %s",
+                                                 erofstar.dumpfile);
+                                       return -errno;
+                               }
+                               erofstar.ios.dumpfd = fd;
+                       }
                } else {
                        err = lstat(cfg.c_src_path, &st);
                        if (err)
@@ -1315,8 +1330,11 @@ exit:
        erofs_rebuild_cleanup();
        erofs_diskbuf_exit();
        erofs_exit_configure();
-       if (tar_mode)
+       if (tar_mode) {
                erofs_iostream_close(&erofstar.ios);
+               if (erofstar.ios.dumpfd >= 0)
+                       close(erofstar.ios.dumpfd);
+       }
 
        if (err) {
                erofs_err("\tCould not format the device : %s\n",