erofs-utils: mkfs: add `--sort=none`

Currently, `--tar=f` writes file data twice due to unseekable streams
and EROFS data ordering requirements.  Some use cases may need to avoid
unnecessary data writes for performance and do NOT require a strict
data ordering.

It adds `--sort=none` to address this.  The image is still reproducible;
it simply means no specific file data ordering is implied.

Currently, It comes into effect if `-E^inline_data` is specified and no
compression is applied.

Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20240923074929.2445674-1-hsiangkao@linux.alibaba.com

diff --git a/include/erofs/inode.h b/include/erofs/inode.h
index 604161c2a39af8326c4d8e914d027b2e6de4f2f8..eb8f45b45552a2987c78f523f6e1d9cce4e70d74 100644 (file)
--- a/include/erofs/inode.h
+++ b/include/erofs/inode.h
@@ -34,6 +34,7 @@ erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
 int erofs_iflush(struct erofs_inode *inode);
 struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
                                   const char *name);
+int erofs_allocate_inode_bh_data(struct erofs_inode *inode, erofs_blk_t nblocks);
 bool erofs_dentry_is_wht(struct erofs_sb_info *sbi, struct erofs_dentry *d);
 int erofs_rebuild_dump_tree(struct erofs_inode *dir, bool incremental);
 int erofs_init_empty_dir(struct erofs_inode *dir);

diff --git a/include/erofs/tar.h b/include/erofs/tar.h
index 42fbb00d20ed2a9dd97b289d0d0a5209bab31e73..6981f9e18e3a1bb996da5c22959dd17d8515c407 100644 (file)
--- a/include/erofs/tar.h
+++ b/include/erofs/tar.h
@@ -52,6 +52,7 @@ struct erofs_tarfile {
        u64 offset;
        bool index_mode, headeronly_mode, rvsp_mode, aufs;
        bool ddtaridx_mode;
+       bool try_no_reorder;
 };
 
 void erofs_iostream_close(struct erofs_iostream *ios);

diff --git a/lib/inode.c b/lib/inode.c
index bc3cb76e8bf2282cb28c49c2fe997601eb143630..f08f3950d7f8dbcd21c9616483846ab16722f9d5 100644 (file)
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -171,14 +171,12 @@ struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
        return d;
 }
 
-/* allocate main data for a inode */
-static int __allocate_inode_bh_data(struct erofs_inode *inode,
-                                   unsigned long nblocks,
-                                   int type)
+/* allocate main data for an inode */
+int erofs_allocate_inode_bh_data(struct erofs_inode *inode, erofs_blk_t nblocks)
 {
        struct erofs_bufmgr *bmgr = inode->sbi->bmgr;
        struct erofs_buffer_head *bh;
-       int ret;
+       int ret, type;
 
        if (!nblocks) {
                /* it has only tail-end data */
@@ -187,6 +185,7 @@ static int __allocate_inode_bh_data(struct erofs_inode *inode,
        }
 
        /* allocate main data buffer */
+       type = S_ISDIR(inode->i_mode) ? DIRA : DATA;
        bh = erofs_balloc(bmgr, type, erofs_pos(inode->sbi, nblocks), 0, 0);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
@@ -431,7 +430,7 @@ static int erofs_write_dir_file(struct erofs_inode *dir)
        q = used = blkno = 0;
 
        /* allocate dir main data */
-       ret = __allocate_inode_bh_data(dir, erofs_blknr(sbi, dir->i_size), DIRA);
+       ret = erofs_allocate_inode_bh_data(dir, erofs_blknr(sbi, dir->i_size));
        if (ret)
                return ret;
 
@@ -487,7 +486,7 @@ int erofs_write_file_from_buffer(struct erofs_inode *inode, char *buf)
 
        inode->datalayout = EROFS_INODE_FLAT_INLINE;
 
-       ret = __allocate_inode_bh_data(inode, nblocks, DATA);
+       ret = erofs_allocate_inode_bh_data(inode, nblocks);
        if (ret)
                return ret;
 
@@ -514,15 +513,15 @@ static bool erofs_file_is_compressible(struct erofs_inode *inode)
 
 static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
 {
-       int ret;
+       struct erofs_sb_info *sbi = inode->sbi;
        erofs_blk_t nblocks, i;
        unsigned int len;
-       struct erofs_sb_info *sbi = inode->sbi;
+       int ret;
 
        inode->datalayout = EROFS_INODE_FLAT_INLINE;
        nblocks = inode->i_size >> sbi->blkszbits;
 
-       ret = __allocate_inode_bh_data(inode, nblocks, DATA);
+       ret = erofs_allocate_inode_bh_data(inode, nblocks);
        if (ret)
                return ret;
 

diff --git a/lib/tar.c b/lib/tar.c
index 6d3529235a7032fceaadb65e6451cd3e6cbe2a22..b32abd43d633b4ef67862f7f7e7f17cf584139a2 100644 (file)
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -119,7 +119,7 @@ int erofs_iostream_open(struct erofs_iostream *ios, int fd, int decoder)
                                           erofs_strerror(-errno));
 #endif
                }
-               ios->bufsize = 16384;
+               ios->bufsize = 32768;
        }
 
        do {
@@ -586,6 +586,38 @@ void tarerofs_remove_inode(struct erofs_inode *inode)
        --inode->i_parent->i_nlink;
 }
 
+static int tarerofs_write_uncompressed_file(struct erofs_inode *inode,
+                                           struct erofs_tarfile *tar)
+{
+       struct erofs_sb_info *sbi = inode->sbi;
+       erofs_blk_t nblocks;
+       erofs_off_t pos;
+       void *buf;
+       int ret;
+
+       inode->datalayout = EROFS_INODE_FLAT_PLAIN;
+       nblocks = DIV_ROUND_UP(inode->i_size, 1U << sbi->blkszbits);
+
+       ret = erofs_allocate_inode_bh_data(inode, nblocks);
+       if (ret)
+               return ret;
+
+       for (pos = 0; pos < inode->i_size; pos += ret) {
+               ret = erofs_iostream_read(&tar->ios, &buf, inode->i_size - pos);
+               if (ret < 0)
+                       break;
+               if (erofs_dev_write(sbi, buf,
+                                   erofs_pos(sbi, inode->u.i_blkaddr) + pos,
+                                   ret)) {
+                       ret = -EIO;
+                       break;
+               }
+       }
+       inode->idata_size = 0;
+       inode->datasource = EROFS_INODE_DATA_SOURCE_NONE;
+       return 0;
+}
+
 static int tarerofs_write_file_data(struct erofs_inode *inode,
                                    struct erofs_tarfile *tar)
 {
@@ -1012,6 +1044,10 @@ new_inode:
                                if (!ret && erofs_iostream_lskip(&tar->ios,
                                                                 inode->i_size))
                                        ret = -EIO;
+                       } else if (tar->try_no_reorder &&
+                                  !cfg.c_compr_opts[0].alg &&
+                                  !cfg.c_inline_data) {
+                               ret = tarerofs_write_uncompressed_file(inode, tar);
                        } else {
                                ret = tarerofs_write_file_data(inode, tar);
                        }

diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index d599fac3d30acc0add8d0182a99a39d60daee6c8..abdd9b9d3fe0d7d25e5557a2a7f368fac83ec66c 100644 (file)
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -192,7 +192,18 @@ Use extended inodes instead of compact inodes if the file modification time
 would overflow compact inodes. This is the default. Overrides
 .BR --ignore-mtime .
 .TP
-.BI "\-\-tar, \-\-tar="MODE
+.BI "\-\-sort=" MODE
+Inode data sorting order for tarballs as input.
+
+\fIMODE\fR may be one of \fBnone\fR or \fBpath\fR.
+
+\fBnone\fR: No particular data order is specified for the target image to
+avoid unnecessary overhead; Currently, it takes effect if `-E^inline_data` is
+specified and no compression is applied.
+
+\fBpath\fR: Data order strictly follows the tree generation order. (default)
+.TP
+.BI "\-\-tar, \-\-tar=" MODE
 Treat \fISOURCE\fR as a tarball or tarball-like "headerball" rather than as a
 directory.
 

diff --git a/mkfs/main.c b/mkfs/main.c
index 8f1fdbc82338ba2bfe38439bf2d55020cbcc2a93..d42278716db9a2d469bb9531c3cd420d65e7769a 100644 (file)
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -84,6 +84,7 @@ static struct option long_options[] = {
        {"root-xattr-isize", required_argument, NULL, 524},
        {"mkfs-time", no_argument, NULL, 525},
        {"all-time", no_argument, NULL, 526},
+       {"sort", required_argument, NULL, 527},
        {0, 0, 0, 0},
 };
 
@@ -180,6 +181,7 @@ static void usage(int argc, char **argv)
                " --offset=#            skip # bytes at the beginning of IMAGE.\n"
                " --root-xattr-isize=#  ensure the inline xattr size of the root directory is # bytes at least\n"
                " --aufs                replace aufs special files with overlayfs metadata\n"
+               " --sort=<path,none>    data sorting order for tarballs as input (default: path)\n"
                " --tar=X               generate a full or index-only image from a tarball(-ish) source\n"
                "                       (X = f|i|headerball; f=full mode, i=index mode,\n"
                "                                            headerball=file data is omited in the source stream)\n"
@@ -840,6 +842,10 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                case 526:
                        cfg.c_timeinherit = TIMESTAMP_FIXED;
                        break;
+               case 527:
+                       if (!strcmp(optarg, "none"))
+                               erofstar.try_no_reorder = true;
+                       break;
                case 'V':
                        version();
                        exit(0);