erofs-utils: mkfs: allow disabling fragment deduplication

Currently, although fragment inode compression is already
multi-threaded, the data parts of inodes prior to their own fragments
are still single-threaded if fragment deduplication is on.  This can
greatly slow down `-Eall-fragments` image building at least for
the current mkfs codebase.

Let's add an extended option `-E^fragdedupe` to explicitly disable it.

After this commit, the Fedora Kiwi builds I'm testing can be reduced
from 1148s (3,096,842,240 bytes, 2.9G) to 137s (2,969,956,352 bytes,
2.8G) with `-Eall-fragments,^fragdedupe -C524288 -z lzma,level=6,
dictsize=524288` on Intel(R) Xeon(R) Platinum 8269CY CPU @ 2.50GHz with
32 cores.

Cc: Neal Gompa <ngompa13@gmail.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20241223094031.1534175-1-hsiangkao@linux.alibaba.com

diff --git a/include/erofs/config.h b/include/erofs/config.h
index bb03e708cfbb95140d73d6656b8cf2671f44abf9..47e4d0005ed55296c3e83e57017ab715d6dc76c3 100644 (file)
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -53,6 +53,7 @@ struct erofs_configure {
        bool c_fragments;
        bool c_all_fragments;
        bool c_dedupe;
+       bool c_nofragdedupe;
        bool c_ignore_mtime;
        bool c_showprogress;
        bool c_extra_ea_name_prefixes;

diff --git a/lib/compress.c b/lib/compress.c
index 07a6e57796d149868dbb7e930d9d2483936db028..a18b4b5a4a778fca14795ab2684863905d38ac00 100644 (file)
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -1527,7 +1527,8 @@ void *erofs_begin_compressed_file(struct erofs_inode *inode, int fd, u64 fpos)
         * Handle tails in advance to avoid writing duplicated
         * parts into the packed inode.
         */
-       if (cfg.c_fragments && !erofs_is_packed_inode(inode)) {
+       if (cfg.c_fragments && !erofs_is_packed_inode(inode) &&
+           !cfg.c_nofragdedupe) {
                ret = z_erofs_fragments_dedupe(inode, fd, &ictx->tof_chksum);
                if (ret < 0)
                        goto err_free_ictx;

diff --git a/mkfs/main.c b/mkfs/main.c
index 9ca7dad93f465135d46eb8ee8b8e682ae10e8bef..af5c08c210dfe200816da078decf72db45ac6326 100644 (file)
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -303,6 +303,15 @@ static int erofs_mkfs_feat_set_dedupe(bool en, const char *val,
        return 0;
 }
 
+static int erofs_mkfs_feat_set_fragdedupe(bool en, const char *val,
+                                         unsigned int vallen)
+{
+       if (vallen)
+               return -EINVAL;
+       cfg.c_nofragdedupe = !en;
+       return 0;
+}
+
 static struct {
        char *feat;
        int (*set)(bool en, const char *val, unsigned int len);
@@ -312,6 +321,7 @@ static struct {
        {"fragments", erofs_mkfs_feat_set_fragments},
        {"all-fragments", erofs_mkfs_feat_set_all_fragments},
        {"dedupe", erofs_mkfs_feat_set_dedupe},
+       {"fragdedupe", erofs_mkfs_feat_set_fragdedupe},
        {NULL, NULL},
 };