From 5aa368528be1b06f919cf520885b67ca6e643703 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 23 Dec 2024 17:40:31 +0800 Subject: [PATCH] erofs-utils: mkfs: allow disabling fragment deduplication Currently, although fragment inode compression is already multi-threaded, the data parts of inodes prior to their own fragments are still single-threaded if fragment deduplication is on. This can greatly slow down `-Eall-fragments` image building at least for the current mkfs codebase. Let's add an extended option `-E^fragdedupe` to explicitly disable it. After this commit, the Fedora Kiwi builds I'm testing can be reduced from 1148s (3,096,842,240 bytes, 2.9G) to 137s (2,969,956,352 bytes, 2.8G) with `-Eall-fragments,^fragdedupe -C524288 -z lzma,level=6, dictsize=524288` on Intel(R) Xeon(R) Platinum 8269CY CPU @ 2.50GHz with 32 cores. Cc: Neal Gompa Signed-off-by: Gao Xiang Link: https://lore.kernel.org/r/20241223094031.1534175-1-hsiangkao@linux.alibaba.com --- include/erofs/config.h | 1 + lib/compress.c | 3 ++- mkfs/main.c | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/include/erofs/config.h b/include/erofs/config.h index bb03e70..47e4d00 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -53,6 +53,7 @@ struct erofs_configure { bool c_fragments; bool c_all_fragments; bool c_dedupe; + bool c_nofragdedupe; bool c_ignore_mtime; bool c_showprogress; bool c_extra_ea_name_prefixes; diff --git a/lib/compress.c b/lib/compress.c index 07a6e57..a18b4b5 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -1527,7 +1527,8 @@ void *erofs_begin_compressed_file(struct erofs_inode *inode, int fd, u64 fpos) * Handle tails in advance to avoid writing duplicated * parts into the packed inode. */ - if (cfg.c_fragments && !erofs_is_packed_inode(inode)) { + if (cfg.c_fragments && !erofs_is_packed_inode(inode) && + !cfg.c_nofragdedupe) { ret = z_erofs_fragments_dedupe(inode, fd, &ictx->tof_chksum); if (ret < 0) goto err_free_ictx; diff --git a/mkfs/main.c b/mkfs/main.c index 9ca7dad..af5c08c 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -303,6 +303,15 @@ static int erofs_mkfs_feat_set_dedupe(bool en, const char *val, return 0; } +static int erofs_mkfs_feat_set_fragdedupe(bool en, const char *val, + unsigned int vallen) +{ + if (vallen) + return -EINVAL; + cfg.c_nofragdedupe = !en; + return 0; +} + static struct { char *feat; int (*set)(bool en, const char *val, unsigned int len); @@ -312,6 +321,7 @@ static struct { {"fragments", erofs_mkfs_feat_set_fragments}, {"all-fragments", erofs_mkfs_feat_set_all_fragments}, {"dedupe", erofs_mkfs_feat_set_dedupe}, + {"fragdedupe", erofs_mkfs_feat_set_fragdedupe}, {NULL, NULL}, }; -- 2.34.1