erofs-utils: enable incremental builds

`--incremental=<data|rvsp>` are now supported for tarerofs but
only `--incremental=rvsp` works for the rebuild mode.

For example:
$ mkfs.erofs --tar=f --gzip --aufs --clean=data foo.erofs f0.tgz
$ mkfs.erofs --tar=f --gzip --aufs --incremental=data foo.erofs f1.tgz
...
$ mkfs.erofs --tar=f --gzip --aufs --incremental=data foo.erofs fn-1.tgz

Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Link: https://lore.kernel.org/r/20240618082414.47876-8-hsiangkao@linux.alibaba.com

diff --git a/include/erofs/io.h b/include/erofs/io.h
index f24a563b877c8948147ae769760ba183953bf874..d3a487f81a8cc68aaace1400cb174c1e909888b2 100644 (file)
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -32,6 +32,7 @@ struct erofs_vfops {
        int (*ftruncate)(struct erofs_vfile *vf, u64 length);
        ssize_t (*read)(struct erofs_vfile *vf, void *buf, size_t len);
        off_t (*lseek)(struct erofs_vfile *vf, u64 offset, int whence);
+       int (*fstat)(struct erofs_vfile *vf, struct stat *buf);
 };
 
 struct erofs_vfile {
@@ -40,6 +41,7 @@ struct erofs_vfile {
        int fd;
 };
 
+int erofs_io_fstat(struct erofs_vfile *vf, struct stat *buf);
 ssize_t erofs_io_pwrite(struct erofs_vfile *vf, const void *buf, u64 pos, size_t len);
 int erofs_io_fsync(struct erofs_vfile *vf);
 ssize_t erofs_io_fallocate(struct erofs_vfile *vf, u64 offset, size_t len, bool pad);

diff --git a/include/erofs/tar.h b/include/erofs/tar.h
index e1de0dfdca7c79ad749266a13c57bc2cae54f38c..403f3a8271c9832d17e4c06facdebcf95931ab22 100644 (file)
--- a/include/erofs/tar.h
+++ b/include/erofs/tar.h
@@ -57,9 +57,10 @@ struct erofs_tarfile {
        struct erofs_iostream ios;
        char *mapfile, *dumpfile;
 
+       u32 dev;
        int fd;
        u64 offset;
-       bool index_mode, headeronly_mode, aufs;
+       bool index_mode, headeronly_mode, rvsp_mode, aufs;
 };
 
 void erofs_iostream_close(struct erofs_iostream *ios);

diff --git a/lib/io.c b/lib/io.c
index 576d55b5b8a0dc7ab413dcdb71b32e64b0d2d2c0..a2ef3446c0cac041cdbc40ba85a595f0fc0d22a8 100644 (file)
--- a/lib/io.c
+++ b/lib/io.c
@@ -26,6 +26,19 @@
 #define EROFS_MODNAME  "erofs_io"
 #include "erofs/print.h"
 
+int erofs_io_fstat(struct erofs_vfile *vf, struct stat *buf)
+{
+       if (unlikely(cfg.c_dry_run)) {
+               buf->st_size = 0;
+               buf->st_mode = S_IFREG | 0777;
+               return 0;
+       }
+
+       if (vf->ops)
+               return vf->ops->fstat(vf, buf);
+       return fstat(vf->fd, buf);
+}
+
 ssize_t erofs_io_pwrite(struct erofs_vfile *vf, const void *buf,
                        u64 pos, size_t len)
 {

diff --git a/lib/tar.c b/lib/tar.c
index 53a11880a7220bb86222f85bb9e132c0bba3bd18..8d2caa5ed4eb04e4125ddfceef2337f2a568ce93 100644 (file)
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -942,6 +942,7 @@ new_inode:
                        ret = PTR_ERR(inode);
                        goto out;
                }
+               inode->dev = tar->dev;
                inode->i_parent = d->inode;
                d->inode = inode;
                d->type = erofs_mode_to_ftype(st.st_mode);
@@ -981,13 +982,21 @@ new_inode:
                        inode->i_link = malloc(inode->i_size + 1);
                        memcpy(inode->i_link, eh.link, inode->i_size + 1);
                } else if (inode->i_size) {
-                       if (tar->headeronly_mode)
+                       if (tar->headeronly_mode) {
                                ret = erofs_write_zero_inode(inode);
-                       else if (tar->index_mode)
+                       } else if (tar->rvsp_mode) {
+                               inode->datasource = EROFS_INODE_DATA_SOURCE_RESVSP;
+                               inode->i_ino[1] = data_offset;
+                               if (erofs_iostream_lskip(&tar->ios, inode->i_size))
+                                       ret = -EIO;
+                               else
+                                       ret = 0;
+                       } else if (tar->index_mode) {
                                ret = tarerofs_write_file_index(inode, tar,
                                                                data_offset);
-                       else
+                       } else {
                                ret = tarerofs_write_file_data(inode, tar);
+                       }
                        if (ret)
                                goto out;
                }

diff --git a/mkfs/main.c b/mkfs/main.c
index 425fe00b80103b9287165c6e48c3a109db6e1515..971f109c5e2c981de4a8d098781b9959aacbe6f7 100644 (file)
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -79,6 +79,8 @@ static struct option long_options[] = {
        {"workers", required_argument, NULL, 520},
 #endif
        {"zfeature-bits", required_argument, NULL, 521},
+       {"clean", optional_argument, NULL, 522},
+       {"incremental", optional_argument, NULL, 523},
        {0, 0, 0, 0},
 };
 
@@ -152,6 +154,10 @@ static void usage(int argc, char **argv)
                " --all-root            make all files owned by root\n"
                " --blobdev=X           specify an extra device X to store chunked data\n"
                " --chunksize=#         generate chunk-based files with #-byte chunks\n"
+               " --clean=X             run full clean build (default) or:\n"
+               " --incremental=X       run incremental build\n"
+               "                       (X = data|rvsp; data=full data, rvsp=space is allocated\n"
+               "                                       and filled with zeroes)\n"
                " --compress-hints=X    specify a file to configure per-file compression strategy\n"
                " --exclude-path=X      avoid including file X (X = exact literal path)\n"
                " --exclude-regex=X     avoid including files that match X (X = regular expression)\n"
@@ -214,7 +220,14 @@ static unsigned int pclustersize_packed, pclustersize_max;
 static struct erofs_tarfile erofstar = {
        .global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
 };
-static bool tar_mode, rebuild_mode;
+static bool tar_mode, rebuild_mode, incremental_mode;
+
+enum {
+       EROFS_MKFS_DATA_IMPORT_DEFAULT,
+       EROFS_MKFS_DATA_IMPORT_FULLDATA,
+       EROFS_MKFS_DATA_IMPORT_RVSP,
+       EROFS_MKFS_DATA_IMPORT_SPARSE,
+} dataimport_mode;
 
 static unsigned int rebuild_src_count;
 static LIST_HEAD(rebuild_src_list);
@@ -775,6 +788,22 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
                        if (err)
                                return err;
                        break;
+               case 522:
+               case 523:
+                       if (!optarg || !strcmp(optarg, "data")) {
+                               dataimport_mode = EROFS_MKFS_DATA_IMPORT_FULLDATA;
+                       } else if (!strcmp(optarg, "rvsp")) {
+                               dataimport_mode = EROFS_MKFS_DATA_IMPORT_RVSP;
+                       } else {
+                               dataimport_mode = strtol(optarg, &endptr, 0);
+                               if (errno || *endptr != '\0') {
+                                       erofs_err("invalid --%s=%s",
+                                                 opt == 523 ? "incremental" : "clean", optarg);
+                                       return -EINVAL;
+                               }
+                       }
+                       incremental_mode = (opt == 523);
+                       break;
                case 'V':
                        version();
                        exit(0);
@@ -998,9 +1027,6 @@ static void erofs_mkfs_default_options(void)
        sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_ZERO_PADDING;
        sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM |
                             EROFS_FEATURE_COMPAT_MTIME;
-
-       /* generate a default uuid first */
-       erofs_uuid_generate(sbi.uuid);
 }
 
 /* https://reproducible-builds.org/specs/source-date-epoch/ for more details */
@@ -1052,16 +1078,30 @@ static struct erofs_inode *erofs_mkfs_alloc_root(struct erofs_sb_info *sbi)
        return root;
 }
 
-static int erofs_rebuild_load_trees(struct erofs_inode *root)
+static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root)
 {
        struct erofs_sb_info *src;
        unsigned int extra_devices = 0;
        erofs_blk_t nblocks;
        int ret, idx;
+       enum erofs_rebuild_datamode datamode;
+
+       switch (dataimport_mode) {
+       case EROFS_MKFS_DATA_IMPORT_DEFAULT:
+               datamode = EROFS_REBUILD_DATA_BLOB_INDEX;
+               break;
+       case EROFS_MKFS_DATA_IMPORT_FULLDATA:
+               datamode = EROFS_REBUILD_DATA_FULL;
+               break;
+       case EROFS_MKFS_DATA_IMPORT_RVSP:
+               datamode = EROFS_REBUILD_DATA_RESVSP;
+               break;
+       default:
+               return -EINVAL;
+       }
 
        list_for_each_entry(src, &rebuild_src_list, list) {
-               ret = erofs_rebuild_load_tree(root, src,
-                                             EROFS_REBUILD_DATA_BLOB_INDEX);
+               ret = erofs_rebuild_load_tree(root, src, datamode);
                if (ret) {
                        erofs_err("failed to load %s", src->devname);
                        return ret;
@@ -1074,7 +1114,10 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
                extra_devices += src->extra_devices;
        }
 
-       if (extra_devices && extra_devices != rebuild_src_count) {
+       if (datamode != EROFS_REBUILD_DATA_BLOB_INDEX)
+               return 0;
+
+       if (extra_devices != rebuild_src_count) {
                erofs_err("extra_devices(%u) is mismatched with source images(%u)",
                          extra_devices, rebuild_src_count);
                return -EOPNOTSUPP;
@@ -1117,6 +1160,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
 static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
 {
        char uuid_str[37] = {};
+       char *incr = incremental_mode ? "new" : "total";
 
        if (!(cfg.c_dbg_lvl > EROFS_ERR && cfg.c_showprogress))
                return;
@@ -1126,11 +1170,11 @@ static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
        fprintf(stdout, "------\nFilesystem UUID: %s\n"
                "Filesystem total blocks: %u (of %u-byte blocks)\n"
                "Filesystem total inodes: %llu\n"
-               "Filesystem total metadata blocks: %u\n"
-               "Filesystem total deduplicated bytes (of source files): %llu\n",
+               "Filesystem %s metadata blocks: %u\n"
+               "Filesystem %s deduplicated bytes (of source files): %llu\n",
                uuid_str, nblocks, 1U << sbi.blkszbits, sbi.inos | 0ULL,
-               erofs_total_metablocks(),
-               sbi.saved_by_deduplication | 0ULL);
+               incr, erofs_total_metablocks(),
+               incr, sbi.saved_by_deduplication | 0ULL);
 }
 
 int main(int argc, char **argv)
@@ -1167,20 +1211,13 @@ int main(int argc, char **argv)
                sbi.build_time_nsec = t.tv_usec;
        }
 
-       err = erofs_dev_open(&sbi, cfg.c_img_path, O_RDWR | O_TRUNC);
+       err = erofs_dev_open(&sbi, cfg.c_img_path, O_RDWR |
+                               (incremental_mode ? 0 : O_TRUNC));
        if (err) {
                fprintf(stderr, "Try '%s --help' for more information.\n", argv[0]);
                return 1;
        }
 
-       if (tar_mode && !erofstar.index_mode) {
-               err = erofs_diskbuf_init(1);
-               if (err) {
-                       erofs_err("failed to initialize diskbuf: %s",
-                                  strerror(-err));
-                       goto exit;
-               }
-       }
 #ifdef WITH_ANDROID
        if (cfg.fs_config_file &&
            load_canned_fs_config(cfg.fs_config_file) < 0) {
@@ -1218,14 +1255,22 @@ int main(int argc, char **argv)
        if (cfg.c_random_pclusterblks)
                srand(time(NULL));
 #endif
-       if (tar_mode && erofstar.index_mode) {
+       if (tar_mode) {
+               if (dataimport_mode == EROFS_MKFS_DATA_IMPORT_RVSP)
+                       erofstar.rvsp_mode = true;
+               erofstar.dev = rebuild_src_count + 1;
+
                if (erofstar.mapfile) {
                        err = erofs_blocklist_open(erofstar.mapfile, true);
                        if (err) {
                                erofs_err("failed to open %s", erofstar.mapfile);
                                goto exit;
                        }
-               } else {
+               } else if (erofstar.index_mode) {
+                       /*
+                        * If mapfile is unspecified for tarfs index mode,
+                        * 512-byte block size is enforced here.
+                        */
                        sbi.blkszbits = 9;
                }
        }
@@ -1246,10 +1291,43 @@ int main(int argc, char **argv)
                sbi.blkszbits = src->blkszbits;
        }
 
-       sb_bh = erofs_reserve_sb();
-       if (IS_ERR(sb_bh)) {
-               err = PTR_ERR(sb_bh);
-               goto exit;
+       if (!incremental_mode) {
+               sb_bh = erofs_reserve_sb();
+               if (IS_ERR(sb_bh)) {
+                       err = PTR_ERR(sb_bh);
+                       goto exit;
+               }
+               /* generate new UUIDs for clean builds */
+               erofs_uuid_generate(sbi.uuid);
+       } else {
+               union {
+                       struct stat st;
+                       erofs_blk_t startblk;
+               } u;
+
+               erofs_warn("EXPERIMENTAL incremental build in use. Use at your own risk!");
+               err = erofs_read_superblock(&sbi);
+               if (err) {
+                       erofs_err("failed to read superblock of %s", sbi.devname);
+                       goto exit;
+               }
+
+               err = erofs_io_fstat(&sbi.bdev, &u.st);
+               if (!err && S_ISREG(u.st.st_mode))
+                       u.startblk = DIV_ROUND_UP(u.st.st_size, erofs_blksiz(&sbi));
+               else
+                       u.startblk = sbi.primarydevice_blocks;
+               erofs_buffer_init(u.startblk);
+               sb_bh = NULL;
+       }
+
+       if (tar_mode && !erofstar.index_mode) {
+               err = erofs_diskbuf_init(1);
+               if (err) {
+                       erofs_err("failed to initialize diskbuf: %s",
+                                  strerror(-err));
+                       goto exit;
+               }
        }
 
        err = erofs_load_compress_hints(&sbi);
@@ -1310,7 +1388,7 @@ int main(int argc, char **argv)
                if (err < 0)
                        goto exit;
 
-               err = erofs_rebuild_dump_tree(root, false);
+               err = erofs_rebuild_dump_tree(root, incremental_mode);
                if (err < 0)
                        goto exit;
        } else if (rebuild_mode) {
@@ -1320,10 +1398,10 @@ int main(int argc, char **argv)
                        goto exit;
                }
 
-               err = erofs_rebuild_load_trees(root);
+               err = erofs_mkfs_rebuild_load_trees(root);
                if (err)
                        goto exit;
-               err = erofs_rebuild_dump_tree(root, false);
+               err = erofs_rebuild_dump_tree(root, incremental_mode);
                if (err)
                        goto exit;
        } else {