From: Gao Xiang Date: Tue, 18 Jun 2024 08:24:09 +0000 (+0800) Subject: erofs-utils: introduce incremental builds X-Git-Tag: v1.8~43 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f64d9d02576be4c9a755a6e6388dd691c16f3a24;p=platform%2Fupstream%2Ferofs-utils.git erofs-utils: introduce incremental builds This introduces incremental build support for mkfs, where new on-disk (meta)data will be appended in a log-structured manner, except for the root inode (due to current on-disk limitations), as illustrated below: ___________________________________________ | base | delta 0 | delta 1 | .. | delta n-1 | |______|_________|_________|____|___________| ---> image/data growth direction Signed-off-by: Gao Xiang Link: https://lore.kernel.org/r/20240618082414.47876-4-hsiangkao@linux.alibaba.com --- diff --git a/include/erofs/inode.h b/include/erofs/inode.h index 0fc9b80..2af8e6c 100644 --- a/include/erofs/inode.h +++ b/include/erofs/inode.h @@ -34,6 +34,7 @@ erofs_nid_t erofs_lookupnid(struct erofs_inode *inode); int erofs_iflush(struct erofs_inode *inode); struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent, const char *name); +bool erofs_dentry_is_wht(struct erofs_sb_info *sbi, struct erofs_dentry *d); int erofs_rebuild_dump_tree(struct erofs_inode *dir, bool incremental); int erofs_init_empty_dir(struct erofs_inode *dir); int __erofs_fill_inode(struct erofs_inode *inode, struct stat *st, diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 4917908..4bfd128 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -303,13 +303,13 @@ static inline struct erofs_inode *erofs_parent_inode(struct erofs_inode *inode) struct erofs_dentry { struct list_head d_child; /* child of parent list */ - - unsigned int type; - char name[EROFS_NAME_LEN]; union { struct erofs_inode *inode; erofs_nid_t nid; }; + char name[EROFS_NAME_LEN]; + u8 type; + bool validnid; }; static inline bool is_dot_dotdot_len(const char *name, unsigned int len) diff --git a/lib/inode.c b/lib/inode.c index a4f61ab..321b2f5 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -165,6 +165,7 @@ struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent, d->name[EROFS_NAME_LEN - 1] = '\0'; d->inode = NULL; d->type = EROFS_FT_UNKNOWN; + d->validnid = false; list_add_tail(&d->d_child, &parent->i_subdirs); return d; } @@ -330,7 +331,10 @@ static void erofs_d_invalidate(struct erofs_dentry *d) { struct erofs_inode *const inode = d->inode; + if (d->validnid) + return; d->nid = erofs_lookupnid(inode); + d->validnid = true; erofs_iput(inode); } @@ -666,6 +670,9 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode) if (inode->extent_isize) inodesize = roundup(inodesize, 8) + inode->extent_isize; + if (inode->datalayout == EROFS_INODE_FLAT_PLAIN) + goto noinline; + /* TODO: tailpacking inline of chunk-based format isn't finalized */ if (inode->datalayout == EROFS_INODE_CHUNK_BASED) goto noinline; @@ -761,6 +768,7 @@ static int erofs_write_tail_end(struct erofs_inode *inode) if (!inode->idata_size) goto out; + DBG_BUGON(!inode->idata); /* have enough room to inline data */ if (inode->bh_inline) { ibh = inode->bh_inline; @@ -1378,23 +1386,49 @@ err_closedir: return ret; } -static int erofs_rebuild_handle_directory(struct erofs_inode *dir) +int erofs_rebuild_load_basedir(struct erofs_inode *dir); + +bool erofs_dentry_is_wht(struct erofs_sb_info *sbi, struct erofs_dentry *d) { + if (!d->validnid) + return erofs_inode_is_whiteout(d->inode); + if (d->type == EROFS_FT_CHRDEV) { + struct erofs_inode ei = { .sbi = sbi, .nid = d->nid }; + int ret; + + ret = erofs_read_inode_from_disk(&ei); + if (ret) { + erofs_err("failed to check DT_WHT: %s", + erofs_strerror(ret)); + DBG_BUGON(1); + return false; + } + return erofs_inode_is_whiteout(&ei); + } + return false; +} + +static int erofs_rebuild_handle_directory(struct erofs_inode *dir, + bool incremental) +{ + struct erofs_sb_info *sbi = dir->sbi; struct erofs_dentry *d, *n; unsigned int nr_subdirs, i_nlink; + bool delwht = cfg.c_ovlfs_strip && dir->whiteouts; int ret; nr_subdirs = 0; i_nlink = 0; + list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) { - if (cfg.c_ovlfs_strip && erofs_inode_is_whiteout(d->inode)) { + if (delwht && erofs_dentry_is_wht(sbi, d)) { erofs_dbg("remove whiteout %s", d->inode->i_srcpath); list_del(&d->d_child); erofs_d_invalidate(d); free(d); continue; } - i_nlink += S_ISDIR(d->inode->i_mode); + i_nlink += (d->type == EROFS_FT_DIR); ++nr_subdirs; } @@ -1404,6 +1438,9 @@ static int erofs_rebuild_handle_directory(struct erofs_inode *dir) if (ret) return ret; + if (IS_ROOT(dir) && incremental) + dir->datalayout = EROFS_INODE_FLAT_PLAIN; + /* * if there're too many subdirs as compact form, set nlink=1 * rather than upgrade to use extented form instead. @@ -1414,7 +1451,7 @@ static int erofs_rebuild_handle_directory(struct erofs_inode *dir) else dir->i_nlink = i_nlink; - return erofs_mkfs_go(dir->sbi, EROFS_MKFS_JOB_DIR, &dir, sizeof(dir)); + return erofs_mkfs_go(sbi, EROFS_MKFS_JOB_DIR, &dir, sizeof(dir)); } static int erofs_mkfs_handle_inode(struct erofs_inode *inode) @@ -1461,7 +1498,8 @@ static int erofs_mkfs_handle_inode(struct erofs_inode *inode) return ret; } -static int erofs_rebuild_handle_inode(struct erofs_inode *inode) +static int erofs_rebuild_handle_inode(struct erofs_inode *inode, + bool incremental) { char *trimmed; int ret; @@ -1482,6 +1520,13 @@ static int erofs_rebuild_handle_inode(struct erofs_inode *inode) inode->inode_isize = sizeof(struct erofs_inode_compact); } + if (incremental && S_ISDIR(inode->i_mode) && + inode->dev == inode->sbi->dev && !inode->opaque) { + ret = erofs_rebuild_load_basedir(inode); + if (ret) + return ret; + } + /* strip all unnecessary overlayfs xattrs when ovlfs_strip is enabled */ if (cfg.c_ovlfs_strip) erofs_clear_opaque_xattr(inode); @@ -1513,7 +1558,7 @@ static int erofs_rebuild_handle_inode(struct erofs_inode *inode) ret = erofs_mkfs_go(inode->sbi, EROFS_MKFS_JOB_NDIR, &ctx, sizeof(ctx)); } else { - ret = erofs_rebuild_handle_directory(inode); + ret = erofs_rebuild_handle_directory(inode, incremental); } erofs_info("file %s dumped (mode %05o)", erofs_fspath(inode->i_srcpath), inode->i_mode); @@ -1540,9 +1585,16 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, erofs_mark_parent_inode(root, root); /* rootdir mark */ root->next_dirwrite = NULL; + /* update dev/i_ino[1] to keep track of the base image */ + if (incremental) { + root->dev = root->sbi->dev; + root->i_ino[1] = sbi->root_nid; + list_del(&root->i_hash); + erofs_insert_ihash(root); + } err = !rebuild ? erofs_mkfs_handle_inode(root) : - erofs_rebuild_handle_inode(root); + erofs_rebuild_handle_inode(root, incremental); if (err) return err; @@ -1564,7 +1616,7 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, list_for_each_entry(d, &dir->i_subdirs, d_child) { struct erofs_inode *inode = d->inode; - if (is_dot_dotdot(d->name)) + if (is_dot_dotdot(d->name) || d->validnid) continue; if (!erofs_inode_visited(inode)) { @@ -1575,7 +1627,8 @@ static int erofs_mkfs_dump_tree(struct erofs_inode *root, bool rebuild, if (!rebuild) err = erofs_mkfs_handle_inode(inode); else - err = erofs_rebuild_handle_inode(inode); + err = erofs_rebuild_handle_inode(inode, + incremental); if (err) break; if (S_ISDIR(inode->i_mode)) { @@ -1773,7 +1826,7 @@ int erofs_fixup_root_inode(struct erofs_inode *root) char *ibuf; int err; - if (sbi->root_nid == root->nid) + if (sbi->root_nid == root->nid) /* for most mkfs cases */ return 0; if (root->nid <= 0xffff) { diff --git a/lib/rebuild.c b/lib/rebuild.c index b9bced2..6f2e301 100644 --- a/lib/rebuild.c +++ b/lib/rebuild.c @@ -235,18 +235,18 @@ static int erofs_rebuild_fill_inode(struct erofs_inode *inode) } /* - * @parent: parent directory in inode tree - * @ctx.dir: parent directory when itering erofs_iterate_dir() + * @mergedir: parent directory in the merged tree + * @ctx.dir: parent directory when itering erofs_iterate_dir() */ struct erofs_rebuild_dir_context { struct erofs_dir_context ctx; - struct erofs_inode *parent; + struct erofs_inode *mergedir; }; static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) { struct erofs_rebuild_dir_context *rctx = (void *)ctx; - struct erofs_inode *parent = rctx->parent; + struct erofs_inode *mergedir = rctx->mergedir; struct erofs_inode *dir = ctx->dir; struct erofs_inode *inode, *candidate; struct erofs_inode src; @@ -258,15 +258,15 @@ static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) if (ctx->dot_dotdot) return 0; - ret = asprintf(&path, "%s/%.*s", rctx->parent->i_srcpath, + ret = asprintf(&path, "%s/%.*s", rctx->mergedir->i_srcpath, ctx->de_namelen, ctx->dname); if (ret < 0) return ret; erofs_dbg("parsing %s", path); - dname = path + strlen(parent->i_srcpath) + 1; + dname = path + strlen(mergedir->i_srcpath) + 1; - d = erofs_rebuild_get_dentry(parent, dname, false, + d = erofs_rebuild_get_dentry(mergedir, dname, false, &dumb, &dumb, false); if (IS_ERR(d)) { ret = PTR_ERR(d); @@ -290,12 +290,12 @@ static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) ret = erofs_read_inode_from_disk(&src); if (ret || !S_ISDIR(src.i_mode)) goto out; - parent = d->inode; + mergedir = d->inode; inode = dir = &src; } else { u64 nid; - DBG_BUGON(parent != d->inode); + DBG_BUGON(mergedir != d->inode); inode = erofs_new_inode(dir->sbi); if (IS_ERR(inode)) { ret = PTR_ERR(inode); @@ -347,7 +347,7 @@ static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) } erofs_insert_ihash(inode); - parent = dir = inode; + mergedir = dir = inode; } d->inode = inode; @@ -357,7 +357,7 @@ static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) if (S_ISDIR(inode->i_mode)) { struct erofs_rebuild_dir_context nctx = *rctx; - nctx.parent = parent; + nctx.mergedir = mergedir; nctx.ctx.dir = dir; ret = erofs_iterate_dir(&nctx.ctx, false); if (ret) @@ -402,9 +402,74 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi) ctx = (struct erofs_rebuild_dir_context) { .ctx.dir = &inode, .ctx.cb = erofs_rebuild_dirent_iter, - .parent = root, + .mergedir = root, }; ret = erofs_iterate_dir(&ctx.ctx, false); free(inode.i_srcpath); return ret; } + +static int erofs_rebuild_basedir_dirent_iter(struct erofs_dir_context *ctx) +{ + struct erofs_rebuild_dir_context *rctx = (void *)ctx; + struct erofs_inode *dir = ctx->dir; + struct erofs_inode *mergedir = rctx->mergedir; + struct erofs_dentry *d; + char *dname; + bool dumb; + int ret; + + if (ctx->dot_dotdot) + return 0; + + dname = strndup(ctx->dname, ctx->de_namelen); + if (!dname) + return -ENOMEM; + d = erofs_rebuild_get_dentry(mergedir, dname, false, + &dumb, &dumb, false); + if (IS_ERR(d)) { + ret = PTR_ERR(d); + goto out; + } + + if (d->type == EROFS_FT_UNKNOWN) { + d->nid = ctx->de_nid; + d->type = ctx->de_ftype; + d->validnid = true; + if (!mergedir->whiteouts && erofs_dentry_is_wht(dir->sbi, d)) + mergedir->whiteouts = true; + } else { + struct erofs_inode *inode = d->inode; + + list_del(&inode->i_hash); + inode->dev = dir->sbi->dev; + inode->i_ino[1] = ctx->de_nid; + erofs_insert_ihash(inode); + } + ret = 0; +out: + free(dname); + return ret; +} + +int erofs_rebuild_load_basedir(struct erofs_inode *dir) +{ + struct erofs_inode fakeinode = { + .sbi = dir->sbi, + .nid = dir->i_ino[1], + }; + struct erofs_rebuild_dir_context ctx; + int ret; + + ret = erofs_read_inode_from_disk(&fakeinode); + if (ret) { + erofs_err("failed to read inode @ %llu", fakeinode.nid); + return ret; + } + ctx = (struct erofs_rebuild_dir_context) { + .ctx.dir = &fakeinode, + .ctx.cb = erofs_rebuild_basedir_dirent_iter, + .mergedir = dir, + }; + return erofs_iterate_dir(&ctx.ctx, false); +}