2 * Copyright (C) 2017 SUSE. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program.
17 #include "kerncompat.h"
18 #include "androidcompat.h"
21 #include <sys/types.h>
22 #include <sys/xattr.h>
23 #include <linux/limits.h>
33 #include "transaction.h"
35 #include "mkfs/rootdir.h"
36 #include "mkfs/common.h"
37 #include "send-utils.h"
39 static u32 fs_block_size;
41 static u64 index_cnt = 2;
44 * Size estimate will be done using the following data:
46 * Since we will later shrink the fs, over-estimate is completely fine here
47 * as long as our estimate ensures we can populate the image without ENOSPC.
48 * So we only record how many inodes there are, and account the maximum
49 * space for each inode.
51 * 2) Data space for each (regular) inode
52 * To estimate data chunk size.
53 * Don't care if it can fit as an inline extent.
54 * Always round them up to sectorsize.
56 static u64 ftw_meta_nr_inode;
57 static u64 ftw_data_size;
59 static int add_directory_items(struct btrfs_trans_handle *trans,
60 struct btrfs_root *root, u64 objectid,
61 ino_t parent_inum, const char *name,
62 struct stat *st, int *dir_index_cnt)
66 struct btrfs_key location;
69 name_len = strlen(name);
71 location.objectid = objectid;
73 location.type = BTRFS_INODE_ITEM_KEY;
75 if (S_ISDIR(st->st_mode))
76 filetype = BTRFS_FT_DIR;
77 if (S_ISREG(st->st_mode))
78 filetype = BTRFS_FT_REG_FILE;
79 if (S_ISLNK(st->st_mode))
80 filetype = BTRFS_FT_SYMLINK;
81 if (S_ISSOCK(st->st_mode))
82 filetype = BTRFS_FT_SOCK;
83 if (S_ISCHR(st->st_mode))
84 filetype = BTRFS_FT_CHRDEV;
85 if (S_ISBLK(st->st_mode))
86 filetype = BTRFS_FT_BLKDEV;
87 if (S_ISFIFO(st->st_mode))
88 filetype = BTRFS_FT_FIFO;
90 ret = btrfs_insert_dir_item(trans, root, name, name_len,
91 parent_inum, &location,
95 ret = btrfs_insert_inode_ref(trans, root, name, name_len,
96 objectid, parent_inum, index_cnt);
97 *dir_index_cnt = index_cnt;
103 static int fill_inode_item(struct btrfs_trans_handle *trans,
104 struct btrfs_root *root,
105 struct btrfs_inode_item *dst, struct stat *src)
108 u64 sectorsize = root->fs_info->sectorsize;
111 * btrfs_inode_item has some reserved fields
112 * and represents on-disk inode entry, so
113 * zero everything to prevent information leak
115 memset(dst, 0, sizeof(*dst));
117 btrfs_set_stack_inode_generation(dst, trans->transid);
118 btrfs_set_stack_inode_size(dst, src->st_size);
119 btrfs_set_stack_inode_nbytes(dst, 0);
120 btrfs_set_stack_inode_block_group(dst, 0);
121 btrfs_set_stack_inode_nlink(dst, src->st_nlink);
122 btrfs_set_stack_inode_uid(dst, src->st_uid);
123 btrfs_set_stack_inode_gid(dst, src->st_gid);
124 btrfs_set_stack_inode_mode(dst, src->st_mode);
125 btrfs_set_stack_inode_rdev(dst, 0);
126 btrfs_set_stack_inode_flags(dst, 0);
127 btrfs_set_stack_timespec_sec(&dst->atime, src->st_atime);
128 btrfs_set_stack_timespec_nsec(&dst->atime, 0);
129 btrfs_set_stack_timespec_sec(&dst->ctime, src->st_ctime);
130 btrfs_set_stack_timespec_nsec(&dst->ctime, 0);
131 btrfs_set_stack_timespec_sec(&dst->mtime, src->st_mtime);
132 btrfs_set_stack_timespec_nsec(&dst->mtime, 0);
133 btrfs_set_stack_timespec_sec(&dst->otime, 0);
134 btrfs_set_stack_timespec_nsec(&dst->otime, 0);
136 if (S_ISDIR(src->st_mode)) {
137 btrfs_set_stack_inode_size(dst, 0);
138 btrfs_set_stack_inode_nlink(dst, 1);
140 if (S_ISREG(src->st_mode)) {
141 btrfs_set_stack_inode_size(dst, (u64)src->st_size);
142 if (src->st_size <= BTRFS_MAX_INLINE_DATA_SIZE(root))
143 btrfs_set_stack_inode_nbytes(dst, src->st_size);
145 blocks = src->st_size / sectorsize;
146 if (src->st_size % sectorsize)
148 blocks *= sectorsize;
149 btrfs_set_stack_inode_nbytes(dst, blocks);
152 if (S_ISLNK(src->st_mode))
153 btrfs_set_stack_inode_nbytes(dst, src->st_size + 1);
158 static int directory_select(const struct direct *entry)
160 if (entry->d_name[0] == '.' &&
161 (entry->d_name[1] == 0 ||
162 (entry->d_name[1] == '.' && entry->d_name[2] == 0)))
167 static void free_namelist(struct direct **files, int count)
174 for (i = 0; i < count; ++i)
179 static u64 calculate_dir_inode_size(const char *dirname)
182 struct direct **files, *cur_file;
183 u64 dir_inode_size = 0;
185 count = scandir(dirname, &files, directory_select, NULL);
187 for (i = 0; i < count; i++) {
189 dir_inode_size += strlen(cur_file->d_name);
192 free_namelist(files, count);
195 return dir_inode_size;
198 static int add_inode_items(struct btrfs_trans_handle *trans,
199 struct btrfs_root *root,
200 struct stat *st, const char *name,
202 struct btrfs_inode_item *inode_ret)
205 struct btrfs_inode_item btrfs_inode;
209 fill_inode_item(trans, root, &btrfs_inode, st);
210 objectid = self_objectid;
212 if (S_ISDIR(st->st_mode)) {
213 inode_size = calculate_dir_inode_size(name);
214 btrfs_set_stack_inode_size(&btrfs_inode, inode_size);
217 ret = btrfs_insert_inode(trans, root, objectid, &btrfs_inode);
219 *inode_ret = btrfs_inode;
223 static int add_xattr_item(struct btrfs_trans_handle *trans,
224 struct btrfs_root *root, u64 objectid,
225 const char *file_name)
229 char xattr_list[XATTR_LIST_MAX];
231 char cur_value[XATTR_SIZE_MAX];
232 char delimiter = '\0';
233 char *next_location = xattr_list;
235 ret = llistxattr(file_name, xattr_list, XATTR_LIST_MAX);
237 if (errno == ENOTSUP)
239 error("getting a list of xattr failed for %s: %s", file_name,
246 cur_name = strtok(xattr_list, &delimiter);
247 while (cur_name != NULL) {
248 cur_name_len = strlen(cur_name);
249 next_location += cur_name_len + 1;
251 ret = getxattr(file_name, cur_name, cur_value, XATTR_SIZE_MAX);
253 if (errno == ENOTSUP)
255 error("gettig a xattr value failed for %s attr %s: %s",
256 file_name, cur_name, strerror(errno));
260 ret = btrfs_insert_xattr_item(trans, root, cur_name,
261 cur_name_len, cur_value,
264 error("inserting a xattr item failed for %s: %s",
265 file_name, strerror(-ret));
268 cur_name = strtok(next_location, &delimiter);
274 static int add_symbolic_link(struct btrfs_trans_handle *trans,
275 struct btrfs_root *root,
276 u64 objectid, const char *path_name)
281 ret = readlink(path_name, buf, sizeof(buf));
283 error("readlink failed for %s: %s", path_name, strerror(errno));
286 if (ret >= sizeof(buf)) {
287 error("symlink too long for %s", path_name);
292 buf[ret] = '\0'; /* readlink does not do it for us */
293 ret = btrfs_insert_inline_extent(trans, root, objectid, 0,
299 static int add_file_items(struct btrfs_trans_handle *trans,
300 struct btrfs_root *root,
301 struct btrfs_inode_item *btrfs_inode, u64 objectid,
302 struct stat *st, const char *path_name)
307 struct btrfs_key key;
309 u32 sectorsize = root->fs_info->sectorsize;
314 struct extent_buffer *eb = NULL;
317 if (st->st_size == 0)
320 fd = open(path_name, O_RDONLY);
322 error("cannot open %s: %s", path_name, strerror(errno));
326 blocks = st->st_size / sectorsize;
327 if (st->st_size % sectorsize)
330 if (st->st_size <= BTRFS_MAX_INLINE_DATA_SIZE(root)) {
331 char *buffer = malloc(st->st_size);
338 ret_read = pread64(fd, buffer, st->st_size, bytes_read);
339 if (ret_read == -1) {
340 error("cannot read %s at offset %llu length %llu: %s",
341 path_name, (unsigned long long)bytes_read,
342 (unsigned long long)st->st_size,
348 ret = btrfs_insert_inline_extent(trans, root, objectid, 0,
349 buffer, st->st_size);
354 /* round up our st_size to the FS blocksize */
355 total_bytes = (u64)blocks * sectorsize;
358 * do our IO in extent buffers so it can work
359 * against any raid type
361 eb = calloc(1, sizeof(*eb) + sectorsize);
370 * keep our extent size at 1MB max, this makes it easier to work inside
371 * the tiny block groups created during mkfs
373 cur_bytes = min(total_bytes, (u64)SZ_1M);
374 ret = btrfs_reserve_extent(trans, root, cur_bytes, 0, 0, (u64)-1,
379 first_block = key.objectid;
382 while (bytes_read < cur_bytes) {
384 memset(eb->data, 0, sectorsize);
386 ret_read = pread64(fd, eb->data, sectorsize, file_pos +
388 if (ret_read == -1) {
389 error("cannot read %s at offset %llu length %llu: %s",
391 (unsigned long long)file_pos + bytes_read,
392 (unsigned long long)sectorsize,
397 eb->start = first_block + bytes_read;
398 eb->len = sectorsize;
401 * we're doing the csum before we record the extent, but
404 ret = btrfs_csum_file_block(trans, root->fs_info->csum_root,
405 first_block + bytes_read + sectorsize,
406 first_block + bytes_read,
407 eb->data, sectorsize);
411 ret = write_and_map_eb(root->fs_info, eb);
413 error("failed to write %s", path_name);
417 bytes_read += sectorsize;
421 ret = btrfs_record_file_extent(trans, root, objectid,
422 btrfs_inode, file_pos, first_block, cur_bytes);
428 file_pos += cur_bytes;
429 total_bytes -= cur_bytes;
440 static int traverse_directory(struct btrfs_trans_handle *trans,
441 struct btrfs_root *root, const char *dir_name,
442 struct directory_name_entry *dir_head)
446 struct btrfs_inode_item cur_inode;
447 struct btrfs_inode_item *inode_item;
448 int count, i, dir_index_cnt;
449 struct direct **files;
451 struct directory_name_entry *dir_entry, *parent_dir_entry;
452 struct direct *cur_file;
453 ino_t parent_inum, cur_inum;
454 ino_t highest_inum = 0;
455 const char *parent_dir_name;
456 char real_path[PATH_MAX];
457 struct btrfs_path path;
458 struct extent_buffer *leaf;
459 struct btrfs_key root_dir_key;
460 u64 root_dir_inode_size = 0;
462 /* Add list for source directory */
463 dir_entry = malloc(sizeof(struct directory_name_entry));
466 dir_entry->dir_name = dir_name;
467 dir_entry->path = realpath(dir_name, real_path);
468 if (!dir_entry->path) {
469 error("realpath failed for %s: %s", dir_name, strerror(errno));
474 parent_inum = highest_inum + BTRFS_FIRST_FREE_OBJECTID;
475 dir_entry->inum = parent_inum;
476 list_add_tail(&dir_entry->list, &dir_head->list);
478 btrfs_init_path(&path);
480 root_dir_key.objectid = btrfs_root_dirid(&root->root_item);
481 root_dir_key.offset = 0;
482 root_dir_key.type = BTRFS_INODE_ITEM_KEY;
483 ret = btrfs_lookup_inode(trans, root, &path, &root_dir_key, 1);
485 error("failed to lookup root dir: %d", ret);
489 leaf = path.nodes[0];
490 inode_item = btrfs_item_ptr(leaf, path.slots[0],
491 struct btrfs_inode_item);
493 root_dir_inode_size = calculate_dir_inode_size(dir_name);
494 btrfs_set_inode_size(leaf, inode_item, root_dir_inode_size);
495 btrfs_mark_buffer_dirty(leaf);
497 btrfs_release_path(&path);
500 parent_dir_entry = list_entry(dir_head->list.next,
501 struct directory_name_entry,
503 list_del(&parent_dir_entry->list);
505 parent_inum = parent_dir_entry->inum;
506 parent_dir_name = parent_dir_entry->dir_name;
507 if (chdir(parent_dir_entry->path)) {
508 error("chdir failed for %s: %s",
509 parent_dir_name, strerror(errno));
514 count = scandir(parent_dir_entry->path, &files,
515 directory_select, NULL);
517 error("scandir failed for %s: %s",
518 parent_dir_name, strerror(errno));
523 for (i = 0; i < count; i++) {
526 if (lstat(cur_file->d_name, &st) == -1) {
527 error("lstat failed for %s: %s",
528 cur_file->d_name, strerror(errno));
533 cur_inum = st.st_ino;
534 ret = add_directory_items(trans, root,
535 cur_inum, parent_inum,
537 &st, &dir_index_cnt);
539 error("unable to add directory items for %s: %d",
540 cur_file->d_name, ret);
544 ret = add_inode_items(trans, root, &st,
545 cur_file->d_name, cur_inum,
547 if (ret == -EEXIST) {
548 if (st.st_nlink <= 1) {
550 "item %s already exists but has wrong st_nlink %lu <= 1",
552 (unsigned long)st.st_nlink);
558 error("unable to add inode items for %s: %d",
559 cur_file->d_name, ret);
563 ret = add_xattr_item(trans, root,
564 cur_inum, cur_file->d_name);
566 error("unable to add xattr items for %s: %d",
567 cur_file->d_name, ret);
572 if (S_ISDIR(st.st_mode)) {
575 dir_entry = malloc(sizeof(*dir_entry));
580 dir_entry->dir_name = cur_file->d_name;
581 if (path_cat_out(tmp, parent_dir_entry->path,
583 error("invalid path: %s/%s",
584 parent_dir_entry->path,
589 dir_entry->path = strdup(tmp);
590 if (!dir_entry->path) {
591 error("not enough memory to store path");
595 dir_entry->inum = cur_inum;
596 list_add_tail(&dir_entry->list,
598 } else if (S_ISREG(st.st_mode)) {
599 ret = add_file_items(trans, root, &cur_inode,
603 error("unable to add file items for %s: %d",
604 cur_file->d_name, ret);
607 } else if (S_ISLNK(st.st_mode)) {
608 ret = add_symbolic_link(trans, root,
609 cur_inum, cur_file->d_name);
611 error("unable to add symlink for %s: %d",
612 cur_file->d_name, ret);
618 free_namelist(files, count);
619 free(parent_dir_entry);
623 } while (!list_empty(&dir_head->list));
628 free_namelist(files, count);
630 free(parent_dir_entry);
637 int btrfs_mkfs_fill_dir(const char *source_dir, struct btrfs_root *root,
641 struct btrfs_trans_handle *trans;
643 struct directory_name_entry dir_head;
644 struct directory_name_entry *dir_entry = NULL;
646 ret = lstat(source_dir, &root_st);
648 error("unable to lstat %s: %s", source_dir, strerror(errno));
653 INIT_LIST_HEAD(&dir_head.list);
655 trans = btrfs_start_transaction(root, 1);
656 BUG_ON(IS_ERR(trans));
657 ret = traverse_directory(trans, root, source_dir, &dir_head);
659 error("unable to traverse directory %s: %d", source_dir, ret);
662 ret = btrfs_commit_transaction(trans, root);
664 error("transaction commit failed: %d", ret);
669 printf("Making image is completed.\n");
673 * Since we don't have btrfs_abort_transaction() yet, uncommitted trans
674 * will trigger a BUG_ON().
676 * However before mkfs is fully finished, the magic number is invalid,
677 * so even we commit transaction here, the fs still can't be mounted.
679 * To do a graceful error out, here we commit transaction as a
681 * Since we have already hit some problem, the return value doesn't
684 btrfs_commit_transaction(trans, root);
685 while (!list_empty(&dir_head.list)) {
686 dir_entry = list_entry(dir_head.list.next,
687 struct directory_name_entry, list);
688 list_del(&dir_entry->list);
695 static int ftw_add_entry_size(const char *fpath, const struct stat *st,
699 * Failed to read the directory, mostly due to EPERM. Abort ASAP, so
700 * we don't need to populate the fs.
702 if (type == FTW_DNR || type == FTW_NS)
705 if (S_ISREG(st->st_mode))
706 ftw_data_size += round_up(st->st_size, fs_block_size);
712 u64 btrfs_mkfs_size_dir(const char *dir_name, u32 sectorsize, u64 min_dev_size,
713 u64 meta_profile, u64 data_profile)
718 u64 meta_size = 0; /* Based on @ftw_meta_nr_inode */
719 u64 meta_chunk_size = 0; /* Based on @meta_size */
720 u64 data_chunk_size = 0; /* Based on @ftw_data_size */
722 u64 meta_threshold = SZ_8M;
723 u64 data_threshold = SZ_8M;
725 float data_multipler = 1;
726 float meta_multipler = 1;
728 fs_block_size = sectorsize;
730 ftw_meta_nr_inode = 0;
731 ret = ftw(dir_name, ftw_add_entry_size, 10);
733 error("ftw subdir walk of %s failed: %s", dir_name,
740 * Maximum metadata useage for every inode, which will be PATH_MAX
741 * for the following items:
746 * Plus possible inline extent size, which is sectorsize.
748 * And finally, allow metadata usage to increase with data size.
749 * Follow the old kernel 8:1 data:meta ratio.
750 * This is especially important for --rootdir, as the file extent size
751 * upper limit is 1M, instead of 128M in kernel.
752 * This can bump meta usage easily.
754 meta_size = ftw_meta_nr_inode * (PATH_MAX * 3 + sectorsize) +
757 /* Minimal chunk size from btrfs_alloc_chunk(). */
758 if (meta_profile & BTRFS_BLOCK_GROUP_DUP) {
759 meta_threshold = SZ_32M;
762 if (data_profile & BTRFS_BLOCK_GROUP_DUP) {
763 data_threshold = SZ_64M;
768 * Only when the usage is larger than the minimal chunk size (threshold)
769 * we need to allocate new chunk, or the initial chunk in the image is
772 if (meta_size > meta_threshold)
773 meta_chunk_size = (round_up(meta_size, meta_threshold) -
774 meta_threshold) * meta_multipler;
775 if (ftw_data_size > data_threshold)
776 data_chunk_size = (round_up(ftw_data_size, data_threshold) -
777 data_threshold) * data_multipler;
779 total_size = data_chunk_size + meta_chunk_size + min_dev_size;
784 * Get the end position of the last device extent for given @devid;
785 * @size_ret is exclsuive (means it should be aligned to sectorsize)
787 static int get_device_extent_end(struct btrfs_fs_info *fs_info,
788 u64 devid, u64 *size_ret)
790 struct btrfs_root *dev_root = fs_info->dev_root;
791 struct btrfs_key key;
792 struct btrfs_path path;
793 struct btrfs_dev_extent *de;
796 key.objectid = devid;
797 key.type = BTRFS_DEV_EXTENT_KEY;
798 key.offset = (u64)-1;
800 btrfs_init_path(&path);
801 ret = btrfs_search_slot(NULL, dev_root, &key, &path, 0, 0);
802 /* Not really possible */
805 ret = btrfs_previous_item(dev_root, &path, devid, BTRFS_DEV_EXTENT_KEY);
809 /* No dev_extent at all, not really possible for rootdir case */
816 btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]);
817 de = btrfs_item_ptr(path.nodes[0], path.slots[0],
818 struct btrfs_dev_extent);
819 *size_ret = key.offset + btrfs_dev_extent_length(path.nodes[0], de);
821 btrfs_release_path(&path);