fs/btrfs/disk-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/blkdev.h>
   8 #include <linux/radix-tree.h>
   9 #include <linux/writeback.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/kthread.h>
  12 #include <linux/slab.h>
  13 #include <linux/migrate.h>
  14 #include <linux/ratelimit.h>
  15 #include <linux/uuid.h>
  16 #include <linux/semaphore.h>
  17 #include <linux/error-injection.h>
  18 #include <linux/crc32c.h>
  19 #include <linux/sched/mm.h>
  20 #include <asm/unaligned.h>
  21 #include <crypto/hash.h>
  22 #include "ctree.h"
  23 #include "disk-io.h"
  24 #include "transaction.h"
  25 #include "btrfs_inode.h"
  26 #include "volumes.h"
  27 #include "print-tree.h"
  28 #include "locking.h"
  29 #include "tree-log.h"
  30 #include "free-space-cache.h"
  31 #include "free-space-tree.h"
  32 #include "check-integrity.h"
  33 #include "rcu-string.h"
  34 #include "dev-replace.h"
  35 #include "raid56.h"
  36 #include "sysfs.h"
  37 #include "qgroup.h"
  38 #include "compression.h"
  39 #include "tree-checker.h"
  40 #include "ref-verify.h"
  41 #include "block-group.h"
  42 #include "discard.h"
  43 #include "space-info.h"
  44 #include "zoned.h"
  45 #include "subpage.h"
  46
  47 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
  48                                  BTRFS_HEADER_FLAG_RELOC |\
  49                                  BTRFS_SUPER_FLAG_ERROR |\
  50                                  BTRFS_SUPER_FLAG_SEEDING |\
  51                                  BTRFS_SUPER_FLAG_METADUMP |\
  52                                  BTRFS_SUPER_FLAG_METADUMP_V2)
  53
  54 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
  55 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
  56                                       struct btrfs_fs_info *fs_info);
  57 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
  58 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
  59                                         struct extent_io_tree *dirty_pages,
  60                                         int mark);
  61 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
  62                                        struct extent_io_tree *pinned_extents);
  63 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
  64 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
  65
  66 static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
  67 {
  68         if (fs_info->csum_shash)
  69                 crypto_free_shash(fs_info->csum_shash);
  70 }
  71
  72 /*
  73  * async submit bios are used to offload expensive checksumming
  74  * onto the worker threads.  They checksum file and metadata bios
  75  * just before they are sent down the IO stack.
  76  */
  77 struct async_submit_bio {
  78         struct inode *inode;
  79         struct bio *bio;
  80         extent_submit_bio_start_t *submit_bio_start;
  81         int mirror_num;
  82
  83         /* Optional parameter for submit_bio_start used by direct io */
  84         u64 dio_file_offset;
  85         struct btrfs_work work;
  86         blk_status_t status;
  87 };
  88
  89 /*
  90  * Compute the csum of a btree block and store the result to provided buffer.
  91  */
  92 static void csum_tree_block(struct extent_buffer *buf, u8 *result)
  93 {
  94         struct btrfs_fs_info *fs_info = buf->fs_info;
  95         const int num_pages = num_extent_pages(buf);
  96         const int first_page_part = min_t(u32, PAGE_SIZE, fs_info->nodesize);
  97         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
  98         char *kaddr;
  99         int i;
 100
 101         shash->tfm = fs_info->csum_shash;
 102         crypto_shash_init(shash);
 103         kaddr = page_address(buf->pages[0]) + offset_in_page(buf->start);
 104         crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
 105                             first_page_part - BTRFS_CSUM_SIZE);
 106
 107         for (i = 1; i < num_pages; i++) {
 108                 kaddr = page_address(buf->pages[i]);
 109                 crypto_shash_update(shash, kaddr, PAGE_SIZE);
 110         }
 111         memset(result, 0, BTRFS_CSUM_SIZE);
 112         crypto_shash_final(shash, result);
 113 }
 114
 115 /*
 116  * we can't consider a given block up to date unless the transid of the
 117  * block matches the transid in the parent node's pointer.  This is how we
 118  * detect blocks that either didn't get written at all or got written
 119  * in the wrong place.
 120  */
 121 static int verify_parent_transid(struct extent_io_tree *io_tree,
 122                                  struct extent_buffer *eb, u64 parent_transid,
 123                                  int atomic)
 124 {
 125         struct extent_state *cached_state = NULL;
 126         int ret;
 127
 128         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
 129                 return 0;
 130
 131         if (atomic)
 132                 return -EAGAIN;
 133
 134         lock_extent_bits(io_tree, eb->start, eb->start + eb->len - 1,
 135                          &cached_state);
 136         if (extent_buffer_uptodate(eb) &&
 137             btrfs_header_generation(eb) == parent_transid) {
 138                 ret = 0;
 139                 goto out;
 140         }
 141         btrfs_err_rl(eb->fs_info,
 142 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
 143                         eb->start, eb->read_mirror,
 144                         parent_transid, btrfs_header_generation(eb));
 145         ret = 1;
 146         clear_extent_buffer_uptodate(eb);
 147 out:
 148         unlock_extent_cached(io_tree, eb->start, eb->start + eb->len - 1,
 149                              &cached_state);
 150         return ret;
 151 }
 152
 153 static bool btrfs_supported_super_csum(u16 csum_type)
 154 {
 155         switch (csum_type) {
 156         case BTRFS_CSUM_TYPE_CRC32:
 157         case BTRFS_CSUM_TYPE_XXHASH:
 158         case BTRFS_CSUM_TYPE_SHA256:
 159         case BTRFS_CSUM_TYPE_BLAKE2:
 160                 return true;
 161         default:
 162                 return false;
 163         }
 164 }
 165
 166 /*
 167  * Return 0 if the superblock checksum type matches the checksum value of that
 168  * algorithm. Pass the raw disk superblock data.
 169  */
 170 static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 171                                   char *raw_disk_sb)
 172 {
 173         struct btrfs_super_block *disk_sb =
 174                 (struct btrfs_super_block *)raw_disk_sb;
 175         char result[BTRFS_CSUM_SIZE];
 176         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 177
 178         shash->tfm = fs_info->csum_shash;
 179
 180         /*
 181          * The super_block structure does not span the whole
 182          * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
 183          * filled with zeros and is included in the checksum.
 184          */
 185         crypto_shash_digest(shash, raw_disk_sb + BTRFS_CSUM_SIZE,
 186                             BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
 187
 188         if (memcmp(disk_sb->csum, result, fs_info->csum_size))
 189                 return 1;
 190
 191         return 0;
 192 }
 193
 194 int btrfs_verify_level_key(struct extent_buffer *eb, int level,
 195                            struct btrfs_key *first_key, u64 parent_transid)
 196 {
 197         struct btrfs_fs_info *fs_info = eb->fs_info;
 198         int found_level;
 199         struct btrfs_key found_key;
 200         int ret;
 201
 202         found_level = btrfs_header_level(eb);
 203         if (found_level != level) {
 204                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 205                      KERN_ERR "BTRFS: tree level check failed\n");
 206                 btrfs_err(fs_info,
 207 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
 208                           eb->start, level, found_level);
 209                 return -EIO;
 210         }
 211
 212         if (!first_key)
 213                 return 0;
 214
 215         /*
 216          * For live tree block (new tree blocks in current transaction),
 217          * we need proper lock context to avoid race, which is impossible here.
 218          * So we only checks tree blocks which is read from disk, whose
 219          * generation <= fs_info->last_trans_committed.
 220          */
 221         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
 222                 return 0;
 223
 224         /* We have @first_key, so this @eb must have at least one item */
 225         if (btrfs_header_nritems(eb) == 0) {
 226                 btrfs_err(fs_info,
 227                 "invalid tree nritems, bytenr=%llu nritems=0 expect >0",
 228                           eb->start);
 229                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 230                 return -EUCLEAN;
 231         }
 232
 233         if (found_level)
 234                 btrfs_node_key_to_cpu(eb, &found_key, 0);
 235         else
 236                 btrfs_item_key_to_cpu(eb, &found_key, 0);
 237         ret = btrfs_comp_cpu_keys(first_key, &found_key);
 238
 239         if (ret) {
 240                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 241                      KERN_ERR "BTRFS: tree first key check failed\n");
 242                 btrfs_err(fs_info,
 243 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 244                           eb->start, parent_transid, first_key->objectid,
 245                           first_key->type, first_key->offset,
 246                           found_key.objectid, found_key.type,
 247                           found_key.offset);
 248         }
 249         return ret;
 250 }
 251
 252 /*
 253  * helper to read a given tree block, doing retries as required when
 254  * the checksums don't match and we have alternate mirrors to try.
 255  *
 256  * @parent_transid:     expected transid, skip check if 0
 257  * @level:              expected level, mandatory check
 258  * @first_key:          expected key of first slot, skip check if NULL
 259  */
 260 int btrfs_read_extent_buffer(struct extent_buffer *eb,
 261                              u64 parent_transid, int level,
 262                              struct btrfs_key *first_key)
 263 {
 264         struct btrfs_fs_info *fs_info = eb->fs_info;
 265         struct extent_io_tree *io_tree;
 266         int failed = 0;
 267         int ret;
 268         int num_copies = 0;
 269         int mirror_num = 0;
 270         int failed_mirror = 0;
 271
 272         io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
 273         while (1) {
 274                 clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 275                 ret = read_extent_buffer_pages(eb, WAIT_COMPLETE, mirror_num);
 276                 if (!ret) {
 277                         if (verify_parent_transid(io_tree, eb,
 278                                                    parent_transid, 0))
 279                                 ret = -EIO;
 280                         else if (btrfs_verify_level_key(eb, level,
 281                                                 first_key, parent_transid))
 282                                 ret = -EUCLEAN;
 283                         else
 284                                 break;
 285                 }
 286
 287                 num_copies = btrfs_num_copies(fs_info,
 288                                               eb->start, eb->len);
 289                 if (num_copies == 1)
 290                         break;
 291
 292                 if (!failed_mirror) {
 293                         failed = 1;
 294                         failed_mirror = eb->read_mirror;
 295                 }
 296
 297                 mirror_num++;
 298                 if (mirror_num == failed_mirror)
 299                         mirror_num++;
 300
 301                 if (mirror_num > num_copies)
 302                         break;
 303         }
 304
 305         if (failed && !ret && failed_mirror)
 306                 btrfs_repair_eb_io_failure(eb, failed_mirror);
 307
 308         return ret;
 309 }
 310
 311 static int csum_one_extent_buffer(struct extent_buffer *eb)
 312 {
 313         struct btrfs_fs_info *fs_info = eb->fs_info;
 314         u8 result[BTRFS_CSUM_SIZE];
 315         int ret;
 316
 317         ASSERT(memcmp_extent_buffer(eb, fs_info->fs_devices->metadata_uuid,
 318                                     offsetof(struct btrfs_header, fsid),
 319                                     BTRFS_FSID_SIZE) == 0);
 320         csum_tree_block(eb, result);
 321
 322         if (btrfs_header_level(eb))
 323                 ret = btrfs_check_node(eb);
 324         else
 325                 ret = btrfs_check_leaf_full(eb);
 326
 327         if (ret < 0)
 328                 goto error;
 329
 330         /*
 331          * Also check the generation, the eb reached here must be newer than
 332          * last committed. Or something seriously wrong happened.
 333          */
 334         if (unlikely(btrfs_header_generation(eb) <= fs_info->last_trans_committed)) {
 335                 ret = -EUCLEAN;
 336                 btrfs_err(fs_info,
 337                         "block=%llu bad generation, have %llu expect > %llu",
 338                           eb->start, btrfs_header_generation(eb),
 339                           fs_info->last_trans_committed);
 340                 goto error;
 341         }
 342         write_extent_buffer(eb, result, 0, fs_info->csum_size);
 343
 344         return 0;
 345
 346 error:
 347         btrfs_print_tree(eb, 0);
 348         btrfs_err(fs_info, "block=%llu write time tree block corruption detected",
 349                   eb->start);
 350         WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 351         return ret;
 352 }
 353
 354 /* Checksum all dirty extent buffers in one bio_vec */
 355 static int csum_dirty_subpage_buffers(struct btrfs_fs_info *fs_info,
 356                                       struct bio_vec *bvec)
 357 {
 358         struct page *page = bvec->bv_page;
 359         u64 bvec_start = page_offset(page) + bvec->bv_offset;
 360         u64 cur;
 361         int ret = 0;
 362
 363         for (cur = bvec_start; cur < bvec_start + bvec->bv_len;
 364              cur += fs_info->nodesize) {
 365                 struct extent_buffer *eb;
 366                 bool uptodate;
 367
 368                 eb = find_extent_buffer(fs_info, cur);
 369                 uptodate = btrfs_subpage_test_uptodate(fs_info, page, cur,
 370                                                        fs_info->nodesize);
 371
 372                 /* A dirty eb shouldn't disappear from buffer_radix */
 373                 if (WARN_ON(!eb))
 374                         return -EUCLEAN;
 375
 376                 if (WARN_ON(cur != btrfs_header_bytenr(eb))) {
 377                         free_extent_buffer(eb);
 378                         return -EUCLEAN;
 379                 }
 380                 if (WARN_ON(!uptodate)) {
 381                         free_extent_buffer(eb);
 382                         return -EUCLEAN;
 383                 }
 384
 385                 ret = csum_one_extent_buffer(eb);
 386                 free_extent_buffer(eb);
 387                 if (ret < 0)
 388                         return ret;
 389         }
 390         return ret;
 391 }
 392
 393 /*
 394  * Checksum a dirty tree block before IO.  This has extra checks to make sure
 395  * we only fill in the checksum field in the first page of a multi-page block.
 396  * For subpage extent buffers we need bvec to also read the offset in the page.
 397  */
 398 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec)
 399 {
 400         struct page *page = bvec->bv_page;
 401         u64 start = page_offset(page);
 402         u64 found_start;
 403         struct extent_buffer *eb;
 404
 405         if (fs_info->nodesize < PAGE_SIZE)
 406                 return csum_dirty_subpage_buffers(fs_info, bvec);
 407
 408         eb = (struct extent_buffer *)page->private;
 409         if (page != eb->pages[0])
 410                 return 0;
 411
 412         found_start = btrfs_header_bytenr(eb);
 413
 414         if (test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)) {
 415                 WARN_ON(found_start != 0);
 416                 return 0;
 417         }
 418
 419         /*
 420          * Please do not consolidate these warnings into a single if.
 421          * It is useful to know what went wrong.
 422          */
 423         if (WARN_ON(found_start != start))
 424                 return -EUCLEAN;
 425         if (WARN_ON(!PageUptodate(page)))
 426                 return -EUCLEAN;
 427
 428         return csum_one_extent_buffer(eb);
 429 }
 430
 431 static int check_tree_block_fsid(struct extent_buffer *eb)
 432 {
 433         struct btrfs_fs_info *fs_info = eb->fs_info;
 434         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
 435         u8 fsid[BTRFS_FSID_SIZE];
 436         u8 *metadata_uuid;
 437
 438         read_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid),
 439                            BTRFS_FSID_SIZE);
 440         /*
 441          * Checking the incompat flag is only valid for the current fs. For
 442          * seed devices it's forbidden to have their uuid changed so reading
 443          * ->fsid in this case is fine
 444          */
 445         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
 446                 metadata_uuid = fs_devices->metadata_uuid;
 447         else
 448                 metadata_uuid = fs_devices->fsid;
 449
 450         if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE))
 451                 return 0;
 452
 453         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list)
 454                 if (!memcmp(fsid, seed_devs->fsid, BTRFS_FSID_SIZE))
 455                         return 0;
 456
 457         return 1;
 458 }
 459
 460 /* Do basic extent buffer checks at read time */
 461 static int validate_extent_buffer(struct extent_buffer *eb)
 462 {
 463         struct btrfs_fs_info *fs_info = eb->fs_info;
 464         u64 found_start;
 465         const u32 csum_size = fs_info->csum_size;
 466         u8 found_level;
 467         u8 result[BTRFS_CSUM_SIZE];
 468         const u8 *header_csum;
 469         int ret = 0;
 470
 471         found_start = btrfs_header_bytenr(eb);
 472         if (found_start != eb->start) {
 473                 btrfs_err_rl(fs_info,
 474                         "bad tree block start, mirror %u want %llu have %llu",
 475                              eb->read_mirror, eb->start, found_start);
 476                 ret = -EIO;
 477                 goto out;
 478         }
 479         if (check_tree_block_fsid(eb)) {
 480                 btrfs_err_rl(fs_info, "bad fsid on logical %llu mirror %u",
 481                              eb->start, eb->read_mirror);
 482                 ret = -EIO;
 483                 goto out;
 484         }
 485         found_level = btrfs_header_level(eb);
 486         if (found_level >= BTRFS_MAX_LEVEL) {
 487                 btrfs_err(fs_info,
 488                         "bad tree block level, mirror %u level %d on logical %llu",
 489                         eb->read_mirror, btrfs_header_level(eb), eb->start);
 490                 ret = -EIO;
 491                 goto out;
 492         }
 493
 494         csum_tree_block(eb, result);
 495         header_csum = page_address(eb->pages[0]) +
 496                 get_eb_offset_in_page(eb, offsetof(struct btrfs_header, csum));
 497
 498         if (memcmp(result, header_csum, csum_size) != 0) {
 499                 btrfs_warn_rl(fs_info,
 500 "checksum verify failed on logical %llu mirror %u wanted " CSUM_FMT " found " CSUM_FMT " level %d",
 501                               eb->start, eb->read_mirror,
 502                               CSUM_FMT_VALUE(csum_size, header_csum),
 503                               CSUM_FMT_VALUE(csum_size, result),
 504                               btrfs_header_level(eb));
 505                 ret = -EUCLEAN;
 506                 goto out;
 507         }
 508
 509         /*
 510          * If this is a leaf block and it is corrupt, set the corrupt bit so
 511          * that we don't try and read the other copies of this block, just
 512          * return -EIO.
 513          */
 514         if (found_level == 0 && btrfs_check_leaf_full(eb)) {
 515                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 516                 ret = -EIO;
 517         }
 518
 519         if (found_level > 0 && btrfs_check_node(eb))
 520                 ret = -EIO;
 521
 522         if (!ret)
 523                 set_extent_buffer_uptodate(eb);
 524         else
 525                 btrfs_err(fs_info,
 526                 "read time tree block corruption detected on logical %llu mirror %u",
 527                           eb->start, eb->read_mirror);
 528 out:
 529         return ret;
 530 }
 531
 532 static int validate_subpage_buffer(struct page *page, u64 start, u64 end,
 533                                    int mirror)
 534 {
 535         struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
 536         struct extent_buffer *eb;
 537         bool reads_done;
 538         int ret = 0;
 539
 540         /*
 541          * We don't allow bio merge for subpage metadata read, so we should
 542          * only get one eb for each endio hook.
 543          */
 544         ASSERT(end == start + fs_info->nodesize - 1);
 545         ASSERT(PagePrivate(page));
 546
 547         eb = find_extent_buffer(fs_info, start);
 548         /*
 549          * When we are reading one tree block, eb must have been inserted into
 550          * the radix tree. If not, something is wrong.
 551          */
 552         ASSERT(eb);
 553
 554         reads_done = atomic_dec_and_test(&eb->io_pages);
 555         /* Subpage read must finish in page read */
 556         ASSERT(reads_done);
 557
 558         eb->read_mirror = mirror;
 559         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 560                 ret = -EIO;
 561                 goto err;
 562         }
 563         ret = validate_extent_buffer(eb);
 564         if (ret < 0)
 565                 goto err;
 566
 567         set_extent_buffer_uptodate(eb);
 568
 569         free_extent_buffer(eb);
 570         return ret;
 571 err:
 572         /*
 573          * end_bio_extent_readpage decrements io_pages in case of error,
 574          * make sure it has something to decrement.
 575          */
 576         atomic_inc(&eb->io_pages);
 577         clear_extent_buffer_uptodate(eb);
 578         free_extent_buffer(eb);
 579         return ret;
 580 }
 581
 582 int btrfs_validate_metadata_buffer(struct btrfs_bio *bbio,
 583                                    struct page *page, u64 start, u64 end,
 584                                    int mirror)
 585 {
 586         struct extent_buffer *eb;
 587         int ret = 0;
 588         int reads_done;
 589
 590         ASSERT(page->private);
 591
 592         if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
 593                 return validate_subpage_buffer(page, start, end, mirror);
 594
 595         eb = (struct extent_buffer *)page->private;
 596
 597         /*
 598          * The pending IO might have been the only thing that kept this buffer
 599          * in memory.  Make sure we have a ref for all this other checks
 600          */
 601         atomic_inc(&eb->refs);
 602
 603         reads_done = atomic_dec_and_test(&eb->io_pages);
 604         if (!reads_done)
 605                 goto err;
 606
 607         eb->read_mirror = mirror;
 608         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 609                 ret = -EIO;
 610                 goto err;
 611         }
 612         ret = validate_extent_buffer(eb);
 613 err:
 614         if (ret) {
 615                 /*
 616                  * our io error hook is going to dec the io pages
 617                  * again, we have to make sure it has something
 618                  * to decrement
 619                  */
 620                 atomic_inc(&eb->io_pages);
 621                 clear_extent_buffer_uptodate(eb);
 622         }
 623         free_extent_buffer(eb);
 624
 625         return ret;
 626 }
 627
 628 static void run_one_async_start(struct btrfs_work *work)
 629 {
 630         struct async_submit_bio *async;
 631         blk_status_t ret;
 632
 633         async = container_of(work, struct  async_submit_bio, work);
 634         ret = async->submit_bio_start(async->inode, async->bio,
 635                                       async->dio_file_offset);
 636         if (ret)
 637                 async->status = ret;
 638 }
 639
 640 /*
 641  * In order to insert checksums into the metadata in large chunks, we wait
 642  * until bio submission time.   All the pages in the bio are checksummed and
 643  * sums are attached onto the ordered extent record.
 644  *
 645  * At IO completion time the csums attached on the ordered extent record are
 646  * inserted into the tree.
 647  */
 648 static void run_one_async_done(struct btrfs_work *work)
 649 {
 650         struct async_submit_bio *async;
 651         struct inode *inode;
 652
 653         async = container_of(work, struct  async_submit_bio, work);
 654         inode = async->inode;
 655
 656         /* If an error occurred we just want to clean up the bio and move on */
 657         if (async->status) {
 658                 async->bio->bi_status = async->status;
 659                 bio_endio(async->bio);
 660                 return;
 661         }
 662
 663         /*
 664          * All of the bios that pass through here are from async helpers.
 665          * Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context.
 666          * This changes nothing when cgroups aren't in use.
 667          */
 668         async->bio->bi_opf |= REQ_CGROUP_PUNT;
 669         btrfs_submit_bio(btrfs_sb(inode->i_sb), async->bio, async->mirror_num);
 670 }
 671
 672 static void run_one_async_free(struct btrfs_work *work)
 673 {
 674         struct async_submit_bio *async;
 675
 676         async = container_of(work, struct  async_submit_bio, work);
 677         kfree(async);
 678 }
 679
 680 /*
 681  * Submit bio to an async queue.
 682  *
 683  * Retrun:
 684  * - true if the work has been succesfuly submitted
 685  * - false in case of error
 686  */
 687 bool btrfs_wq_submit_bio(struct inode *inode, struct bio *bio, int mirror_num,
 688                          u64 dio_file_offset,
 689                          extent_submit_bio_start_t *submit_bio_start)
 690 {
 691         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
 692         struct async_submit_bio *async;
 693
 694         async = kmalloc(sizeof(*async), GFP_NOFS);
 695         if (!async)
 696                 return false;
 697
 698         async->inode = inode;
 699         async->bio = bio;
 700         async->mirror_num = mirror_num;
 701         async->submit_bio_start = submit_bio_start;
 702
 703         btrfs_init_work(&async->work, run_one_async_start, run_one_async_done,
 704                         run_one_async_free);
 705
 706         async->dio_file_offset = dio_file_offset;
 707
 708         async->status = 0;
 709
 710         if (op_is_sync(bio->bi_opf))
 711                 btrfs_queue_work(fs_info->hipri_workers, &async->work);
 712         else
 713                 btrfs_queue_work(fs_info->workers, &async->work);
 714         return true;
 715 }
 716
 717 static blk_status_t btree_csum_one_bio(struct bio *bio)
 718 {
 719         struct bio_vec *bvec;
 720         struct btrfs_root *root;
 721         int ret = 0;
 722         struct bvec_iter_all iter_all;
 723
 724         ASSERT(!bio_flagged(bio, BIO_CLONED));
 725         bio_for_each_segment_all(bvec, bio, iter_all) {
 726                 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
 727                 ret = csum_dirty_buffer(root->fs_info, bvec);
 728                 if (ret)
 729                         break;
 730         }
 731
 732         return errno_to_blk_status(ret);
 733 }
 734
 735 static blk_status_t btree_submit_bio_start(struct inode *inode, struct bio *bio,
 736                                            u64 dio_file_offset)
 737 {
 738         /*
 739          * when we're called for a write, we're already in the async
 740          * submission context.  Just jump into btrfs_submit_bio.
 741          */
 742         return btree_csum_one_bio(bio);
 743 }
 744
 745 static bool should_async_write(struct btrfs_fs_info *fs_info,
 746                              struct btrfs_inode *bi)
 747 {
 748         if (btrfs_is_zoned(fs_info))
 749                 return false;
 750         if (atomic_read(&bi->sync_writers))
 751                 return false;
 752         if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags))
 753                 return false;
 754         return true;
 755 }
 756
 757 void btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int mirror_num)
 758 {
 759         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 760         blk_status_t ret;
 761
 762         bio->bi_opf |= REQ_META;
 763
 764         if (btrfs_op(bio) != BTRFS_MAP_WRITE) {
 765                 btrfs_submit_bio(fs_info, bio, mirror_num);
 766                 return;
 767         }
 768
 769         /*
 770          * Kthread helpers are used to submit writes so that checksumming can
 771          * happen in parallel across all CPUs.
 772          */
 773         if (should_async_write(fs_info, BTRFS_I(inode)) &&
 774             btrfs_wq_submit_bio(inode, bio, mirror_num, 0, btree_submit_bio_start))
 775                 return;
 776
 777         ret = btree_csum_one_bio(bio);
 778         if (ret) {
 779                 bio->bi_status = ret;
 780                 bio_endio(bio);
 781                 return;
 782         }
 783
 784         btrfs_submit_bio(fs_info, bio, mirror_num);
 785 }
 786
 787 #ifdef CONFIG_MIGRATION
 788 static int btree_migratepage(struct address_space *mapping,
 789                         struct page *newpage, struct page *page,
 790                         enum migrate_mode mode)
 791 {
 792         /*
 793          * we can't safely write a btree page from here,
 794          * we haven't done the locking hook
 795          */
 796         if (PageDirty(page))
 797                 return -EAGAIN;
 798         /*
 799          * Buffers may be managed in a filesystem specific way.
 800          * We must have no buffers or drop them.
 801          */
 802         if (page_has_private(page) &&
 803             !try_to_release_page(page, GFP_KERNEL))
 804                 return -EAGAIN;
 805         return migrate_page(mapping, newpage, page, mode);
 806 }
 807 #endif
 808
 809
 810 static int btree_writepages(struct address_space *mapping,
 811                             struct writeback_control *wbc)
 812 {
 813         struct btrfs_fs_info *fs_info;
 814         int ret;
 815
 816         if (wbc->sync_mode == WB_SYNC_NONE) {
 817
 818                 if (wbc->for_kupdate)
 819                         return 0;
 820
 821                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
 822                 /* this is a bit racy, but that's ok */
 823                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
 824                                              BTRFS_DIRTY_METADATA_THRESH,
 825                                              fs_info->dirty_metadata_batch);
 826                 if (ret < 0)
 827                         return 0;
 828         }
 829         return btree_write_cache_pages(mapping, wbc);
 830 }
 831
 832 static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags)
 833 {
 834         if (folio_test_writeback(folio) || folio_test_dirty(folio))
 835                 return false;
 836
 837         return try_release_extent_buffer(&folio->page);
 838 }
 839
 840 static void btree_invalidate_folio(struct folio *folio, size_t offset,
 841                                  size_t length)
 842 {
 843         struct extent_io_tree *tree;
 844         tree = &BTRFS_I(folio->mapping->host)->io_tree;
 845         extent_invalidate_folio(tree, folio, offset);
 846         btree_release_folio(folio, GFP_NOFS);
 847         if (folio_get_private(folio)) {
 848                 btrfs_warn(BTRFS_I(folio->mapping->host)->root->fs_info,
 849                            "folio private not zero on folio %llu",
 850                            (unsigned long long)folio_pos(folio));
 851                 folio_detach_private(folio);
 852         }
 853 }
 854
 855 #ifdef DEBUG
 856 static bool btree_dirty_folio(struct address_space *mapping,
 857                 struct folio *folio)
 858 {
 859         struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
 860         struct btrfs_subpage *subpage;
 861         struct extent_buffer *eb;
 862         int cur_bit = 0;
 863         u64 page_start = folio_pos(folio);
 864
 865         if (fs_info->sectorsize == PAGE_SIZE) {
 866                 eb = folio_get_private(folio);
 867                 BUG_ON(!eb);
 868                 BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 869                 BUG_ON(!atomic_read(&eb->refs));
 870                 btrfs_assert_tree_write_locked(eb);
 871                 return filemap_dirty_folio(mapping, folio);
 872         }
 873         subpage = folio_get_private(folio);
 874
 875         ASSERT(subpage->dirty_bitmap);
 876         while (cur_bit < BTRFS_SUBPAGE_BITMAP_SIZE) {
 877                 unsigned long flags;
 878                 u64 cur;
 879                 u16 tmp = (1 << cur_bit);
 880
 881                 spin_lock_irqsave(&subpage->lock, flags);
 882                 if (!(tmp & subpage->dirty_bitmap)) {
 883                         spin_unlock_irqrestore(&subpage->lock, flags);
 884                         cur_bit++;
 885                         continue;
 886                 }
 887                 spin_unlock_irqrestore(&subpage->lock, flags);
 888                 cur = page_start + cur_bit * fs_info->sectorsize;
 889
 890                 eb = find_extent_buffer(fs_info, cur);
 891                 ASSERT(eb);
 892                 ASSERT(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 893                 ASSERT(atomic_read(&eb->refs));
 894                 btrfs_assert_tree_write_locked(eb);
 895                 free_extent_buffer(eb);
 896
 897                 cur_bit += (fs_info->nodesize >> fs_info->sectorsize_bits);
 898         }
 899         return filemap_dirty_folio(mapping, folio);
 900 }
 901 #else
 902 #define btree_dirty_folio filemap_dirty_folio
 903 #endif
 904
 905 static const struct address_space_operations btree_aops = {
 906         .writepages     = btree_writepages,
 907         .release_folio  = btree_release_folio,
 908         .invalidate_folio = btree_invalidate_folio,
 909 #ifdef CONFIG_MIGRATION
 910         .migratepage    = btree_migratepage,
 911 #endif
 912         .dirty_folio = btree_dirty_folio,
 913 };
 914
 915 struct extent_buffer *btrfs_find_create_tree_block(
 916                                                 struct btrfs_fs_info *fs_info,
 917                                                 u64 bytenr, u64 owner_root,
 918                                                 int level)
 919 {
 920         if (btrfs_is_testing(fs_info))
 921                 return alloc_test_extent_buffer(fs_info, bytenr);
 922         return alloc_extent_buffer(fs_info, bytenr, owner_root, level);
 923 }
 924
 925 /*
 926  * Read tree block at logical address @bytenr and do variant basic but critical
 927  * verification.
 928  *
 929  * @owner_root:         the objectid of the root owner for this block.
 930  * @parent_transid:     expected transid of this tree block, skip check if 0
 931  * @level:              expected level, mandatory check
 932  * @first_key:          expected key in slot 0, skip check if NULL
 933  */
 934 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
 935                                       u64 owner_root, u64 parent_transid,
 936                                       int level, struct btrfs_key *first_key)
 937 {
 938         struct extent_buffer *buf = NULL;
 939         int ret;
 940
 941         buf = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
 942         if (IS_ERR(buf))
 943                 return buf;
 944
 945         ret = btrfs_read_extent_buffer(buf, parent_transid, level, first_key);
 946         if (ret) {
 947                 free_extent_buffer_stale(buf);
 948                 return ERR_PTR(ret);
 949         }
 950         if (btrfs_check_eb_owner(buf, owner_root)) {
 951                 free_extent_buffer_stale(buf);
 952                 return ERR_PTR(-EUCLEAN);
 953         }
 954         return buf;
 955
 956 }
 957
 958 void btrfs_clean_tree_block(struct extent_buffer *buf)
 959 {
 960         struct btrfs_fs_info *fs_info = buf->fs_info;
 961         if (btrfs_header_generation(buf) ==
 962             fs_info->running_transaction->transid) {
 963                 btrfs_assert_tree_write_locked(buf);
 964
 965                 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) {
 966                         percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
 967                                                  -buf->len,
 968                                                  fs_info->dirty_metadata_batch);
 969                         clear_extent_buffer_dirty(buf);
 970                 }
 971         }
 972 }
 973
 974 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
 975                          u64 objectid)
 976 {
 977         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
 978
 979         memset(&root->root_key, 0, sizeof(root->root_key));
 980         memset(&root->root_item, 0, sizeof(root->root_item));
 981         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
 982         root->fs_info = fs_info;
 983         root->root_key.objectid = objectid;
 984         root->node = NULL;
 985         root->commit_root = NULL;
 986         root->state = 0;
 987         RB_CLEAR_NODE(&root->rb_node);
 988
 989         root->last_trans = 0;
 990         root->free_objectid = 0;
 991         root->nr_delalloc_inodes = 0;
 992         root->nr_ordered_extents = 0;
 993         root->inode_tree = RB_ROOT;
 994         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
 995
 996         btrfs_init_root_block_rsv(root);
 997
 998         INIT_LIST_HEAD(&root->dirty_list);
 999         INIT_LIST_HEAD(&root->root_list);
1000         INIT_LIST_HEAD(&root->delalloc_inodes);
1001         INIT_LIST_HEAD(&root->delalloc_root);
1002         INIT_LIST_HEAD(&root->ordered_extents);
1003         INIT_LIST_HEAD(&root->ordered_root);
1004         INIT_LIST_HEAD(&root->reloc_dirty_list);
1005         INIT_LIST_HEAD(&root->logged_list[0]);
1006         INIT_LIST_HEAD(&root->logged_list[1]);
1007         spin_lock_init(&root->inode_lock);
1008         spin_lock_init(&root->delalloc_lock);
1009         spin_lock_init(&root->ordered_extent_lock);
1010         spin_lock_init(&root->accounting_lock);
1011         spin_lock_init(&root->log_extents_lock[0]);
1012         spin_lock_init(&root->log_extents_lock[1]);
1013         spin_lock_init(&root->qgroup_meta_rsv_lock);
1014         mutex_init(&root->objectid_mutex);
1015         mutex_init(&root->log_mutex);
1016         mutex_init(&root->ordered_extent_mutex);
1017         mutex_init(&root->delalloc_mutex);
1018         init_waitqueue_head(&root->qgroup_flush_wait);
1019         init_waitqueue_head(&root->log_writer_wait);
1020         init_waitqueue_head(&root->log_commit_wait[0]);
1021         init_waitqueue_head(&root->log_commit_wait[1]);
1022         INIT_LIST_HEAD(&root->log_ctxs[0]);
1023         INIT_LIST_HEAD(&root->log_ctxs[1]);
1024         atomic_set(&root->log_commit[0], 0);
1025         atomic_set(&root->log_commit[1], 0);
1026         atomic_set(&root->log_writers, 0);
1027         atomic_set(&root->log_batch, 0);
1028         refcount_set(&root->refs, 1);
1029         atomic_set(&root->snapshot_force_cow, 0);
1030         atomic_set(&root->nr_swapfiles, 0);
1031         root->log_transid = 0;
1032         root->log_transid_committed = -1;
1033         root->last_log_commit = 0;
1034         root->anon_dev = 0;
1035         if (!dummy) {
1036                 extent_io_tree_init(fs_info, &root->dirty_log_pages,
1037                                     IO_TREE_ROOT_DIRTY_LOG_PAGES, NULL);
1038                 extent_io_tree_init(fs_info, &root->log_csum_range,
1039                                     IO_TREE_LOG_CSUM_RANGE, NULL);
1040         }
1041
1042         spin_lock_init(&root->root_item_lock);
1043         btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks);
1044 #ifdef CONFIG_BTRFS_DEBUG
1045         INIT_LIST_HEAD(&root->leak_list);
1046         spin_lock(&fs_info->fs_roots_radix_lock);
1047         list_add_tail(&root->leak_list, &fs_info->allocated_roots);
1048         spin_unlock(&fs_info->fs_roots_radix_lock);
1049 #endif
1050 }
1051
1052 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
1053                                            u64 objectid, gfp_t flags)
1054 {
1055         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
1056         if (root)
1057                 __setup_root(root, fs_info, objectid);
1058         return root;
1059 }
1060
1061 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1062 /* Should only be used by the testing infrastructure */
1063 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
1064 {
1065         struct btrfs_root *root;
1066
1067         if (!fs_info)
1068                 return ERR_PTR(-EINVAL);
1069
1070         root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, GFP_KERNEL);
1071         if (!root)
1072                 return ERR_PTR(-ENOMEM);
1073
1074         /* We don't use the stripesize in selftest, set it as sectorsize */
1075         root->alloc_bytenr = 0;
1076
1077         return root;
1078 }
1079 #endif
1080
1081 static int global_root_cmp(struct rb_node *a_node, const struct rb_node *b_node)
1082 {
1083         const struct btrfs_root *a = rb_entry(a_node, struct btrfs_root, rb_node);
1084         const struct btrfs_root *b = rb_entry(b_node, struct btrfs_root, rb_node);
1085
1086         return btrfs_comp_cpu_keys(&a->root_key, &b->root_key);
1087 }
1088
1089 static int global_root_key_cmp(const void *k, const struct rb_node *node)
1090 {
1091         const struct btrfs_key *key = k;
1092         const struct btrfs_root *root = rb_entry(node, struct btrfs_root, rb_node);
1093
1094         return btrfs_comp_cpu_keys(key, &root->root_key);
1095 }
1096
1097 int btrfs_global_root_insert(struct btrfs_root *root)
1098 {
1099         struct btrfs_fs_info *fs_info = root->fs_info;
1100         struct rb_node *tmp;
1101
1102         write_lock(&fs_info->global_root_lock);
1103         tmp = rb_find_add(&root->rb_node, &fs_info->global_root_tree, global_root_cmp);
1104         write_unlock(&fs_info->global_root_lock);
1105         ASSERT(!tmp);
1106
1107         return tmp ? -EEXIST : 0;
1108 }
1109
1110 void btrfs_global_root_delete(struct btrfs_root *root)
1111 {
1112         struct btrfs_fs_info *fs_info = root->fs_info;
1113
1114         write_lock(&fs_info->global_root_lock);
1115         rb_erase(&root->rb_node, &fs_info->global_root_tree);
1116         write_unlock(&fs_info->global_root_lock);
1117 }
1118
1119 struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info,
1120                                      struct btrfs_key *key)
1121 {
1122         struct rb_node *node;
1123         struct btrfs_root *root = NULL;
1124
1125         read_lock(&fs_info->global_root_lock);
1126         node = rb_find(key, &fs_info->global_root_tree, global_root_key_cmp);
1127         if (node)
1128                 root = container_of(node, struct btrfs_root, rb_node);
1129         read_unlock(&fs_info->global_root_lock);
1130
1131         return root;
1132 }
1133
1134 static u64 btrfs_global_root_id(struct btrfs_fs_info *fs_info, u64 bytenr)
1135 {
1136         struct btrfs_block_group *block_group;
1137         u64 ret;
1138
1139         if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))
1140                 return 0;
1141
1142         if (bytenr)
1143                 block_group = btrfs_lookup_block_group(fs_info, bytenr);
1144         else
1145                 block_group = btrfs_lookup_first_block_group(fs_info, bytenr);
1146         ASSERT(block_group);
1147         if (!block_group)
1148                 return 0;
1149         ret = block_group->global_root_id;
1150         btrfs_put_block_group(block_group);
1151
1152         return ret;
1153 }
1154
1155 struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1156 {
1157         struct btrfs_key key = {
1158                 .objectid = BTRFS_CSUM_TREE_OBJECTID,
1159                 .type = BTRFS_ROOT_ITEM_KEY,
1160                 .offset = btrfs_global_root_id(fs_info, bytenr),
1161         };
1162
1163         return btrfs_global_root(fs_info, &key);
1164 }
1165
1166 struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1167 {
1168         struct btrfs_key key = {
1169                 .objectid = BTRFS_EXTENT_TREE_OBJECTID,
1170                 .type = BTRFS_ROOT_ITEM_KEY,
1171                 .offset = btrfs_global_root_id(fs_info, bytenr),
1172         };
1173
1174         return btrfs_global_root(fs_info, &key);
1175 }
1176
1177 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1178                                      u64 objectid)
1179 {
1180         struct btrfs_fs_info *fs_info = trans->fs_info;
1181         struct extent_buffer *leaf;
1182         struct btrfs_root *tree_root = fs_info->tree_root;
1183         struct btrfs_root *root;
1184         struct btrfs_key key;
1185         unsigned int nofs_flag;
1186         int ret = 0;
1187
1188         /*
1189          * We're holding a transaction handle, so use a NOFS memory allocation
1190          * context to avoid deadlock if reclaim happens.
1191          */
1192         nofs_flag = memalloc_nofs_save();
1193         root = btrfs_alloc_root(fs_info, objectid, GFP_KERNEL);
1194         memalloc_nofs_restore(nofs_flag);
1195         if (!root)
1196                 return ERR_PTR(-ENOMEM);
1197
1198         root->root_key.objectid = objectid;
1199         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1200         root->root_key.offset = 0;
1201
1202         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0,
1203                                       BTRFS_NESTING_NORMAL);
1204         if (IS_ERR(leaf)) {
1205                 ret = PTR_ERR(leaf);
1206                 leaf = NULL;
1207                 goto fail_unlock;
1208         }
1209
1210         root->node = leaf;
1211         btrfs_mark_buffer_dirty(leaf);
1212
1213         root->commit_root = btrfs_root_node(root);
1214         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1215
1216         btrfs_set_root_flags(&root->root_item, 0);
1217         btrfs_set_root_limit(&root->root_item, 0);
1218         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1219         btrfs_set_root_generation(&root->root_item, trans->transid);
1220         btrfs_set_root_level(&root->root_item, 0);
1221         btrfs_set_root_refs(&root->root_item, 1);
1222         btrfs_set_root_used(&root->root_item, leaf->len);
1223         btrfs_set_root_last_snapshot(&root->root_item, 0);
1224         btrfs_set_root_dirid(&root->root_item, 0);
1225         if (is_fstree(objectid))
1226                 generate_random_guid(root->root_item.uuid);
1227         else
1228                 export_guid(root->root_item.uuid, &guid_null);
1229         btrfs_set_root_drop_level(&root->root_item, 0);
1230
1231         btrfs_tree_unlock(leaf);
1232
1233         key.objectid = objectid;
1234         key.type = BTRFS_ROOT_ITEM_KEY;
1235         key.offset = 0;
1236         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1237         if (ret)
1238                 goto fail;
1239
1240         return root;
1241
1242 fail_unlock:
1243         if (leaf)
1244                 btrfs_tree_unlock(leaf);
1245 fail:
1246         btrfs_put_root(root);
1247
1248         return ERR_PTR(ret);
1249 }
1250
1251 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1252                                          struct btrfs_fs_info *fs_info)
1253 {
1254         struct btrfs_root *root;
1255
1256         root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS);
1257         if (!root)
1258                 return ERR_PTR(-ENOMEM);
1259
1260         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1261         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1262         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1263
1264         return root;
1265 }
1266
1267 int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans,
1268                               struct btrfs_root *root)
1269 {
1270         struct extent_buffer *leaf;
1271
1272         /*
1273          * DON'T set SHAREABLE bit for log trees.
1274          *
1275          * Log trees are not exposed to user space thus can't be snapshotted,
1276          * and they go away before a real commit is actually done.
1277          *
1278          * They do store pointers to file data extents, and those reference
1279          * counts still get updated (along with back refs to the log tree).
1280          */
1281
1282         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1283                         NULL, 0, 0, 0, BTRFS_NESTING_NORMAL);
1284         if (IS_ERR(leaf))
1285                 return PTR_ERR(leaf);
1286
1287         root->node = leaf;
1288
1289         btrfs_mark_buffer_dirty(root->node);
1290         btrfs_tree_unlock(root->node);
1291
1292         return 0;
1293 }
1294
1295 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1296                              struct btrfs_fs_info *fs_info)
1297 {
1298         struct btrfs_root *log_root;
1299
1300         log_root = alloc_log_tree(trans, fs_info);
1301         if (IS_ERR(log_root))
1302                 return PTR_ERR(log_root);
1303
1304         if (!btrfs_is_zoned(fs_info)) {
1305                 int ret = btrfs_alloc_log_tree_node(trans, log_root);
1306
1307                 if (ret) {
1308                         btrfs_put_root(log_root);
1309                         return ret;
1310                 }
1311         }
1312
1313         WARN_ON(fs_info->log_root_tree);
1314         fs_info->log_root_tree = log_root;
1315         return 0;
1316 }
1317
1318 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1319                        struct btrfs_root *root)
1320 {
1321         struct btrfs_fs_info *fs_info = root->fs_info;
1322         struct btrfs_root *log_root;
1323         struct btrfs_inode_item *inode_item;
1324         int ret;
1325
1326         log_root = alloc_log_tree(trans, fs_info);
1327         if (IS_ERR(log_root))
1328                 return PTR_ERR(log_root);
1329
1330         ret = btrfs_alloc_log_tree_node(trans, log_root);
1331         if (ret) {
1332                 btrfs_put_root(log_root);
1333                 return ret;
1334         }
1335
1336         log_root->last_trans = trans->transid;
1337         log_root->root_key.offset = root->root_key.objectid;
1338
1339         inode_item = &log_root->root_item.inode;
1340         btrfs_set_stack_inode_generation(inode_item, 1);
1341         btrfs_set_stack_inode_size(inode_item, 3);
1342         btrfs_set_stack_inode_nlink(inode_item, 1);
1343         btrfs_set_stack_inode_nbytes(inode_item,
1344                                      fs_info->nodesize);
1345         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1346
1347         btrfs_set_root_node(&log_root->root_item, log_root->node);
1348
1349         WARN_ON(root->log_root);
1350         root->log_root = log_root;
1351         root->log_transid = 0;
1352         root->log_transid_committed = -1;
1353         root->last_log_commit = 0;
1354         return 0;
1355 }
1356
1357 static struct btrfs_root *read_tree_root_path(struct btrfs_root *tree_root,
1358                                               struct btrfs_path *path,
1359                                               struct btrfs_key *key)
1360 {
1361         struct btrfs_root *root;
1362         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1363         u64 generation;
1364         int ret;
1365         int level;
1366
1367         root = btrfs_alloc_root(fs_info, key->objectid, GFP_NOFS);
1368         if (!root)
1369                 return ERR_PTR(-ENOMEM);
1370
1371         ret = btrfs_find_root(tree_root, key, path,
1372                               &root->root_item, &root->root_key);
1373         if (ret) {
1374                 if (ret > 0)
1375                         ret = -ENOENT;
1376                 goto fail;
1377         }
1378
1379         generation = btrfs_root_generation(&root->root_item);
1380         level = btrfs_root_level(&root->root_item);
1381         root->node = read_tree_block(fs_info,
1382                                      btrfs_root_bytenr(&root->root_item),
1383                                      key->objectid, generation, level, NULL);
1384         if (IS_ERR(root->node)) {
1385                 ret = PTR_ERR(root->node);
1386                 root->node = NULL;
1387                 goto fail;
1388         }
1389         if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1390                 ret = -EIO;
1391                 goto fail;
1392         }
1393
1394         /*
1395          * For real fs, and not log/reloc trees, root owner must
1396          * match its root node owner
1397          */
1398         if (!test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state) &&
1399             root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1400             root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
1401             root->root_key.objectid != btrfs_header_owner(root->node)) {
1402                 btrfs_crit(fs_info,
1403 "root=%llu block=%llu, tree root owner mismatch, have %llu expect %llu",
1404                            root->root_key.objectid, root->node->start,
1405                            btrfs_header_owner(root->node),
1406                            root->root_key.objectid);
1407                 ret = -EUCLEAN;
1408                 goto fail;
1409         }
1410         root->commit_root = btrfs_root_node(root);
1411         return root;
1412 fail:
1413         btrfs_put_root(root);
1414         return ERR_PTR(ret);
1415 }
1416
1417 struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1418                                         struct btrfs_key *key)
1419 {
1420         struct btrfs_root *root;
1421         struct btrfs_path *path;
1422
1423         path = btrfs_alloc_path();
1424         if (!path)
1425                 return ERR_PTR(-ENOMEM);
1426         root = read_tree_root_path(tree_root, path, key);
1427         btrfs_free_path(path);
1428
1429         return root;
1430 }
1431
1432 /*
1433  * Initialize subvolume root in-memory structure
1434  *
1435  * @anon_dev:   anonymous device to attach to the root, if zero, allocate new
1436  */
1437 static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev)
1438 {
1439         int ret;
1440         unsigned int nofs_flag;
1441
1442         /*
1443          * We might be called under a transaction (e.g. indirect backref
1444          * resolution) which could deadlock if it triggers memory reclaim
1445          */
1446         nofs_flag = memalloc_nofs_save();
1447         ret = btrfs_drew_lock_init(&root->snapshot_lock);
1448         memalloc_nofs_restore(nofs_flag);
1449         if (ret)
1450                 goto fail;
1451
1452         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1453             !btrfs_is_data_reloc_root(root)) {
1454                 set_bit(BTRFS_ROOT_SHAREABLE, &root->state);
1455                 btrfs_check_and_init_root_item(&root->root_item);
1456         }
1457
1458         /*
1459          * Don't assign anonymous block device to roots that are not exposed to
1460          * userspace, the id pool is limited to 1M
1461          */
1462         if (is_fstree(root->root_key.objectid) &&
1463             btrfs_root_refs(&root->root_item) > 0) {
1464                 if (!anon_dev) {
1465                         ret = get_anon_bdev(&root->anon_dev);
1466                         if (ret)
1467                                 goto fail;
1468                 } else {
1469                         root->anon_dev = anon_dev;
1470                 }
1471         }
1472
1473         mutex_lock(&root->objectid_mutex);
1474         ret = btrfs_init_root_free_objectid(root);
1475         if (ret) {
1476                 mutex_unlock(&root->objectid_mutex);
1477                 goto fail;
1478         }
1479
1480         ASSERT(root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
1481
1482         mutex_unlock(&root->objectid_mutex);
1483
1484         return 0;
1485 fail:
1486         /* The caller is responsible to call btrfs_free_fs_root */
1487         return ret;
1488 }
1489
1490 static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1491                                                u64 root_id)
1492 {
1493         struct btrfs_root *root;
1494
1495         spin_lock(&fs_info->fs_roots_radix_lock);
1496         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1497                                  (unsigned long)root_id);
1498         if (root)
1499                 root = btrfs_grab_root(root);
1500         spin_unlock(&fs_info->fs_roots_radix_lock);
1501         return root;
1502 }
1503
1504 static struct btrfs_root *btrfs_get_global_root(struct btrfs_fs_info *fs_info,
1505                                                 u64 objectid)
1506 {
1507         struct btrfs_key key = {
1508                 .objectid = objectid,
1509                 .type = BTRFS_ROOT_ITEM_KEY,
1510                 .offset = 0,
1511         };
1512
1513         if (objectid == BTRFS_ROOT_TREE_OBJECTID)
1514                 return btrfs_grab_root(fs_info->tree_root);
1515         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
1516                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1517         if (objectid == BTRFS_CHUNK_TREE_OBJECTID)
1518                 return btrfs_grab_root(fs_info->chunk_root);
1519         if (objectid == BTRFS_DEV_TREE_OBJECTID)
1520                 return btrfs_grab_root(fs_info->dev_root);
1521         if (objectid == BTRFS_CSUM_TREE_OBJECTID)
1522                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1523         if (objectid == BTRFS_QUOTA_TREE_OBJECTID)
1524                 return btrfs_grab_root(fs_info->quota_root) ?
1525                         fs_info->quota_root : ERR_PTR(-ENOENT);
1526         if (objectid == BTRFS_UUID_TREE_OBJECTID)
1527                 return btrfs_grab_root(fs_info->uuid_root) ?
1528                         fs_info->uuid_root : ERR_PTR(-ENOENT);
1529         if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) {
1530                 struct btrfs_root *root = btrfs_global_root(fs_info, &key);
1531
1532                 return btrfs_grab_root(root) ? root : ERR_PTR(-ENOENT);
1533         }
1534         return NULL;
1535 }
1536
1537 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1538                          struct btrfs_root *root)
1539 {
1540         int ret;
1541
1542         ret = radix_tree_preload(GFP_NOFS);
1543         if (ret)
1544                 return ret;
1545
1546         spin_lock(&fs_info->fs_roots_radix_lock);
1547         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1548                                 (unsigned long)root->root_key.objectid,
1549                                 root);
1550         if (ret == 0) {
1551                 btrfs_grab_root(root);
1552                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1553         }
1554         spin_unlock(&fs_info->fs_roots_radix_lock);
1555         radix_tree_preload_end();
1556
1557         return ret;
1558 }
1559
1560 void btrfs_check_leaked_roots(struct btrfs_fs_info *fs_info)
1561 {
1562 #ifdef CONFIG_BTRFS_DEBUG
1563         struct btrfs_root *root;
1564
1565         while (!list_empty(&fs_info->allocated_roots)) {
1566                 char buf[BTRFS_ROOT_NAME_BUF_LEN];
1567
1568                 root = list_first_entry(&fs_info->allocated_roots,
1569                                         struct btrfs_root, leak_list);
1570                 btrfs_err(fs_info, "leaked root %s refcount %d",
1571                           btrfs_root_name(&root->root_key, buf),
1572                           refcount_read(&root->refs));
1573                 while (refcount_read(&root->refs) > 1)
1574                         btrfs_put_root(root);
1575                 btrfs_put_root(root);
1576         }
1577 #endif
1578 }
1579
1580 static void free_global_roots(struct btrfs_fs_info *fs_info)
1581 {
1582         struct btrfs_root *root;
1583         struct rb_node *node;
1584
1585         while ((node = rb_first_postorder(&fs_info->global_root_tree)) != NULL) {
1586                 root = rb_entry(node, struct btrfs_root, rb_node);
1587                 rb_erase(&root->rb_node, &fs_info->global_root_tree);
1588                 btrfs_put_root(root);
1589         }
1590 }
1591
1592 void btrfs_free_fs_info(struct btrfs_fs_info *fs_info)
1593 {
1594         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
1595         percpu_counter_destroy(&fs_info->delalloc_bytes);
1596         percpu_counter_destroy(&fs_info->ordered_bytes);
1597         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
1598         btrfs_free_csum_hash(fs_info);
1599         btrfs_free_stripe_hash_table(fs_info);
1600         btrfs_free_ref_cache(fs_info);
1601         kfree(fs_info->balance_ctl);
1602         kfree(fs_info->delayed_root);
1603         free_global_roots(fs_info);
1604         btrfs_put_root(fs_info->tree_root);
1605         btrfs_put_root(fs_info->chunk_root);
1606         btrfs_put_root(fs_info->dev_root);
1607         btrfs_put_root(fs_info->quota_root);
1608         btrfs_put_root(fs_info->uuid_root);
1609         btrfs_put_root(fs_info->fs_root);
1610         btrfs_put_root(fs_info->data_reloc_root);
1611         btrfs_put_root(fs_info->block_group_root);
1612         btrfs_check_leaked_roots(fs_info);
1613         btrfs_extent_buffer_leak_debug_check(fs_info);
1614         kfree(fs_info->super_copy);
1615         kfree(fs_info->super_for_commit);
1616         kfree(fs_info->subpage_info);
1617         kvfree(fs_info);
1618 }
1619
1620
1621 /*
1622  * Get an in-memory reference of a root structure.
1623  *
1624  * For essential trees like root/extent tree, we grab it from fs_info directly.
1625  * For subvolume trees, we check the cached filesystem roots first. If not
1626  * found, then read it from disk and add it to cached fs roots.
1627  *
1628  * Caller should release the root by calling btrfs_put_root() after the usage.
1629  *
1630  * NOTE: Reloc and log trees can't be read by this function as they share the
1631  *       same root objectid.
1632  *
1633  * @objectid:   root id
1634  * @anon_dev:   preallocated anonymous block device number for new roots,
1635  *              pass 0 for new allocation.
1636  * @check_ref:  whether to check root item references, If true, return -ENOENT
1637  *              for orphan roots
1638  */
1639 static struct btrfs_root *btrfs_get_root_ref(struct btrfs_fs_info *fs_info,
1640                                              u64 objectid, dev_t anon_dev,
1641                                              bool check_ref)
1642 {
1643         struct btrfs_root *root;
1644         struct btrfs_path *path;
1645         struct btrfs_key key;
1646         int ret;
1647
1648         root = btrfs_get_global_root(fs_info, objectid);
1649         if (root)
1650                 return root;
1651 again:
1652         root = btrfs_lookup_fs_root(fs_info, objectid);
1653         if (root) {
1654                 /* Shouldn't get preallocated anon_dev for cached roots */
1655                 ASSERT(!anon_dev);
1656                 if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1657                         btrfs_put_root(root);
1658                         return ERR_PTR(-ENOENT);
1659                 }
1660                 return root;
1661         }
1662
1663         key.objectid = objectid;
1664         key.type = BTRFS_ROOT_ITEM_KEY;
1665         key.offset = (u64)-1;
1666         root = btrfs_read_tree_root(fs_info->tree_root, &key);
1667         if (IS_ERR(root))
1668                 return root;
1669
1670         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1671                 ret = -ENOENT;
1672                 goto fail;
1673         }
1674
1675         ret = btrfs_init_fs_root(root, anon_dev);
1676         if (ret)
1677                 goto fail;
1678
1679         path = btrfs_alloc_path();
1680         if (!path) {
1681                 ret = -ENOMEM;
1682                 goto fail;
1683         }
1684         key.objectid = BTRFS_ORPHAN_OBJECTID;
1685         key.type = BTRFS_ORPHAN_ITEM_KEY;
1686         key.offset = objectid;
1687
1688         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1689         btrfs_free_path(path);
1690         if (ret < 0)
1691                 goto fail;
1692         if (ret == 0)
1693                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1694
1695         ret = btrfs_insert_fs_root(fs_info, root);
1696         if (ret) {
1697                 if (ret == -EEXIST) {
1698                         btrfs_put_root(root);
1699                         goto again;
1700                 }
1701                 goto fail;
1702         }
1703         return root;
1704 fail:
1705         /*
1706          * If our caller provided us an anonymous device, then it's his
1707          * responsibility to free it in case we fail. So we have to set our
1708          * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root()
1709          * and once again by our caller.
1710          */
1711         if (anon_dev)
1712                 root->anon_dev = 0;
1713         btrfs_put_root(root);
1714         return ERR_PTR(ret);
1715 }
1716
1717 /*
1718  * Get in-memory reference of a root structure
1719  *
1720  * @objectid:   tree objectid
1721  * @check_ref:  if set, verify that the tree exists and the item has at least
1722  *              one reference
1723  */
1724 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1725                                      u64 objectid, bool check_ref)
1726 {
1727         return btrfs_get_root_ref(fs_info, objectid, 0, check_ref);
1728 }
1729
1730 /*
1731  * Get in-memory reference of a root structure, created as new, optionally pass
1732  * the anonymous block device id
1733  *
1734  * @objectid:   tree objectid
1735  * @anon_dev:   if zero, allocate a new anonymous block device or use the
1736  *              parameter value
1737  */
1738 struct btrfs_root *btrfs_get_new_fs_root(struct btrfs_fs_info *fs_info,
1739                                          u64 objectid, dev_t anon_dev)
1740 {
1741         return btrfs_get_root_ref(fs_info, objectid, anon_dev, true);
1742 }
1743
1744 /*
1745  * btrfs_get_fs_root_commit_root - return a root for the given objectid
1746  * @fs_info:    the fs_info
1747  * @objectid:   the objectid we need to lookup
1748  *
1749  * This is exclusively used for backref walking, and exists specifically because
1750  * of how qgroups does lookups.  Qgroups will do a backref lookup at delayed ref
1751  * creation time, which means we may have to read the tree_root in order to look
1752  * up a fs root that is not in memory.  If the root is not in memory we will
1753  * read the tree root commit root and look up the fs root from there.  This is a
1754  * temporary root, it will not be inserted into the radix tree as it doesn't
1755  * have the most uptodate information, it'll simply be discarded once the
1756  * backref code is finished using the root.
1757  */
1758 struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info,
1759                                                  struct btrfs_path *path,
1760                                                  u64 objectid)
1761 {
1762         struct btrfs_root *root;
1763         struct btrfs_key key;
1764
1765         ASSERT(path->search_commit_root && path->skip_locking);
1766
1767         /*
1768          * This can return -ENOENT if we ask for a root that doesn't exist, but
1769          * since this is called via the backref walking code we won't be looking
1770          * up a root that doesn't exist, unless there's corruption.  So if root
1771          * != NULL just return it.
1772          */
1773         root = btrfs_get_global_root(fs_info, objectid);
1774         if (root)
1775                 return root;
1776
1777         root = btrfs_lookup_fs_root(fs_info, objectid);
1778         if (root)
1779                 return root;
1780
1781         key.objectid = objectid;
1782         key.type = BTRFS_ROOT_ITEM_KEY;
1783         key.offset = (u64)-1;
1784         root = read_tree_root_path(fs_info->tree_root, path, &key);
1785         btrfs_release_path(path);
1786
1787         return root;
1788 }
1789
1790 static int cleaner_kthread(void *arg)
1791 {
1792         struct btrfs_fs_info *fs_info = arg;
1793         int again;
1794
1795         while (1) {
1796                 again = 0;
1797
1798                 set_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1799
1800                 /* Make the cleaner go to sleep early. */
1801                 if (btrfs_need_cleaner_sleep(fs_info))
1802                         goto sleep;
1803
1804                 /*
1805                  * Do not do anything if we might cause open_ctree() to block
1806                  * before we have finished mounting the filesystem.
1807                  */
1808                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1809                         goto sleep;
1810
1811                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1812                         goto sleep;
1813
1814                 /*
1815                  * Avoid the problem that we change the status of the fs
1816                  * during the above check and trylock.
1817                  */
1818                 if (btrfs_need_cleaner_sleep(fs_info)) {
1819                         mutex_unlock(&fs_info->cleaner_mutex);
1820                         goto sleep;
1821                 }
1822
1823                 btrfs_run_delayed_iputs(fs_info);
1824
1825                 again = btrfs_clean_one_deleted_snapshot(fs_info);
1826                 mutex_unlock(&fs_info->cleaner_mutex);
1827
1828                 /*
1829                  * The defragger has dealt with the R/O remount and umount,
1830                  * needn't do anything special here.
1831                  */
1832                 btrfs_run_defrag_inodes(fs_info);
1833
1834                 /*
1835                  * Acquires fs_info->reclaim_bgs_lock to avoid racing
1836                  * with relocation (btrfs_relocate_chunk) and relocation
1837                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1838                  * after acquiring fs_info->reclaim_bgs_lock. So we
1839                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1840                  * unused block groups.
1841                  */
1842                 btrfs_delete_unused_bgs(fs_info);
1843
1844                 /*
1845                  * Reclaim block groups in the reclaim_bgs list after we deleted
1846                  * all unused block_groups. This possibly gives us some more free
1847                  * space.
1848                  */
1849                 btrfs_reclaim_bgs(fs_info);
1850 sleep:
1851                 clear_and_wake_up_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1852                 if (kthread_should_park())
1853                         kthread_parkme();
1854                 if (kthread_should_stop())
1855                         return 0;
1856                 if (!again) {
1857                         set_current_state(TASK_INTERRUPTIBLE);
1858                         schedule();
1859                         __set_current_state(TASK_RUNNING);
1860                 }
1861         }
1862 }
1863
1864 static int transaction_kthread(void *arg)
1865 {
1866         struct btrfs_root *root = arg;
1867         struct btrfs_fs_info *fs_info = root->fs_info;
1868         struct btrfs_trans_handle *trans;
1869         struct btrfs_transaction *cur;
1870         u64 transid;
1871         time64_t delta;
1872         unsigned long delay;
1873         bool cannot_commit;
1874
1875         do {
1876                 cannot_commit = false;
1877                 delay = msecs_to_jiffies(fs_info->commit_interval * 1000);
1878                 mutex_lock(&fs_info->transaction_kthread_mutex);
1879
1880                 spin_lock(&fs_info->trans_lock);
1881                 cur = fs_info->running_transaction;
1882                 if (!cur) {
1883                         spin_unlock(&fs_info->trans_lock);
1884                         goto sleep;
1885                 }
1886
1887                 delta = ktime_get_seconds() - cur->start_time;
1888                 if (!test_and_clear_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags) &&
1889                     cur->state < TRANS_STATE_COMMIT_START &&
1890                     delta < fs_info->commit_interval) {
1891                         spin_unlock(&fs_info->trans_lock);
1892                         delay -= msecs_to_jiffies((delta - 1) * 1000);
1893                         delay = min(delay,
1894                                     msecs_to_jiffies(fs_info->commit_interval * 1000));
1895                         goto sleep;
1896                 }
1897                 transid = cur->transid;
1898                 spin_unlock(&fs_info->trans_lock);
1899
1900                 /* If the file system is aborted, this will always fail. */
1901                 trans = btrfs_attach_transaction(root);
1902                 if (IS_ERR(trans)) {
1903                         if (PTR_ERR(trans) != -ENOENT)
1904                                 cannot_commit = true;
1905                         goto sleep;
1906                 }
1907                 if (transid == trans->transid) {
1908                         btrfs_commit_transaction(trans);
1909                 } else {
1910                         btrfs_end_transaction(trans);
1911                 }
1912 sleep:
1913                 wake_up_process(fs_info->cleaner_kthread);
1914                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1915
1916                 if (BTRFS_FS_ERROR(fs_info))
1917                         btrfs_cleanup_transaction(fs_info);
1918                 if (!kthread_should_stop() &&
1919                                 (!btrfs_transaction_blocked(fs_info) ||
1920                                  cannot_commit))
1921                         schedule_timeout_interruptible(delay);
1922         } while (!kthread_should_stop());
1923         return 0;
1924 }
1925
1926 /*
1927  * This will find the highest generation in the array of root backups.  The
1928  * index of the highest array is returned, or -EINVAL if we can't find
1929  * anything.
1930  *
1931  * We check to make sure the array is valid by comparing the
1932  * generation of the latest  root in the array with the generation
1933  * in the super block.  If they don't match we pitch it.
1934  */
1935 static int find_newest_super_backup(struct btrfs_fs_info *info)
1936 {
1937         const u64 newest_gen = btrfs_super_generation(info->super_copy);
1938         u64 cur;
1939         struct btrfs_root_backup *root_backup;
1940         int i;
1941
1942         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1943                 root_backup = info->super_copy->super_roots + i;
1944                 cur = btrfs_backup_tree_root_gen(root_backup);
1945                 if (cur == newest_gen)
1946                         return i;
1947         }
1948
1949         return -EINVAL;
1950 }
1951
1952 /*
1953  * copy all the root pointers into the super backup array.
1954  * this will bump the backup pointer by one when it is
1955  * done
1956  */
1957 static void backup_super_roots(struct btrfs_fs_info *info)
1958 {
1959         const int next_backup = info->backup_root_index;
1960         struct btrfs_root_backup *root_backup;
1961
1962         root_backup = info->super_for_commit->super_roots + next_backup;
1963
1964         /*
1965          * make sure all of our padding and empty slots get zero filled
1966          * regardless of which ones we use today
1967          */
1968         memset(root_backup, 0, sizeof(*root_backup));
1969
1970         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1971
1972         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1973         btrfs_set_backup_tree_root_gen(root_backup,
1974                                btrfs_header_generation(info->tree_root->node));
1975
1976         btrfs_set_backup_tree_root_level(root_backup,
1977                                btrfs_header_level(info->tree_root->node));
1978
1979         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1980         btrfs_set_backup_chunk_root_gen(root_backup,
1981                                btrfs_header_generation(info->chunk_root->node));
1982         btrfs_set_backup_chunk_root_level(root_backup,
1983                                btrfs_header_level(info->chunk_root->node));
1984
1985         if (btrfs_fs_incompat(info, EXTENT_TREE_V2)) {
1986                 btrfs_set_backup_block_group_root(root_backup,
1987                                         info->block_group_root->node->start);
1988                 btrfs_set_backup_block_group_root_gen(root_backup,
1989                         btrfs_header_generation(info->block_group_root->node));
1990                 btrfs_set_backup_block_group_root_level(root_backup,
1991                         btrfs_header_level(info->block_group_root->node));
1992         } else {
1993                 struct btrfs_root *extent_root = btrfs_extent_root(info, 0);
1994                 struct btrfs_root *csum_root = btrfs_csum_root(info, 0);
1995
1996                 btrfs_set_backup_extent_root(root_backup,
1997                                              extent_root->node->start);
1998                 btrfs_set_backup_extent_root_gen(root_backup,
1999                                 btrfs_header_generation(extent_root->node));
2000                 btrfs_set_backup_extent_root_level(root_backup,
2001                                         btrfs_header_level(extent_root->node));
2002
2003                 btrfs_set_backup_csum_root(root_backup, csum_root->node->start);
2004                 btrfs_set_backup_csum_root_gen(root_backup,
2005                                                btrfs_header_generation(csum_root->node));
2006                 btrfs_set_backup_csum_root_level(root_backup,
2007                                                  btrfs_header_level(csum_root->node));
2008         }
2009
2010         /*
2011          * we might commit during log recovery, which happens before we set
2012          * the fs_root.  Make sure it is valid before we fill it in.
2013          */
2014         if (info->fs_root && info->fs_root->node) {
2015                 btrfs_set_backup_fs_root(root_backup,
2016                                          info->fs_root->node->start);
2017                 btrfs_set_backup_fs_root_gen(root_backup,
2018                                btrfs_header_generation(info->fs_root->node));
2019                 btrfs_set_backup_fs_root_level(root_backup,
2020                                btrfs_header_level(info->fs_root->node));
2021         }
2022
2023         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
2024         btrfs_set_backup_dev_root_gen(root_backup,
2025                                btrfs_header_generation(info->dev_root->node));
2026         btrfs_set_backup_dev_root_level(root_backup,
2027                                        btrfs_header_level(info->dev_root->node));
2028
2029         btrfs_set_backup_total_bytes(root_backup,
2030                              btrfs_super_total_bytes(info->super_copy));
2031         btrfs_set_backup_bytes_used(root_backup,
2032                              btrfs_super_bytes_used(info->super_copy));
2033         btrfs_set_backup_num_devices(root_backup,
2034                              btrfs_super_num_devices(info->super_copy));
2035
2036         /*
2037          * if we don't copy this out to the super_copy, it won't get remembered
2038          * for the next commit
2039          */
2040         memcpy(&info->super_copy->super_roots,
2041                &info->super_for_commit->super_roots,
2042                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
2043 }
2044
2045 /*
2046  * read_backup_root - Reads a backup root based on the passed priority. Prio 0
2047  * is the newest, prio 1/2/3 are 2nd newest/3rd newest/4th (oldest) backup roots
2048  *
2049  * fs_info - filesystem whose backup roots need to be read
2050  * priority - priority of backup root required
2051  *
2052  * Returns backup root index on success and -EINVAL otherwise.
2053  */
2054 static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority)
2055 {
2056         int backup_index = find_newest_super_backup(fs_info);
2057         struct btrfs_super_block *super = fs_info->super_copy;
2058         struct btrfs_root_backup *root_backup;
2059
2060         if (priority < BTRFS_NUM_BACKUP_ROOTS && backup_index >= 0) {
2061                 if (priority == 0)
2062                         return backup_index;
2063
2064                 backup_index = backup_index + BTRFS_NUM_BACKUP_ROOTS - priority;
2065                 backup_index %= BTRFS_NUM_BACKUP_ROOTS;
2066         } else {
2067                 return -EINVAL;
2068         }
2069
2070         root_backup = super->super_roots + backup_index;
2071
2072         btrfs_set_super_generation(super,
2073                                    btrfs_backup_tree_root_gen(root_backup));
2074         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
2075         btrfs_set_super_root_level(super,
2076                                    btrfs_backup_tree_root_level(root_backup));
2077         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
2078
2079         /*
2080          * Fixme: the total bytes and num_devices need to match or we should
2081          * need a fsck
2082          */
2083         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
2084         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
2085
2086         return backup_index;
2087 }
2088
2089 /* helper to cleanup workers */
2090 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
2091 {
2092         btrfs_destroy_workqueue(fs_info->fixup_workers);
2093         btrfs_destroy_workqueue(fs_info->delalloc_workers);
2094         btrfs_destroy_workqueue(fs_info->hipri_workers);
2095         btrfs_destroy_workqueue(fs_info->workers);
2096         if (fs_info->endio_workers)
2097                 destroy_workqueue(fs_info->endio_workers);
2098         if (fs_info->endio_raid56_workers)
2099                 destroy_workqueue(fs_info->endio_raid56_workers);
2100         if (fs_info->rmw_workers)
2101                 destroy_workqueue(fs_info->rmw_workers);
2102         if (fs_info->compressed_write_workers)
2103                 destroy_workqueue(fs_info->compressed_write_workers);
2104         btrfs_destroy_workqueue(fs_info->endio_write_workers);
2105         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
2106         btrfs_destroy_workqueue(fs_info->delayed_workers);
2107         btrfs_destroy_workqueue(fs_info->caching_workers);
2108         btrfs_destroy_workqueue(fs_info->flush_workers);
2109         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
2110         if (fs_info->discard_ctl.discard_workers)
2111                 destroy_workqueue(fs_info->discard_ctl.discard_workers);
2112         /*
2113          * Now that all other work queues are destroyed, we can safely destroy
2114          * the queues used for metadata I/O, since tasks from those other work
2115          * queues can do metadata I/O operations.
2116          */
2117         if (fs_info->endio_meta_workers)
2118                 destroy_workqueue(fs_info->endio_meta_workers);
2119 }
2120
2121 static void free_root_extent_buffers(struct btrfs_root *root)
2122 {
2123         if (root) {
2124                 free_extent_buffer(root->node);
2125                 free_extent_buffer(root->commit_root);
2126                 root->node = NULL;
2127                 root->commit_root = NULL;
2128         }
2129 }
2130
2131 static void free_global_root_pointers(struct btrfs_fs_info *fs_info)
2132 {
2133         struct btrfs_root *root, *tmp;
2134
2135         rbtree_postorder_for_each_entry_safe(root, tmp,
2136                                              &fs_info->global_root_tree,
2137                                              rb_node)
2138                 free_root_extent_buffers(root);
2139 }
2140
2141 /* helper to cleanup tree roots */
2142 static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root)
2143 {
2144         free_root_extent_buffers(info->tree_root);
2145
2146         free_global_root_pointers(info);
2147         free_root_extent_buffers(info->dev_root);
2148         free_root_extent_buffers(info->quota_root);
2149         free_root_extent_buffers(info->uuid_root);
2150         free_root_extent_buffers(info->fs_root);
2151         free_root_extent_buffers(info->data_reloc_root);
2152         free_root_extent_buffers(info->block_group_root);
2153         if (free_chunk_root)
2154                 free_root_extent_buffers(info->chunk_root);
2155 }
2156
2157 void btrfs_put_root(struct btrfs_root *root)
2158 {
2159         if (!root)
2160                 return;
2161
2162         if (refcount_dec_and_test(&root->refs)) {
2163                 WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
2164                 WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state));
2165                 if (root->anon_dev)
2166                         free_anon_bdev(root->anon_dev);
2167                 btrfs_drew_lock_destroy(&root->snapshot_lock);
2168                 free_root_extent_buffers(root);
2169 #ifdef CONFIG_BTRFS_DEBUG
2170                 spin_lock(&root->fs_info->fs_roots_radix_lock);
2171                 list_del_init(&root->leak_list);
2172                 spin_unlock(&root->fs_info->fs_roots_radix_lock);
2173 #endif
2174                 kfree(root);
2175         }
2176 }
2177
2178 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
2179 {
2180         int ret;
2181         struct btrfs_root *gang[8];
2182         int i;
2183
2184         while (!list_empty(&fs_info->dead_roots)) {
2185                 gang[0] = list_entry(fs_info->dead_roots.next,
2186                                      struct btrfs_root, root_list);
2187                 list_del(&gang[0]->root_list);
2188
2189                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state))
2190                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
2191                 btrfs_put_root(gang[0]);
2192         }
2193
2194         while (1) {
2195                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
2196                                              (void **)gang, 0,
2197                                              ARRAY_SIZE(gang));
2198                 if (!ret)
2199                         break;
2200                 for (i = 0; i < ret; i++)
2201                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2202         }
2203 }
2204
2205 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2206 {
2207         mutex_init(&fs_info->scrub_lock);
2208         atomic_set(&fs_info->scrubs_running, 0);
2209         atomic_set(&fs_info->scrub_pause_req, 0);
2210         atomic_set(&fs_info->scrubs_paused, 0);
2211         atomic_set(&fs_info->scrub_cancel_req, 0);
2212         init_waitqueue_head(&fs_info->scrub_pause_wait);
2213         refcount_set(&fs_info->scrub_workers_refcnt, 0);
2214 }
2215
2216 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2217 {
2218         spin_lock_init(&fs_info->balance_lock);
2219         mutex_init(&fs_info->balance_mutex);
2220         atomic_set(&fs_info->balance_pause_req, 0);
2221         atomic_set(&fs_info->balance_cancel_req, 0);
2222         fs_info->balance_ctl = NULL;
2223         init_waitqueue_head(&fs_info->balance_wait_q);
2224         atomic_set(&fs_info->reloc_cancel_req, 0);
2225 }
2226
2227 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2228 {
2229         struct inode *inode = fs_info->btree_inode;
2230
2231         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2232         set_nlink(inode, 1);
2233         /*
2234          * we set the i_size on the btree inode to the max possible int.
2235          * the real end of the address space is determined by all of
2236          * the devices in the system
2237          */
2238         inode->i_size = OFFSET_MAX;
2239         inode->i_mapping->a_ops = &btree_aops;
2240
2241         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2242         extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
2243                             IO_TREE_BTREE_INODE_IO, inode);
2244         BTRFS_I(inode)->io_tree.track_uptodate = false;
2245         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2246
2247         BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root);
2248         BTRFS_I(inode)->location.objectid = BTRFS_BTREE_INODE_OBJECTID;
2249         BTRFS_I(inode)->location.type = 0;
2250         BTRFS_I(inode)->location.offset = 0;
2251         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2252         btrfs_insert_inode_hash(inode);
2253 }
2254
2255 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2256 {
2257         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2258         init_rwsem(&fs_info->dev_replace.rwsem);
2259         init_waitqueue_head(&fs_info->dev_replace.replace_wait);
2260 }
2261
2262 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2263 {
2264         spin_lock_init(&fs_info->qgroup_lock);
2265         mutex_init(&fs_info->qgroup_ioctl_lock);
2266         fs_info->qgroup_tree = RB_ROOT;
2267         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2268         fs_info->qgroup_seq = 1;
2269         fs_info->qgroup_ulist = NULL;
2270         fs_info->qgroup_rescan_running = false;
2271         mutex_init(&fs_info->qgroup_rescan_lock);
2272 }
2273
2274 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
2275 {
2276         u32 max_active = fs_info->thread_pool_size;
2277         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2278
2279         fs_info->workers =
2280                 btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16);
2281         fs_info->hipri_workers =
2282                 btrfs_alloc_workqueue(fs_info, "worker-high",
2283                                       flags | WQ_HIGHPRI, max_active, 16);
2284
2285         fs_info->delalloc_workers =
2286                 btrfs_alloc_workqueue(fs_info, "delalloc",
2287                                       flags, max_active, 2);
2288
2289         fs_info->flush_workers =
2290                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2291                                       flags, max_active, 0);
2292
2293         fs_info->caching_workers =
2294                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2295
2296         fs_info->fixup_workers =
2297                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2298
2299         fs_info->endio_workers =
2300                 alloc_workqueue("btrfs-endio", flags, max_active);
2301         fs_info->endio_meta_workers =
2302                 alloc_workqueue("btrfs-endio-meta", flags, max_active);
2303         fs_info->endio_raid56_workers =
2304                 alloc_workqueue("btrfs-endio-raid56", flags, max_active);
2305         fs_info->rmw_workers = alloc_workqueue("btrfs-rmw", flags, max_active);
2306         fs_info->endio_write_workers =
2307                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2308                                       max_active, 2);
2309         fs_info->compressed_write_workers =
2310                 alloc_workqueue("btrfs-compressed-write", flags, max_active);
2311         fs_info->endio_freespace_worker =
2312                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2313                                       max_active, 0);
2314         fs_info->delayed_workers =
2315                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2316                                       max_active, 0);
2317         fs_info->qgroup_rescan_workers =
2318                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2319         fs_info->discard_ctl.discard_workers =
2320                 alloc_workqueue("btrfs_discard", WQ_UNBOUND | WQ_FREEZABLE, 1);
2321
2322         if (!(fs_info->workers && fs_info->hipri_workers &&
2323               fs_info->delalloc_workers && fs_info->flush_workers &&
2324               fs_info->endio_workers && fs_info->endio_meta_workers &&
2325               fs_info->compressed_write_workers &&
2326               fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
2327               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2328               fs_info->caching_workers && fs_info->fixup_workers &&
2329               fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
2330               fs_info->discard_ctl.discard_workers)) {
2331                 return -ENOMEM;
2332         }
2333
2334         return 0;
2335 }
2336
2337 static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
2338 {
2339         struct crypto_shash *csum_shash;
2340         const char *csum_driver = btrfs_super_csum_driver(csum_type);
2341
2342         csum_shash = crypto_alloc_shash(csum_driver, 0, 0);
2343
2344         if (IS_ERR(csum_shash)) {
2345                 btrfs_err(fs_info, "error allocating %s hash for checksum",
2346                           csum_driver);
2347                 return PTR_ERR(csum_shash);
2348         }
2349
2350         fs_info->csum_shash = csum_shash;
2351
2352         btrfs_info(fs_info, "using %s (%s) checksum algorithm",
2353                         btrfs_super_csum_name(csum_type),
2354                         crypto_shash_driver_name(csum_shash));
2355         return 0;
2356 }
2357
2358 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2359                             struct btrfs_fs_devices *fs_devices)
2360 {
2361         int ret;
2362         struct btrfs_root *log_tree_root;
2363         struct btrfs_super_block *disk_super = fs_info->super_copy;
2364         u64 bytenr = btrfs_super_log_root(disk_super);
2365         int level = btrfs_super_log_root_level(disk_super);
2366
2367         if (fs_devices->rw_devices == 0) {
2368                 btrfs_warn(fs_info, "log replay required on RO media");
2369                 return -EIO;
2370         }
2371
2372         log_tree_root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID,
2373                                          GFP_KERNEL);
2374         if (!log_tree_root)
2375                 return -ENOMEM;
2376
2377         log_tree_root->node = read_tree_block(fs_info, bytenr,
2378                                               BTRFS_TREE_LOG_OBJECTID,
2379                                               fs_info->generation + 1, level,
2380                                               NULL);
2381         if (IS_ERR(log_tree_root->node)) {
2382                 btrfs_warn(fs_info, "failed to read log tree");
2383                 ret = PTR_ERR(log_tree_root->node);
2384                 log_tree_root->node = NULL;
2385                 btrfs_put_root(log_tree_root);
2386                 return ret;
2387         }
2388         if (!extent_buffer_uptodate(log_tree_root->node)) {
2389                 btrfs_err(fs_info, "failed to read log tree");
2390                 btrfs_put_root(log_tree_root);
2391                 return -EIO;
2392         }
2393
2394         /* returns with log_tree_root freed on success */
2395         ret = btrfs_recover_log_trees(log_tree_root);
2396         if (ret) {
2397                 btrfs_handle_fs_error(fs_info, ret,
2398                                       "Failed to recover log tree");
2399                 btrfs_put_root(log_tree_root);
2400                 return ret;
2401         }
2402
2403         if (sb_rdonly(fs_info->sb)) {
2404                 ret = btrfs_commit_super(fs_info);
2405                 if (ret)
2406                         return ret;
2407         }
2408
2409         return 0;
2410 }
2411
2412 static int load_global_roots_objectid(struct btrfs_root *tree_root,
2413                                       struct btrfs_path *path, u64 objectid,
2414                                       const char *name)
2415 {
2416         struct btrfs_fs_info *fs_info = tree_root->fs_info;
2417         struct btrfs_root *root;
2418         u64 max_global_id = 0;
2419         int ret;
2420         struct btrfs_key key = {
2421                 .objectid = objectid,
2422                 .type = BTRFS_ROOT_ITEM_KEY,
2423                 .offset = 0,
2424         };
2425         bool found = false;
2426
2427         /* If we have IGNOREDATACSUMS skip loading these roots. */
2428         if (objectid == BTRFS_CSUM_TREE_OBJECTID &&
2429             btrfs_test_opt(fs_info, IGNOREDATACSUMS)) {
2430                 set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2431                 return 0;
2432         }
2433
2434         while (1) {
2435                 ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0);
2436                 if (ret < 0)
2437                         break;
2438
2439                 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2440                         ret = btrfs_next_leaf(tree_root, path);
2441                         if (ret) {
2442                                 if (ret > 0)
2443                                         ret = 0;
2444                                 break;
2445                         }
2446                 }
2447                 ret = 0;
2448
2449                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2450                 if (key.objectid != objectid)
2451                         break;
2452                 btrfs_release_path(path);
2453
2454                 /*
2455                  * Just worry about this for extent tree, it'll be the same for
2456                  * everybody.
2457                  */
2458                 if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2459                         max_global_id = max(max_global_id, key.offset);
2460
2461                 found = true;
2462                 root = read_tree_root_path(tree_root, path, &key);
2463                 if (IS_ERR(root)) {
2464                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2465                                 ret = PTR_ERR(root);
2466                         break;
2467                 }
2468                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2469                 ret = btrfs_global_root_insert(root);
2470                 if (ret) {
2471                         btrfs_put_root(root);
2472                         break;
2473                 }
2474                 key.offset++;
2475         }
2476         btrfs_release_path(path);
2477
2478         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2479                 fs_info->nr_global_roots = max_global_id + 1;
2480
2481         if (!found || ret) {
2482                 if (objectid == BTRFS_CSUM_TREE_OBJECTID)
2483                         set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2484
2485                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2486                         ret = ret ? ret : -ENOENT;
2487                 else
2488                         ret = 0;
2489                 btrfs_err(fs_info, "failed to load root %s", name);
2490         }
2491         return ret;
2492 }
2493
2494 static int load_global_roots(struct btrfs_root *tree_root)
2495 {
2496         struct btrfs_path *path;
2497         int ret = 0;
2498
2499         path = btrfs_alloc_path();
2500         if (!path)
2501                 return -ENOMEM;
2502
2503         ret = load_global_roots_objectid(tree_root, path,
2504                                          BTRFS_EXTENT_TREE_OBJECTID, "extent");
2505         if (ret)
2506                 goto out;
2507         ret = load_global_roots_objectid(tree_root, path,
2508                                          BTRFS_CSUM_TREE_OBJECTID, "csum");
2509         if (ret)
2510                 goto out;
2511         if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE))
2512                 goto out;
2513         ret = load_global_roots_objectid(tree_root, path,
2514                                          BTRFS_FREE_SPACE_TREE_OBJECTID,
2515                                          "free space");
2516 out:
2517         btrfs_free_path(path);
2518         return ret;
2519 }
2520
2521 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2522 {
2523         struct btrfs_root *tree_root = fs_info->tree_root;
2524         struct btrfs_root *root;
2525         struct btrfs_key location;
2526         int ret;
2527
2528         BUG_ON(!fs_info->tree_root);
2529
2530         ret = load_global_roots(tree_root);
2531         if (ret)
2532                 return ret;
2533
2534         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2535         location.type = BTRFS_ROOT_ITEM_KEY;
2536         location.offset = 0;
2537
2538         root = btrfs_read_tree_root(tree_root, &location);
2539         if (IS_ERR(root)) {
2540                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2541                         ret = PTR_ERR(root);
2542                         goto out;
2543                 }
2544         } else {
2545                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2546                 fs_info->dev_root = root;
2547         }
2548         /* Initialize fs_info for all devices in any case */
2549         btrfs_init_devices_late(fs_info);
2550
2551         /*
2552          * This tree can share blocks with some other fs tree during relocation
2553          * and we need a proper setup by btrfs_get_fs_root
2554          */
2555         root = btrfs_get_fs_root(tree_root->fs_info,
2556                                  BTRFS_DATA_RELOC_TREE_OBJECTID, true);
2557         if (IS_ERR(root)) {
2558                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2559                         ret = PTR_ERR(root);
2560                         goto out;
2561                 }
2562         } else {
2563                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2564                 fs_info->data_reloc_root = root;
2565         }
2566
2567         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2568         root = btrfs_read_tree_root(tree_root, &location);
2569         if (!IS_ERR(root)) {
2570                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2571                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2572                 fs_info->quota_root = root;
2573         }
2574
2575         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2576         root = btrfs_read_tree_root(tree_root, &location);
2577         if (IS_ERR(root)) {
2578                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2579                         ret = PTR_ERR(root);
2580                         if (ret != -ENOENT)
2581                                 goto out;
2582                 }
2583         } else {
2584                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2585                 fs_info->uuid_root = root;
2586         }
2587
2588         return 0;
2589 out:
2590         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2591                    location.objectid, ret);
2592         return ret;
2593 }
2594
2595 /*
2596  * Real super block validation
2597  * NOTE: super csum type and incompat features will not be checked here.
2598  *
2599  * @sb:         super block to check
2600  * @mirror_num: the super block number to check its bytenr:
2601  *              0       the primary (1st) sb
2602  *              1, 2    2nd and 3rd backup copy
2603  *             -1       skip bytenr check
2604  */
2605 static int validate_super(struct btrfs_fs_info *fs_info,
2606                             struct btrfs_super_block *sb, int mirror_num)
2607 {
2608         u64 nodesize = btrfs_super_nodesize(sb);
2609         u64 sectorsize = btrfs_super_sectorsize(sb);
2610         int ret = 0;
2611
2612         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2613                 btrfs_err(fs_info, "no valid FS found");
2614                 ret = -EINVAL;
2615         }
2616         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2617                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2618                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2619                 ret = -EINVAL;
2620         }
2621         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2622                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2623                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2624                 ret = -EINVAL;
2625         }
2626         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2627                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2628                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2629                 ret = -EINVAL;
2630         }
2631         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2632                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2633                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2634                 ret = -EINVAL;
2635         }
2636
2637         /*
2638          * Check sectorsize and nodesize first, other check will need it.
2639          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2640          */
2641         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2642             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2643                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2644                 ret = -EINVAL;
2645         }
2646
2647         /*
2648          * We only support at most two sectorsizes: 4K and PAGE_SIZE.
2649          *
2650          * We can support 16K sectorsize with 64K page size without problem,
2651          * but such sectorsize/pagesize combination doesn't make much sense.
2652          * 4K will be our future standard, PAGE_SIZE is supported from the very
2653          * beginning.
2654          */
2655         if (sectorsize > PAGE_SIZE || (sectorsize != SZ_4K && sectorsize != PAGE_SIZE)) {
2656                 btrfs_err(fs_info,
2657                         "sectorsize %llu not yet supported for page size %lu",
2658                         sectorsize, PAGE_SIZE);
2659                 ret = -EINVAL;
2660         }
2661
2662         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2663             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2664                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2665                 ret = -EINVAL;
2666         }
2667         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2668                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2669                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2670                 ret = -EINVAL;
2671         }
2672
2673         /* Root alignment check */
2674         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2675                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2676                            btrfs_super_root(sb));
2677                 ret = -EINVAL;
2678         }
2679         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2680                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2681                            btrfs_super_chunk_root(sb));
2682                 ret = -EINVAL;
2683         }
2684         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2685                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2686                            btrfs_super_log_root(sb));
2687                 ret = -EINVAL;
2688         }
2689
2690         if (memcmp(fs_info->fs_devices->fsid, fs_info->super_copy->fsid,
2691                    BTRFS_FSID_SIZE)) {
2692                 btrfs_err(fs_info,
2693                 "superblock fsid doesn't match fsid of fs_devices: %pU != %pU",
2694                         fs_info->super_copy->fsid, fs_info->fs_devices->fsid);
2695                 ret = -EINVAL;
2696         }
2697
2698         if (btrfs_fs_incompat(fs_info, METADATA_UUID) &&
2699             memcmp(fs_info->fs_devices->metadata_uuid,
2700                    fs_info->super_copy->metadata_uuid, BTRFS_FSID_SIZE)) {
2701                 btrfs_err(fs_info,
2702 "superblock metadata_uuid doesn't match metadata uuid of fs_devices: %pU != %pU",
2703                         fs_info->super_copy->metadata_uuid,
2704                         fs_info->fs_devices->metadata_uuid);
2705                 ret = -EINVAL;
2706         }
2707
2708         if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
2709                    BTRFS_FSID_SIZE) != 0) {
2710                 btrfs_err(fs_info,
2711                         "dev_item UUID does not match metadata fsid: %pU != %pU",
2712                         fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid);
2713                 ret = -EINVAL;
2714         }
2715
2716         /*
2717          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2718          * done later
2719          */
2720         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2721                 btrfs_err(fs_info, "bytes_used is too small %llu",
2722                           btrfs_super_bytes_used(sb));
2723                 ret = -EINVAL;
2724         }
2725         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2726                 btrfs_err(fs_info, "invalid stripesize %u",
2727                           btrfs_super_stripesize(sb));
2728                 ret = -EINVAL;
2729         }
2730         if (btrfs_super_num_devices(sb) > (1UL << 31))
2731                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2732                            btrfs_super_num_devices(sb));
2733         if (btrfs_super_num_devices(sb) == 0) {
2734                 btrfs_err(fs_info, "number of devices is 0");
2735                 ret = -EINVAL;
2736         }
2737
2738         if (mirror_num >= 0 &&
2739             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2740                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2741                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2742                 ret = -EINVAL;
2743         }
2744
2745         /*
2746          * Obvious sys_chunk_array corruptions, it must hold at least one key
2747          * and one chunk
2748          */
2749         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2750                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2751                           btrfs_super_sys_array_size(sb),
2752                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2753                 ret = -EINVAL;
2754         }
2755         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2756                         + sizeof(struct btrfs_chunk)) {
2757                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2758                           btrfs_super_sys_array_size(sb),
2759                           sizeof(struct btrfs_disk_key)
2760                           + sizeof(struct btrfs_chunk));
2761                 ret = -EINVAL;
2762         }
2763
2764         /*
2765          * The generation is a global counter, we'll trust it more than the others
2766          * but it's still possible that it's the one that's wrong.
2767          */
2768         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2769                 btrfs_warn(fs_info,
2770                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2771                         btrfs_super_generation(sb),
2772                         btrfs_super_chunk_root_generation(sb));
2773         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2774             && btrfs_super_cache_generation(sb) != (u64)-1)
2775                 btrfs_warn(fs_info,
2776                         "suspicious: generation < cache_generation: %llu < %llu",
2777                         btrfs_super_generation(sb),
2778                         btrfs_super_cache_generation(sb));
2779
2780         return ret;
2781 }
2782
2783 /*
2784  * Validation of super block at mount time.
2785  * Some checks already done early at mount time, like csum type and incompat
2786  * flags will be skipped.
2787  */
2788 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2789 {
2790         return validate_super(fs_info, fs_info->super_copy, 0);
2791 }
2792
2793 /*
2794  * Validation of super block at write time.
2795  * Some checks like bytenr check will be skipped as their values will be
2796  * overwritten soon.
2797  * Extra checks like csum type and incompat flags will be done here.
2798  */
2799 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2800                                       struct btrfs_super_block *sb)
2801 {
2802         int ret;
2803
2804         ret = validate_super(fs_info, sb, -1);
2805         if (ret < 0)
2806                 goto out;
2807         if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) {
2808                 ret = -EUCLEAN;
2809                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2810                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2811                 goto out;
2812         }
2813         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2814                 ret = -EUCLEAN;
2815                 btrfs_err(fs_info,
2816                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2817                           btrfs_super_incompat_flags(sb),
2818                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2819                 goto out;
2820         }
2821 out:
2822         if (ret < 0)
2823                 btrfs_err(fs_info,
2824                 "super block corruption detected before writing it to disk");
2825         return ret;
2826 }
2827
2828 static int load_super_root(struct btrfs_root *root, u64 bytenr, u64 gen, int level)
2829 {
2830         int ret = 0;
2831
2832         root->node = read_tree_block(root->fs_info, bytenr,
2833                                      root->root_key.objectid, gen, level, NULL);
2834         if (IS_ERR(root->node)) {
2835                 ret = PTR_ERR(root->node);
2836                 root->node = NULL;
2837                 return ret;
2838         }
2839         if (!extent_buffer_uptodate(root->node)) {
2840                 free_extent_buffer(root->node);
2841                 root->node = NULL;
2842                 return -EIO;
2843         }
2844
2845         btrfs_set_root_node(&root->root_item, root->node);
2846         root->commit_root = btrfs_root_node(root);
2847         btrfs_set_root_refs(&root->root_item, 1);
2848         return ret;
2849 }
2850
2851 static int load_important_roots(struct btrfs_fs_info *fs_info)
2852 {
2853         struct btrfs_super_block *sb = fs_info->super_copy;
2854         u64 gen, bytenr;
2855         int level, ret;
2856
2857         bytenr = btrfs_super_root(sb);
2858         gen = btrfs_super_generation(sb);
2859         level = btrfs_super_root_level(sb);
2860         ret = load_super_root(fs_info->tree_root, bytenr, gen, level);
2861         if (ret) {
2862                 btrfs_warn(fs_info, "couldn't read tree root");
2863                 return ret;
2864         }
2865
2866         if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))
2867                 return 0;
2868
2869         bytenr = btrfs_super_block_group_root(sb);
2870         gen = btrfs_super_block_group_root_generation(sb);
2871         level = btrfs_super_block_group_root_level(sb);
2872         ret = load_super_root(fs_info->block_group_root, bytenr, gen, level);
2873         if (ret)
2874                 btrfs_warn(fs_info, "couldn't read block group root");
2875         return ret;
2876 }
2877
2878 static int __cold init_tree_roots(struct btrfs_fs_info *fs_info)
2879 {
2880         int backup_index = find_newest_super_backup(fs_info);
2881         struct btrfs_super_block *sb = fs_info->super_copy;
2882         struct btrfs_root *tree_root = fs_info->tree_root;
2883         bool handle_error = false;
2884         int ret = 0;
2885         int i;
2886
2887         if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
2888                 struct btrfs_root *root;
2889
2890                 root = btrfs_alloc_root(fs_info, BTRFS_BLOCK_GROUP_TREE_OBJECTID,
2891                                         GFP_KERNEL);
2892                 if (!root)
2893                         return -ENOMEM;
2894                 fs_info->block_group_root = root;
2895         }
2896
2897         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
2898                 if (handle_error) {
2899                         if (!IS_ERR(tree_root->node))
2900                                 free_extent_buffer(tree_root->node);
2901                         tree_root->node = NULL;
2902
2903                         if (!btrfs_test_opt(fs_info, USEBACKUPROOT))
2904                                 break;
2905
2906                         free_root_pointers(fs_info, 0);
2907
2908                         /*
2909                          * Don't use the log in recovery mode, it won't be
2910                          * valid
2911                          */
2912                         btrfs_set_super_log_root(sb, 0);
2913
2914                         /* We can't trust the free space cache either */
2915                         btrfs_set_opt(fs_info->mount_opt, CLEAR_CACHE);
2916
2917                         ret = read_backup_root(fs_info, i);
2918                         backup_index = ret;
2919                         if (ret < 0)
2920                                 return ret;
2921                 }
2922
2923                 ret = load_important_roots(fs_info);
2924                 if (ret) {
2925                         handle_error = true;
2926                         continue;
2927                 }
2928
2929                 /*
2930                  * No need to hold btrfs_root::objectid_mutex since the fs
2931                  * hasn't been fully initialised and we are the only user
2932                  */
2933                 ret = btrfs_init_root_free_objectid(tree_root);
2934                 if (ret < 0) {
2935                         handle_error = true;
2936                         continue;
2937                 }
2938
2939                 ASSERT(tree_root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
2940
2941                 ret = btrfs_read_roots(fs_info);
2942                 if (ret < 0) {
2943                         handle_error = true;
2944                         continue;
2945                 }
2946
2947                 /* All successful */
2948                 fs_info->generation = btrfs_header_generation(tree_root->node);
2949                 fs_info->last_trans_committed = fs_info->generation;
2950                 fs_info->last_reloc_trans = 0;
2951
2952                 /* Always begin writing backup roots after the one being used */
2953                 if (backup_index < 0) {
2954                         fs_info->backup_root_index = 0;
2955                 } else {
2956                         fs_info->backup_root_index = backup_index + 1;
2957                         fs_info->backup_root_index %= BTRFS_NUM_BACKUP_ROOTS;
2958                 }
2959                 break;
2960         }
2961
2962         return ret;
2963 }
2964
2965 void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
2966 {
2967         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2968         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2969         INIT_LIST_HEAD(&fs_info->trans_list);
2970         INIT_LIST_HEAD(&fs_info->dead_roots);
2971         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2972         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2973         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2974         spin_lock_init(&fs_info->delalloc_root_lock);
2975         spin_lock_init(&fs_info->trans_lock);
2976         spin_lock_init(&fs_info->fs_roots_radix_lock);
2977         spin_lock_init(&fs_info->delayed_iput_lock);
2978         spin_lock_init(&fs_info->defrag_inodes_lock);
2979         spin_lock_init(&fs_info->super_lock);
2980         spin_lock_init(&fs_info->buffer_lock);
2981         spin_lock_init(&fs_info->unused_bgs_lock);
2982         spin_lock_init(&fs_info->treelog_bg_lock);
2983         spin_lock_init(&fs_info->zone_active_bgs_lock);
2984         spin_lock_init(&fs_info->relocation_bg_lock);
2985         rwlock_init(&fs_info->tree_mod_log_lock);
2986         rwlock_init(&fs_info->global_root_lock);
2987         mutex_init(&fs_info->unused_bg_unpin_mutex);
2988         mutex_init(&fs_info->reclaim_bgs_lock);
2989         mutex_init(&fs_info->reloc_mutex);
2990         mutex_init(&fs_info->delalloc_root_mutex);
2991         mutex_init(&fs_info->zoned_meta_io_lock);
2992         mutex_init(&fs_info->zoned_data_reloc_io_lock);
2993         seqlock_init(&fs_info->profiles_lock);
2994
2995         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
2996         INIT_LIST_HEAD(&fs_info->space_info);
2997         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
2998         INIT_LIST_HEAD(&fs_info->unused_bgs);
2999         INIT_LIST_HEAD(&fs_info->reclaim_bgs);
3000         INIT_LIST_HEAD(&fs_info->zone_active_bgs);
3001 #ifdef CONFIG_BTRFS_DEBUG
3002         INIT_LIST_HEAD(&fs_info->allocated_roots);
3003         INIT_LIST_HEAD(&fs_info->allocated_ebs);
3004         spin_lock_init(&fs_info->eb_leak_lock);
3005 #endif
3006         extent_map_tree_init(&fs_info->mapping_tree);
3007         btrfs_init_block_rsv(&fs_info->global_block_rsv,
3008                              BTRFS_BLOCK_RSV_GLOBAL);
3009         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
3010         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
3011         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
3012         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
3013                              BTRFS_BLOCK_RSV_DELOPS);
3014         btrfs_init_block_rsv(&fs_info->delayed_refs_rsv,
3015                              BTRFS_BLOCK_RSV_DELREFS);
3016
3017         atomic_set(&fs_info->async_delalloc_pages, 0);
3018         atomic_set(&fs_info->defrag_running, 0);
3019         atomic_set(&fs_info->nr_delayed_iputs, 0);
3020         atomic64_set(&fs_info->tree_mod_seq, 0);
3021         fs_info->global_root_tree = RB_ROOT;
3022         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
3023         fs_info->metadata_ratio = 0;
3024         fs_info->defrag_inodes = RB_ROOT;
3025         atomic64_set(&fs_info->free_chunk_space, 0);
3026         fs_info->tree_mod_log = RB_ROOT;
3027         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
3028         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
3029         btrfs_init_ref_verify(fs_info);
3030
3031         fs_info->thread_pool_size = min_t(unsigned long,
3032                                           num_online_cpus() + 2, 8);
3033
3034         INIT_LIST_HEAD(&fs_info->ordered_roots);
3035         spin_lock_init(&fs_info->ordered_root_lock);
3036
3037         btrfs_init_scrub(fs_info);
3038 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3039         fs_info->check_integrity_print_mask = 0;
3040 #endif
3041         btrfs_init_balance(fs_info);
3042         btrfs_init_async_reclaim_work(fs_info);
3043
3044         rwlock_init(&fs_info->block_group_cache_lock);
3045         fs_info->block_group_cache_tree = RB_ROOT_CACHED;
3046
3047         extent_io_tree_init(fs_info, &fs_info->excluded_extents,
3048                             IO_TREE_FS_EXCLUDED_EXTENTS, NULL);
3049
3050         mutex_init(&fs_info->ordered_operations_mutex);
3051         mutex_init(&fs_info->tree_log_mutex);
3052         mutex_init(&fs_info->chunk_mutex);
3053         mutex_init(&fs_info->transaction_kthread_mutex);
3054         mutex_init(&fs_info->cleaner_mutex);
3055         mutex_init(&fs_info->ro_block_group_mutex);
3056         init_rwsem(&fs_info->commit_root_sem);
3057         init_rwsem(&fs_info->cleanup_work_sem);
3058         init_rwsem(&fs_info->subvol_sem);
3059         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
3060
3061         btrfs_init_dev_replace_locks(fs_info);
3062         btrfs_init_qgroup(fs_info);
3063         btrfs_discard_init(fs_info);
3064
3065         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
3066         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
3067
3068         init_waitqueue_head(&fs_info->transaction_throttle);
3069         init_waitqueue_head(&fs_info->transaction_wait);
3070         init_waitqueue_head(&fs_info->transaction_blocked_wait);
3071         init_waitqueue_head(&fs_info->async_submit_wait);
3072         init_waitqueue_head(&fs_info->delayed_iputs_wait);
3073
3074         /* Usable values until the real ones are cached from the superblock */
3075         fs_info->nodesize = 4096;
3076         fs_info->sectorsize = 4096;
3077         fs_info->sectorsize_bits = ilog2(4096);
3078         fs_info->stripesize = 4096;
3079
3080         fs_info->max_extent_size = BTRFS_MAX_EXTENT_SIZE;
3081
3082         spin_lock_init(&fs_info->swapfile_pins_lock);
3083         fs_info->swapfile_pins = RB_ROOT;
3084
3085         fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH;
3086         INIT_WORK(&fs_info->reclaim_bgs_work, btrfs_reclaim_bgs_work);
3087 }
3088
3089 static int init_mount_fs_info(struct btrfs_fs_info *fs_info, struct super_block *sb)
3090 {
3091         int ret;
3092
3093         fs_info->sb = sb;
3094         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
3095         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
3096
3097         ret = percpu_counter_init(&fs_info->ordered_bytes, 0, GFP_KERNEL);
3098         if (ret)
3099                 return ret;
3100
3101         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
3102         if (ret)
3103                 return ret;
3104
3105         fs_info->dirty_metadata_batch = PAGE_SIZE *
3106                                         (1 + ilog2(nr_cpu_ids));
3107
3108         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
3109         if (ret)
3110                 return ret;
3111
3112         ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0,
3113                         GFP_KERNEL);
3114         if (ret)
3115                 return ret;
3116
3117         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
3118                                         GFP_KERNEL);
3119         if (!fs_info->delayed_root)
3120                 return -ENOMEM;
3121         btrfs_init_delayed_root(fs_info->delayed_root);
3122
3123         if (sb_rdonly(sb))
3124                 set_bit(BTRFS_FS_STATE_RO, &fs_info->fs_state);
3125
3126         return btrfs_alloc_stripe_hash_table(fs_info);
3127 }
3128
3129 static int btrfs_uuid_rescan_kthread(void *data)
3130 {
3131         struct btrfs_fs_info *fs_info = data;
3132         int ret;
3133
3134         /*
3135          * 1st step is to iterate through the existing UUID tree and
3136          * to delete all entries that contain outdated data.
3137          * 2nd step is to add all missing entries to the UUID tree.
3138          */
3139         ret = btrfs_uuid_tree_iterate(fs_info);
3140         if (ret < 0) {
3141                 if (ret != -EINTR)
3142                         btrfs_warn(fs_info, "iterating uuid_tree failed %d",
3143                                    ret);
3144                 up(&fs_info->uuid_tree_rescan_sem);
3145                 return ret;
3146         }
3147         return btrfs_uuid_scan_kthread(data);
3148 }
3149
3150 static int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info)
3151 {
3152         struct task_struct *task;
3153
3154         down(&fs_info->uuid_tree_rescan_sem);
3155         task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid");
3156         if (IS_ERR(task)) {
3157                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
3158                 btrfs_warn(fs_info, "failed to start uuid_rescan task");
3159                 up(&fs_info->uuid_tree_rescan_sem);
3160                 return PTR_ERR(task);
3161         }
3162
3163         return 0;
3164 }
3165
3166 /*
3167  * Some options only have meaning at mount time and shouldn't persist across
3168  * remounts, or be displayed. Clear these at the end of mount and remount
3169  * code paths.
3170  */
3171 void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info)
3172 {
3173         btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT);
3174         btrfs_clear_opt(fs_info->mount_opt, CLEAR_CACHE);
3175 }
3176
3177 /*
3178  * Mounting logic specific to read-write file systems. Shared by open_ctree
3179  * and btrfs_remount when remounting from read-only to read-write.
3180  */
3181 int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info)
3182 {
3183         int ret;
3184         const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE);
3185         bool clear_free_space_tree = false;
3186
3187         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3188             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3189                 clear_free_space_tree = true;
3190         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3191                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3192                 btrfs_warn(fs_info, "free space tree is invalid");
3193                 clear_free_space_tree = true;
3194         }
3195
3196         if (clear_free_space_tree) {
3197                 btrfs_info(fs_info, "clearing free space tree");
3198                 ret = btrfs_clear_free_space_tree(fs_info);
3199                 if (ret) {
3200                         btrfs_warn(fs_info,
3201                                    "failed to clear free space tree: %d", ret);
3202                         goto out;
3203                 }
3204         }
3205
3206         /*
3207          * btrfs_find_orphan_roots() is responsible for finding all the dead
3208          * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load
3209          * them into the fs_info->fs_roots_radix tree. This must be done before
3210          * calling btrfs_orphan_cleanup() on the tree root. If we don't do it
3211          * first, then btrfs_orphan_cleanup() will delete a dead root's orphan
3212          * item before the root's tree is deleted - this means that if we unmount
3213          * or crash before the deletion completes, on the next mount we will not
3214          * delete what remains of the tree because the orphan item does not
3215          * exists anymore, which is what tells us we have a pending deletion.
3216          */
3217         ret = btrfs_find_orphan_roots(fs_info);
3218         if (ret)
3219                 goto out;
3220
3221         ret = btrfs_cleanup_fs_roots(fs_info);
3222         if (ret)
3223                 goto out;
3224
3225         down_read(&fs_info->cleanup_work_sem);
3226         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3227             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3228                 up_read(&fs_info->cleanup_work_sem);
3229                 goto out;
3230         }
3231         up_read(&fs_info->cleanup_work_sem);
3232
3233         mutex_lock(&fs_info->cleaner_mutex);
3234         ret = btrfs_recover_relocation(fs_info);
3235         mutex_unlock(&fs_info->cleaner_mutex);
3236         if (ret < 0) {
3237                 btrfs_warn(fs_info, "failed to recover relocation: %d", ret);
3238                 goto out;
3239         }
3240
3241         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3242             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3243                 btrfs_info(fs_info, "creating free space tree");
3244                 ret = btrfs_create_free_space_tree(fs_info);
3245                 if (ret) {
3246                         btrfs_warn(fs_info,
3247                                 "failed to create free space tree: %d", ret);
3248                         goto out;
3249                 }
3250         }
3251
3252         if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) {
3253                 ret = btrfs_set_free_space_cache_v1_active(fs_info, cache_opt);
3254                 if (ret)
3255                         goto out;
3256         }
3257
3258         ret = btrfs_resume_balance_async(fs_info);
3259         if (ret)
3260                 goto out;
3261
3262         ret = btrfs_resume_dev_replace_async(fs_info);
3263         if (ret) {
3264                 btrfs_warn(fs_info, "failed to resume dev_replace");
3265                 goto out;
3266         }
3267
3268         btrfs_qgroup_rescan_resume(fs_info);
3269
3270         if (!fs_info->uuid_root) {
3271                 btrfs_info(fs_info, "creating UUID tree");
3272                 ret = btrfs_create_uuid_tree(fs_info);
3273                 if (ret) {
3274                         btrfs_warn(fs_info,
3275                                    "failed to create the UUID tree %d", ret);
3276                         goto out;
3277                 }
3278         }
3279
3280 out:
3281         return ret;
3282 }
3283
3284 int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices,
3285                       char *options)
3286 {
3287         u32 sectorsize;
3288         u32 nodesize;
3289         u32 stripesize;
3290         u64 generation;
3291         u64 features;
3292         u16 csum_type;
3293         struct btrfs_super_block *disk_super;
3294         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
3295         struct btrfs_root *tree_root;
3296         struct btrfs_root *chunk_root;
3297         int ret;
3298         int err = -EINVAL;
3299         int level;
3300
3301         ret = init_mount_fs_info(fs_info, sb);
3302         if (ret) {
3303                 err = ret;
3304                 goto fail;
3305         }
3306
3307         /* These need to be init'ed before we start creating inodes and such. */
3308         tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID,
3309                                      GFP_KERNEL);
3310         fs_info->tree_root = tree_root;
3311         chunk_root = btrfs_alloc_root(fs_info, BTRFS_CHUNK_TREE_OBJECTID,
3312                                       GFP_KERNEL);
3313         fs_info->chunk_root = chunk_root;
3314         if (!tree_root || !chunk_root) {
3315                 err = -ENOMEM;
3316                 goto fail;
3317         }
3318
3319         fs_info->btree_inode = new_inode(sb);
3320         if (!fs_info->btree_inode) {
3321                 err = -ENOMEM;
3322                 goto fail;
3323         }
3324         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
3325         btrfs_init_btree_inode(fs_info);
3326
3327         invalidate_bdev(fs_devices->latest_dev->bdev);
3328
3329         /*
3330          * Read super block and check the signature bytes only
3331          */
3332         disk_super = btrfs_read_dev_super(fs_devices->latest_dev->bdev);
3333         if (IS_ERR(disk_super)) {
3334                 err = PTR_ERR(disk_super);
3335                 goto fail_alloc;
3336         }
3337
3338         /*
3339          * Verify the type first, if that or the checksum value are
3340          * corrupted, we'll find out
3341          */
3342         csum_type = btrfs_super_csum_type(disk_super);
3343         if (!btrfs_supported_super_csum(csum_type)) {
3344                 btrfs_err(fs_info, "unsupported checksum algorithm: %u",
3345                           csum_type);
3346                 err = -EINVAL;
3347                 btrfs_release_disk_super(disk_super);
3348                 goto fail_alloc;
3349         }
3350
3351         fs_info->csum_size = btrfs_super_csum_size(disk_super);
3352
3353         ret = btrfs_init_csum_hash(fs_info, csum_type);
3354         if (ret) {
3355                 err = ret;
3356                 btrfs_release_disk_super(disk_super);
3357                 goto fail_alloc;
3358         }
3359
3360         /*
3361          * We want to check superblock checksum, the type is stored inside.
3362          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
3363          */
3364         if (btrfs_check_super_csum(fs_info, (u8 *)disk_super)) {
3365                 btrfs_err(fs_info, "superblock checksum mismatch");
3366                 err = -EINVAL;
3367                 btrfs_release_disk_super(disk_super);
3368                 goto fail_alloc;
3369         }
3370
3371         /*
3372          * super_copy is zeroed at allocation time and we never touch the
3373          * following bytes up to INFO_SIZE, the checksum is calculated from
3374          * the whole block of INFO_SIZE
3375          */
3376         memcpy(fs_info->super_copy, disk_super, sizeof(*fs_info->super_copy));
3377         btrfs_release_disk_super(disk_super);
3378
3379         disk_super = fs_info->super_copy;
3380
3381
3382         features = btrfs_super_flags(disk_super);
3383         if (features & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
3384                 features &= ~BTRFS_SUPER_FLAG_CHANGING_FSID_V2;
3385                 btrfs_set_super_flags(disk_super, features);
3386                 btrfs_info(fs_info,
3387                         "found metadata UUID change in progress flag, clearing");
3388         }
3389
3390         memcpy(fs_info->super_for_commit, fs_info->super_copy,
3391                sizeof(*fs_info->super_for_commit));
3392
3393         ret = btrfs_validate_mount_super(fs_info);
3394         if (ret) {
3395                 btrfs_err(fs_info, "superblock contains fatal errors");
3396                 err = -EINVAL;
3397                 goto fail_alloc;
3398         }
3399
3400         if (!btrfs_super_root(disk_super))
3401                 goto fail_alloc;
3402
3403         /* check FS state, whether FS is broken. */
3404         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
3405                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
3406
3407         /*
3408          * In the long term, we'll store the compression type in the super
3409          * block, and it'll be used for per file compression control.
3410          */
3411         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
3412
3413
3414         /* Set up fs_info before parsing mount options */
3415         nodesize = btrfs_super_nodesize(disk_super);
3416         sectorsize = btrfs_super_sectorsize(disk_super);
3417         stripesize = sectorsize;
3418         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
3419         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
3420
3421         fs_info->nodesize = nodesize;
3422         fs_info->sectorsize = sectorsize;
3423         fs_info->sectorsize_bits = ilog2(sectorsize);
3424         fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size;
3425         fs_info->stripesize = stripesize;
3426
3427         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
3428         if (ret) {
3429                 err = ret;
3430                 goto fail_alloc;
3431         }
3432
3433         features = btrfs_super_incompat_flags(disk_super) &
3434                 ~BTRFS_FEATURE_INCOMPAT_SUPP;
3435         if (features) {
3436                 btrfs_err(fs_info,
3437                     "cannot mount because of unsupported optional features (0x%llx)",
3438                     features);
3439                 err = -EINVAL;
3440                 goto fail_alloc;
3441         }
3442
3443         features = btrfs_super_incompat_flags(disk_super);
3444         features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
3445         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
3446                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
3447         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
3448                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
3449
3450         /*
3451          * Flag our filesystem as having big metadata blocks if they are bigger
3452          * than the page size.
3453          */
3454         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE)
3455                 features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
3456
3457         /*
3458          * mixed block groups end up with duplicate but slightly offset
3459          * extent buffers for the same range.  It leads to corruptions
3460          */
3461         if ((features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
3462             (sectorsize != nodesize)) {
3463                 btrfs_err(fs_info,
3464 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
3465                         nodesize, sectorsize);
3466                 goto fail_alloc;
3467         }
3468
3469         /*
3470          * Needn't use the lock because there is no other task which will
3471          * update the flag.
3472          */
3473         btrfs_set_super_incompat_flags(disk_super, features);
3474
3475         features = btrfs_super_compat_ro_flags(disk_super) &
3476                 ~BTRFS_FEATURE_COMPAT_RO_SUPP;
3477         if (!sb_rdonly(sb) && features) {
3478                 btrfs_err(fs_info,
3479         "cannot mount read-write because of unsupported optional features (0x%llx)",
3480                        features);
3481                 err = -EINVAL;
3482                 goto fail_alloc;
3483         }
3484         /*
3485          * We have unsupported RO compat features, although RO mounted, we
3486          * should not cause any metadata write, including log replay.
3487          * Or we could screw up whatever the new feature requires.
3488          */
3489         if (unlikely(features && btrfs_super_log_root(disk_super) &&
3490                      !btrfs_test_opt(fs_info, NOLOGREPLAY))) {
3491                 btrfs_err(fs_info,
3492 "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay",
3493                           features);
3494                 err = -EINVAL;
3495                 goto fail_alloc;
3496         }
3497
3498
3499         if (sectorsize < PAGE_SIZE) {
3500                 struct btrfs_subpage_info *subpage_info;
3501
3502                 /*
3503                  * V1 space cache has some hardcoded PAGE_SIZE usage, and is
3504                  * going to be deprecated.
3505                  *
3506                  * Force to use v2 cache for subpage case.
3507                  */
3508                 btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
3509                 btrfs_set_and_info(fs_info, FREE_SPACE_TREE,
3510                         "forcing free space tree for sector size %u with page size %lu",
3511                         sectorsize, PAGE_SIZE);
3512
3513                 btrfs_warn(fs_info,
3514                 "read-write for sector size %u with page size %lu is experimental",
3515                            sectorsize, PAGE_SIZE);
3516                 subpage_info = kzalloc(sizeof(*subpage_info), GFP_KERNEL);
3517                 if (!subpage_info)
3518                         goto fail_alloc;
3519                 btrfs_init_subpage_info(subpage_info, sectorsize);
3520                 fs_info->subpage_info = subpage_info;
3521         }
3522
3523         ret = btrfs_init_workqueues(fs_info);
3524         if (ret) {
3525                 err = ret;
3526                 goto fail_sb_buffer;
3527         }
3528
3529         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
3530         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
3531
3532         sb->s_blocksize = sectorsize;
3533         sb->s_blocksize_bits = blksize_bits(sectorsize);
3534         memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE);
3535
3536         mutex_lock(&fs_info->chunk_mutex);
3537         ret = btrfs_read_sys_array(fs_info);
3538         mutex_unlock(&fs_info->chunk_mutex);
3539         if (ret) {
3540                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
3541                 goto fail_sb_buffer;
3542         }
3543
3544         generation = btrfs_super_chunk_root_generation(disk_super);
3545         level = btrfs_super_chunk_root_level(disk_super);
3546         ret = load_super_root(chunk_root, btrfs_super_chunk_root(disk_super),
3547                               generation, level);
3548         if (ret) {
3549                 btrfs_err(fs_info, "failed to read chunk root");
3550                 goto fail_tree_roots;
3551         }
3552
3553         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
3554                            offsetof(struct btrfs_header, chunk_tree_uuid),
3555                            BTRFS_UUID_SIZE);
3556
3557         ret = btrfs_read_chunk_tree(fs_info);
3558         if (ret) {
3559                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
3560                 goto fail_tree_roots;
3561         }
3562
3563         /*
3564          * At this point we know all the devices that make this filesystem,
3565          * including the seed devices but we don't know yet if the replace
3566          * target is required. So free devices that are not part of this
3567          * filesystem but skip the replace target device which is checked
3568          * below in btrfs_init_dev_replace().
3569          */
3570         btrfs_free_extra_devids(fs_devices);
3571         if (!fs_devices->latest_dev->bdev) {
3572                 btrfs_err(fs_info, "failed to read devices");
3573                 goto fail_tree_roots;
3574         }
3575
3576         ret = init_tree_roots(fs_info);
3577         if (ret)
3578                 goto fail_tree_roots;
3579
3580         /*
3581          * Get zone type information of zoned block devices. This will also
3582          * handle emulation of a zoned filesystem if a regular device has the
3583          * zoned incompat feature flag set.
3584          */
3585         ret = btrfs_get_dev_zone_info_all_devices(fs_info);
3586         if (ret) {
3587                 btrfs_err(fs_info,
3588                           "zoned: failed to read device zone info: %d",
3589                           ret);
3590                 goto fail_block_groups;
3591         }
3592
3593         /*
3594          * If we have a uuid root and we're not being told to rescan we need to
3595          * check the generation here so we can set the
3596          * BTRFS_FS_UPDATE_UUID_TREE_GEN bit.  Otherwise we could commit the
3597          * transaction during a balance or the log replay without updating the
3598          * uuid generation, and then if we crash we would rescan the uuid tree,
3599          * even though it was perfectly fine.
3600          */
3601         if (fs_info->uuid_root && !btrfs_test_opt(fs_info, RESCAN_UUID_TREE) &&
3602             fs_info->generation == btrfs_super_uuid_tree_generation(disk_super))
3603                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
3604
3605         ret = btrfs_verify_dev_extents(fs_info);
3606         if (ret) {
3607                 btrfs_err(fs_info,
3608                           "failed to verify dev extents against chunks: %d",
3609                           ret);
3610                 goto fail_block_groups;
3611         }
3612         ret = btrfs_recover_balance(fs_info);
3613         if (ret) {
3614                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3615                 goto fail_block_groups;
3616         }
3617
3618         ret = btrfs_init_dev_stats(fs_info);
3619         if (ret) {
3620                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3621                 goto fail_block_groups;
3622         }
3623
3624         ret = btrfs_init_dev_replace(fs_info);
3625         if (ret) {
3626                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3627                 goto fail_block_groups;
3628         }
3629
3630         ret = btrfs_check_zoned_mode(fs_info);
3631         if (ret) {
3632                 btrfs_err(fs_info, "failed to initialize zoned mode: %d",
3633                           ret);
3634                 goto fail_block_groups;
3635         }
3636
3637         ret = btrfs_sysfs_add_fsid(fs_devices);
3638         if (ret) {
3639                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3640                                 ret);
3641                 goto fail_block_groups;
3642         }
3643
3644         ret = btrfs_sysfs_add_mounted(fs_info);
3645         if (ret) {
3646                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3647                 goto fail_fsdev_sysfs;
3648         }
3649
3650         ret = btrfs_init_space_info(fs_info);
3651         if (ret) {
3652                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3653                 goto fail_sysfs;
3654         }
3655
3656         ret = btrfs_read_block_groups(fs_info);
3657         if (ret) {
3658                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3659                 goto fail_sysfs;
3660         }
3661
3662         btrfs_free_zone_cache(fs_info);
3663
3664         if (!sb_rdonly(sb) && fs_info->fs_devices->missing_devices &&
3665             !btrfs_check_rw_degradable(fs_info, NULL)) {
3666                 btrfs_warn(fs_info,
3667                 "writable mount is not allowed due to too many missing devices");
3668                 goto fail_sysfs;
3669         }
3670
3671         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, fs_info,
3672                                                "btrfs-cleaner");
3673         if (IS_ERR(fs_info->cleaner_kthread))
3674                 goto fail_sysfs;
3675
3676         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3677                                                    tree_root,
3678                                                    "btrfs-transaction");
3679         if (IS_ERR(fs_info->transaction_kthread))
3680                 goto fail_cleaner;
3681
3682         if (!btrfs_test_opt(fs_info, NOSSD) &&
3683             !fs_info->fs_devices->rotating) {
3684                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3685         }
3686
3687         /*
3688          * Mount does not set all options immediately, we can do it now and do
3689          * not have to wait for transaction commit
3690          */
3691         btrfs_apply_pending_changes(fs_info);
3692
3693 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3694         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3695                 ret = btrfsic_mount(fs_info, fs_devices,
3696                                     btrfs_test_opt(fs_info,
3697                                         CHECK_INTEGRITY_DATA) ? 1 : 0,
3698                                     fs_info->check_integrity_print_mask);
3699                 if (ret)
3700                         btrfs_warn(fs_info,
3701                                 "failed to initialize integrity check module: %d",
3702                                 ret);
3703         }
3704 #endif
3705         ret = btrfs_read_qgroup_config(fs_info);
3706         if (ret)
3707                 goto fail_trans_kthread;
3708
3709         if (btrfs_build_ref_tree(fs_info))
3710                 btrfs_err(fs_info, "couldn't build ref tree");
3711
3712         /* do not make disk changes in broken FS or nologreplay is given */
3713         if (btrfs_super_log_root(disk_super) != 0 &&
3714             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3715                 btrfs_info(fs_info, "start tree-log replay");
3716                 ret = btrfs_replay_log(fs_info, fs_devices);
3717                 if (ret) {
3718                         err = ret;
3719                         goto fail_qgroup;
3720                 }
3721         }
3722
3723         fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true);
3724         if (IS_ERR(fs_info->fs_root)) {
3725                 err = PTR_ERR(fs_info->fs_root);
3726                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3727                 fs_info->fs_root = NULL;
3728                 goto fail_qgroup;
3729         }
3730
3731         if (sb_rdonly(sb))
3732                 goto clear_oneshot;
3733
3734         ret = btrfs_start_pre_rw_mount(fs_info);
3735         if (ret) {
3736                 close_ctree(fs_info);
3737                 return ret;
3738         }
3739         btrfs_discard_resume(fs_info);
3740
3741         if (fs_info->uuid_root &&
3742             (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) ||
3743              fs_info->generation != btrfs_super_uuid_tree_generation(disk_super))) {
3744                 btrfs_info(fs_info, "checking UUID tree");
3745                 ret = btrfs_check_uuid_tree(fs_info);
3746                 if (ret) {
3747                         btrfs_warn(fs_info,
3748                                 "failed to check the UUID tree: %d", ret);
3749                         close_ctree(fs_info);
3750                         return ret;
3751                 }
3752         }
3753
3754         set_bit(BTRFS_FS_OPEN, &fs_info->flags);
3755
3756         /* Kick the cleaner thread so it'll start deleting snapshots. */
3757         if (test_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags))
3758                 wake_up_process(fs_info->cleaner_kthread);
3759
3760 clear_oneshot:
3761         btrfs_clear_oneshot_options(fs_info);
3762         return 0;
3763
3764 fail_qgroup:
3765         btrfs_free_qgroup_config(fs_info);
3766 fail_trans_kthread:
3767         kthread_stop(fs_info->transaction_kthread);
3768         btrfs_cleanup_transaction(fs_info);
3769         btrfs_free_fs_roots(fs_info);
3770 fail_cleaner:
3771         kthread_stop(fs_info->cleaner_kthread);
3772
3773         /*
3774          * make sure we're done with the btree inode before we stop our
3775          * kthreads
3776          */
3777         filemap_write_and_wait(fs_info->btree_inode->i_mapping);
3778
3779 fail_sysfs:
3780         btrfs_sysfs_remove_mounted(fs_info);
3781
3782 fail_fsdev_sysfs:
3783         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3784
3785 fail_block_groups:
3786         btrfs_put_block_group_cache(fs_info);
3787
3788 fail_tree_roots:
3789         if (fs_info->data_reloc_root)
3790                 btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root);
3791         free_root_pointers(fs_info, true);
3792         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
3793
3794 fail_sb_buffer:
3795         btrfs_stop_all_workers(fs_info);
3796         btrfs_free_block_groups(fs_info);
3797 fail_alloc:
3798         btrfs_mapping_tree_free(&fs_info->mapping_tree);
3799
3800         iput(fs_info->btree_inode);
3801 fail:
3802         btrfs_close_devices(fs_info->fs_devices);
3803         return err;
3804 }
3805 ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
3806
3807 static void btrfs_end_super_write(struct bio *bio)
3808 {
3809         struct btrfs_device *device = bio->bi_private;
3810         struct bio_vec *bvec;
3811         struct bvec_iter_all iter_all;
3812         struct page *page;
3813
3814         bio_for_each_segment_all(bvec, bio, iter_all) {
3815                 page = bvec->bv_page;
3816
3817                 if (bio->bi_status) {
3818                         btrfs_warn_rl_in_rcu(device->fs_info,
3819                                 "lost page write due to IO error on %s (%d)",
3820                                 rcu_str_deref(device->name),
3821                                 blk_status_to_errno(bio->bi_status));
3822                         ClearPageUptodate(page);
3823                         SetPageError(page);
3824                         btrfs_dev_stat_inc_and_print(device,
3825                                                      BTRFS_DEV_STAT_WRITE_ERRS);
3826                 } else {
3827                         SetPageUptodate(page);
3828                 }
3829
3830                 put_page(page);
3831                 unlock_page(page);
3832         }
3833
3834         bio_put(bio);
3835 }
3836
3837 struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev,
3838                                                    int copy_num)
3839 {
3840         struct btrfs_super_block *super;
3841         struct page *page;
3842         u64 bytenr, bytenr_orig;
3843         struct address_space *mapping = bdev->bd_inode->i_mapping;
3844         int ret;
3845
3846         bytenr_orig = btrfs_sb_offset(copy_num);
3847         ret = btrfs_sb_log_location_bdev(bdev, copy_num, READ, &bytenr);
3848         if (ret == -ENOENT)
3849                 return ERR_PTR(-EINVAL);
3850         else if (ret)
3851                 return ERR_PTR(ret);
3852
3853         if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev))
3854                 return ERR_PTR(-EINVAL);
3855
3856         page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS);
3857         if (IS_ERR(page))
3858                 return ERR_CAST(page);
3859
3860         super = page_address(page);
3861         if (btrfs_super_magic(super) != BTRFS_MAGIC) {
3862                 btrfs_release_disk_super(super);
3863                 return ERR_PTR(-ENODATA);
3864         }
3865
3866         if (btrfs_super_bytenr(super) != bytenr_orig) {
3867                 btrfs_release_disk_super(super);
3868                 return ERR_PTR(-EINVAL);
3869         }
3870
3871         return super;
3872 }
3873
3874
3875 struct btrfs_super_block *btrfs_read_dev_super(struct block_device *bdev)
3876 {
3877         struct btrfs_super_block *super, *latest = NULL;
3878         int i;
3879         u64 transid = 0;
3880
3881         /* we would like to check all the supers, but that would make
3882          * a btrfs mount succeed after a mkfs from a different FS.
3883          * So, we need to add a special mount option to scan for
3884          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
3885          */
3886         for (i = 0; i < 1; i++) {
3887                 super = btrfs_read_dev_one_super(bdev, i);
3888                 if (IS_ERR(super))
3889                         continue;
3890
3891                 if (!latest || btrfs_super_generation(super) > transid) {
3892                         if (latest)
3893                                 btrfs_release_disk_super(super);
3894
3895                         latest = super;
3896                         transid = btrfs_super_generation(super);
3897                 }
3898         }
3899
3900         return super;
3901 }
3902
3903 /*
3904  * Write superblock @sb to the @device. Do not wait for completion, all the
3905  * pages we use for writing are locked.
3906  *
3907  * Write @max_mirrors copies of the superblock, where 0 means default that fit
3908  * the expected device size at commit time. Note that max_mirrors must be
3909  * same for write and wait phases.
3910  *
3911  * Return number of errors when page is not found or submission fails.
3912  */
3913 static int write_dev_supers(struct btrfs_device *device,
3914                             struct btrfs_super_block *sb, int max_mirrors)
3915 {
3916         struct btrfs_fs_info *fs_info = device->fs_info;
3917         struct address_space *mapping = device->bdev->bd_inode->i_mapping;
3918         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3919         int i;
3920         int errors = 0;
3921         int ret;
3922         u64 bytenr, bytenr_orig;
3923
3924         if (max_mirrors == 0)
3925                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3926
3927         shash->tfm = fs_info->csum_shash;
3928
3929         for (i = 0; i < max_mirrors; i++) {
3930                 struct page *page;
3931                 struct bio *bio;
3932                 struct btrfs_super_block *disk_super;
3933
3934                 bytenr_orig = btrfs_sb_offset(i);
3935                 ret = btrfs_sb_log_location(device, i, WRITE, &bytenr);
3936                 if (ret == -ENOENT) {
3937                         continue;
3938                 } else if (ret < 0) {
3939                         btrfs_err(device->fs_info,
3940                                 "couldn't get super block location for mirror %d",
3941                                 i);
3942                         errors++;
3943                         continue;
3944                 }
3945                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
3946                     device->commit_total_bytes)
3947                         break;
3948
3949                 btrfs_set_super_bytenr(sb, bytenr_orig);
3950
3951                 crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE,
3952                                     BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE,
3953                                     sb->csum);
3954
3955                 page = find_or_create_page(mapping, bytenr >> PAGE_SHIFT,
3956                                            GFP_NOFS);
3957                 if (!page) {
3958                         btrfs_err(device->fs_info,
3959                             "couldn't get super block page for bytenr %llu",
3960                             bytenr);
3961                         errors++;
3962                         continue;
3963                 }
3964
3965                 /* Bump the refcount for wait_dev_supers() */
3966                 get_page(page);
3967
3968                 disk_super = page_address(page);
3969                 memcpy(disk_super, sb, BTRFS_SUPER_INFO_SIZE);
3970
3971                 /*
3972                  * Directly use bios here instead of relying on the page cache
3973                  * to do I/O, so we don't lose the ability to do integrity
3974                  * checking.
3975                  */
3976                 bio = bio_alloc(device->bdev, 1,
3977                                 REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO,
3978                                 GFP_NOFS);
3979                 bio->bi_iter.bi_sector = bytenr >> SECTOR_SHIFT;
3980                 bio->bi_private = device;
3981                 bio->bi_end_io = btrfs_end_super_write;
3982                 __bio_add_page(bio, page, BTRFS_SUPER_INFO_SIZE,
3983                                offset_in_page(bytenr));
3984
3985                 /*
3986                  * We FUA only the first super block.  The others we allow to
3987                  * go down lazy and there's a short window where the on-disk
3988                  * copies might still contain the older version.
3989                  */
3990                 if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER))
3991                         bio->bi_opf |= REQ_FUA;
3992
3993                 btrfsic_check_bio(bio);
3994                 submit_bio(bio);
3995
3996                 if (btrfs_advance_sb_log(device, i))
3997                         errors++;
3998         }
3999         return errors < i ? 0 : -1;
4000 }
4001
4002 /*
4003  * Wait for write completion of superblocks done by write_dev_supers,
4004  * @max_mirrors same for write and wait phases.
4005  *
4006  * Return number of errors when page is not found or not marked up to
4007  * date.
4008  */
4009 static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
4010 {
4011         int i;
4012         int errors = 0;
4013         bool primary_failed = false;
4014         int ret;
4015         u64 bytenr;
4016
4017         if (max_mirrors == 0)
4018                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
4019
4020         for (i = 0; i < max_mirrors; i++) {
4021                 struct page *page;
4022
4023                 ret = btrfs_sb_log_location(device, i, READ, &bytenr);
4024                 if (ret == -ENOENT) {
4025                         break;
4026                 } else if (ret < 0) {
4027                         errors++;
4028                         if (i == 0)
4029                                 primary_failed = true;
4030                         continue;
4031                 }
4032                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
4033                     device->commit_total_bytes)
4034                         break;
4035
4036                 page = find_get_page(device->bdev->bd_inode->i_mapping,
4037                                      bytenr >> PAGE_SHIFT);
4038                 if (!page) {
4039                         errors++;
4040                         if (i == 0)
4041                                 primary_failed = true;
4042                         continue;
4043                 }
4044                 /* Page is submitted locked and unlocked once the IO completes */
4045                 wait_on_page_locked(page);
4046                 if (PageError(page)) {
4047                         errors++;
4048                         if (i == 0)
4049                                 primary_failed = true;
4050                 }
4051
4052                 /* Drop our reference */
4053                 put_page(page);
4054
4055                 /* Drop the reference from the writing run */
4056                 put_page(page);
4057         }
4058
4059         /* log error, force error return */
4060         if (primary_failed) {
4061                 btrfs_err(device->fs_info, "error writing primary super block to device %llu",
4062                           device->devid);
4063                 return -1;
4064         }
4065
4066         return errors < i ? 0 : -1;
4067 }
4068
4069 /*
4070  * endio for the write_dev_flush, this will wake anyone waiting
4071  * for the barrier when it is done
4072  */
4073 static void btrfs_end_empty_barrier(struct bio *bio)
4074 {
4075         bio_uninit(bio);
4076         complete(bio->bi_private);
4077 }
4078
4079 /*
4080  * Submit a flush request to the device if it supports it. Error handling is
4081  * done in the waiting counterpart.
4082  */
4083 static void write_dev_flush(struct btrfs_device *device)
4084 {
4085         struct bio *bio = &device->flush_bio;
4086
4087 #ifndef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4088         /*
4089          * When a disk has write caching disabled, we skip submission of a bio
4090          * with flush and sync requests before writing the superblock, since
4091          * it's not needed. However when the integrity checker is enabled, this
4092          * results in reports that there are metadata blocks referred by a
4093          * superblock that were not properly flushed. So don't skip the bio
4094          * submission only when the integrity checker is enabled for the sake
4095          * of simplicity, since this is a debug tool and not meant for use in
4096          * non-debug builds.
4097          */
4098         if (!bdev_write_cache(device->bdev))
4099                 return;
4100 #endif
4101
4102         bio_init(bio, device->bdev, NULL, 0,
4103                  REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH);
4104         bio->bi_end_io = btrfs_end_empty_barrier;
4105         init_completion(&device->flush_wait);
4106         bio->bi_private = &device->flush_wait;
4107
4108         btrfsic_check_bio(bio);
4109         submit_bio(bio);
4110         set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4111 }
4112
4113 /*
4114  * If the flush bio has been submitted by write_dev_flush, wait for it.
4115  */
4116 static blk_status_t wait_dev_flush(struct btrfs_device *device)
4117 {
4118         struct bio *bio = &device->flush_bio;
4119
4120         if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
4121                 return BLK_STS_OK;
4122
4123         clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4124         wait_for_completion_io(&device->flush_wait);
4125
4126         return bio->bi_status;
4127 }
4128
4129 static int check_barrier_error(struct btrfs_fs_info *fs_info)
4130 {
4131         if (!btrfs_check_rw_degradable(fs_info, NULL))
4132                 return -EIO;
4133         return 0;
4134 }
4135
4136 /*
4137  * send an empty flush down to each device in parallel,
4138  * then wait for them
4139  */
4140 static int barrier_all_devices(struct btrfs_fs_info *info)
4141 {
4142         struct list_head *head;
4143         struct btrfs_device *dev;
4144         int errors_wait = 0;
4145         blk_status_t ret;
4146
4147         lockdep_assert_held(&info->fs_devices->device_list_mutex);
4148         /* send down all the barriers */
4149         head = &info->fs_devices->devices;
4150         list_for_each_entry(dev, head, dev_list) {
4151                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4152                         continue;
4153                 if (!dev->bdev)
4154                         continue;
4155                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4156                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4157                         continue;
4158
4159                 write_dev_flush(dev);
4160                 dev->last_flush_error = BLK_STS_OK;
4161         }
4162
4163         /* wait for all the barriers */
4164         list_for_each_entry(dev, head, dev_list) {
4165                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4166                         continue;
4167                 if (!dev->bdev) {
4168                         errors_wait++;
4169                         continue;
4170                 }
4171                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4172                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4173                         continue;
4174
4175                 ret = wait_dev_flush(dev);
4176                 if (ret) {
4177                         dev->last_flush_error = ret;
4178                         btrfs_dev_stat_inc_and_print(dev,
4179                                         BTRFS_DEV_STAT_FLUSH_ERRS);
4180                         errors_wait++;
4181                 }
4182         }
4183
4184         if (errors_wait) {
4185                 /*
4186                  * At some point we need the status of all disks
4187                  * to arrive at the volume status. So error checking
4188                  * is being pushed to a separate loop.
4189                  */
4190                 return check_barrier_error(info);
4191         }
4192         return 0;
4193 }
4194
4195 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
4196 {
4197         int raid_type;
4198         int min_tolerated = INT_MAX;
4199
4200         if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 ||
4201             (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE))
4202                 min_tolerated = min_t(int, min_tolerated,
4203                                     btrfs_raid_array[BTRFS_RAID_SINGLE].
4204                                     tolerated_failures);
4205
4206         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
4207                 if (raid_type == BTRFS_RAID_SINGLE)
4208                         continue;
4209                 if (!(flags & btrfs_raid_array[raid_type].bg_flag))
4210                         continue;
4211                 min_tolerated = min_t(int, min_tolerated,
4212                                     btrfs_raid_array[raid_type].
4213                                     tolerated_failures);
4214         }
4215
4216         if (min_tolerated == INT_MAX) {
4217                 pr_warn("BTRFS: unknown raid flag: %llu", flags);
4218                 min_tolerated = 0;
4219         }
4220
4221         return min_tolerated;
4222 }
4223
4224 int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
4225 {
4226         struct list_head *head;
4227         struct btrfs_device *dev;
4228         struct btrfs_super_block *sb;
4229         struct btrfs_dev_item *dev_item;
4230         int ret;
4231         int do_barriers;
4232         int max_errors;
4233         int total_errors = 0;
4234         u64 flags;
4235
4236         do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
4237
4238         /*
4239          * max_mirrors == 0 indicates we're from commit_transaction,
4240          * not from fsync where the tree roots in fs_info have not
4241          * been consistent on disk.
4242          */
4243         if (max_mirrors == 0)
4244                 backup_super_roots(fs_info);
4245
4246         sb = fs_info->super_for_commit;
4247         dev_item = &sb->dev_item;
4248
4249         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4250         head = &fs_info->fs_devices->devices;
4251         max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1;
4252
4253         if (do_barriers) {
4254                 ret = barrier_all_devices(fs_info);
4255                 if (ret) {
4256                         mutex_unlock(
4257                                 &fs_info->fs_devices->device_list_mutex);
4258                         btrfs_handle_fs_error(fs_info, ret,
4259                                               "errors while submitting device barriers.");
4260                         return ret;
4261                 }
4262         }
4263
4264         list_for_each_entry(dev, head, dev_list) {
4265                 if (!dev->bdev) {
4266                         total_errors++;
4267                         continue;
4268                 }
4269                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4270                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4271                         continue;
4272
4273                 btrfs_set_stack_device_generation(dev_item, 0);
4274                 btrfs_set_stack_device_type(dev_item, dev->type);
4275                 btrfs_set_stack_device_id(dev_item, dev->devid);
4276                 btrfs_set_stack_device_total_bytes(dev_item,
4277                                                    dev->commit_total_bytes);
4278                 btrfs_set_stack_device_bytes_used(dev_item,
4279                                                   dev->commit_bytes_used);
4280                 btrfs_set_stack_device_io_align(dev_item, dev->io_align);
4281                 btrfs_set_stack_device_io_width(dev_item, dev->io_width);
4282                 btrfs_set_stack_device_sector_size(dev_item, dev->sector_size);
4283                 memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE);
4284                 memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid,
4285                        BTRFS_FSID_SIZE);
4286
4287                 flags = btrfs_super_flags(sb);
4288                 btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
4289
4290                 ret = btrfs_validate_write_super(fs_info, sb);
4291                 if (ret < 0) {
4292                         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4293                         btrfs_handle_fs_error(fs_info, -EUCLEAN,
4294                                 "unexpected superblock corruption detected");
4295                         return -EUCLEAN;
4296                 }
4297
4298                 ret = write_dev_supers(dev, sb, max_mirrors);
4299                 if (ret)
4300                         total_errors++;
4301         }
4302         if (total_errors > max_errors) {
4303                 btrfs_err(fs_info, "%d errors while writing supers",
4304                           total_errors);
4305                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4306
4307                 /* FUA is masked off if unsupported and can't be the reason */
4308                 btrfs_handle_fs_error(fs_info, -EIO,
4309                                       "%d errors while writing supers",
4310                                       total_errors);
4311                 return -EIO;
4312         }
4313
4314         total_errors = 0;
4315         list_for_each_entry(dev, head, dev_list) {
4316                 if (!dev->bdev)
4317                         continue;
4318                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4319                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4320                         continue;
4321
4322                 ret = wait_dev_supers(dev, max_mirrors);
4323                 if (ret)
4324                         total_errors++;
4325         }
4326         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4327         if (total_errors > max_errors) {
4328                 btrfs_handle_fs_error(fs_info, -EIO,
4329                                       "%d errors while writing supers",
4330                                       total_errors);
4331                 return -EIO;
4332         }
4333         return 0;
4334 }
4335
4336 /* Drop a fs root from the radix tree and free it. */
4337 void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
4338                                   struct btrfs_root *root)
4339 {
4340         bool drop_ref = false;
4341
4342         spin_lock(&fs_info->fs_roots_radix_lock);
4343         radix_tree_delete(&fs_info->fs_roots_radix,
4344                           (unsigned long)root->root_key.objectid);
4345         if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state))
4346                 drop_ref = true;
4347         spin_unlock(&fs_info->fs_roots_radix_lock);
4348
4349         if (BTRFS_FS_ERROR(fs_info)) {
4350                 ASSERT(root->log_root == NULL);
4351                 if (root->reloc_root) {
4352                         btrfs_put_root(root->reloc_root);
4353                         root->reloc_root = NULL;
4354                 }
4355         }
4356
4357         if (drop_ref)
4358                 btrfs_put_root(root);
4359 }
4360
4361 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
4362 {
4363         u64 root_objectid = 0;
4364         struct btrfs_root *gang[8];
4365         int i = 0;
4366         int err = 0;
4367         unsigned int ret = 0;
4368
4369         while (1) {
4370                 spin_lock(&fs_info->fs_roots_radix_lock);
4371                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4372                                              (void **)gang, root_objectid,
4373                                              ARRAY_SIZE(gang));
4374                 if (!ret) {
4375                         spin_unlock(&fs_info->fs_roots_radix_lock);
4376                         break;
4377                 }
4378                 root_objectid = gang[ret - 1]->root_key.objectid + 1;
4379
4380                 for (i = 0; i < ret; i++) {
4381                         /* Avoid to grab roots in dead_roots */
4382                         if (btrfs_root_refs(&gang[i]->root_item) == 0) {
4383                                 gang[i] = NULL;
4384                                 continue;
4385                         }
4386                         /* grab all the search result for later use */
4387                         gang[i] = btrfs_grab_root(gang[i]);
4388                 }
4389                 spin_unlock(&fs_info->fs_roots_radix_lock);
4390
4391                 for (i = 0; i < ret; i++) {
4392                         if (!gang[i])
4393                                 continue;
4394                         root_objectid = gang[i]->root_key.objectid;
4395                         err = btrfs_orphan_cleanup(gang[i]);
4396                         if (err)
4397                                 break;
4398                         btrfs_put_root(gang[i]);
4399                 }
4400                 root_objectid++;
4401         }
4402
4403         /* release the uncleaned roots due to error */
4404         for (; i < ret; i++) {
4405                 if (gang[i])
4406                         btrfs_put_root(gang[i]);
4407         }
4408         return err;
4409 }
4410
4411 int btrfs_commit_super(struct btrfs_fs_info *fs_info)
4412 {
4413         struct btrfs_root *root = fs_info->tree_root;
4414         struct btrfs_trans_handle *trans;
4415
4416         mutex_lock(&fs_info->cleaner_mutex);
4417         btrfs_run_delayed_iputs(fs_info);
4418         mutex_unlock(&fs_info->cleaner_mutex);
4419         wake_up_process(fs_info->cleaner_kthread);
4420
4421         /* wait until ongoing cleanup work done */
4422         down_write(&fs_info->cleanup_work_sem);
4423         up_write(&fs_info->cleanup_work_sem);
4424
4425         trans = btrfs_join_transaction(root);
4426         if (IS_ERR(trans))
4427                 return PTR_ERR(trans);
4428         return btrfs_commit_transaction(trans);
4429 }
4430
4431 static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info)
4432 {
4433         struct btrfs_transaction *trans;
4434         struct btrfs_transaction *tmp;
4435         bool found = false;
4436
4437         if (list_empty(&fs_info->trans_list))
4438                 return;
4439
4440         /*
4441          * This function is only called at the very end of close_ctree(),
4442          * thus no other running transaction, no need to take trans_lock.
4443          */
4444         ASSERT(test_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags));
4445         list_for_each_entry_safe(trans, tmp, &fs_info->trans_list, list) {
4446                 struct extent_state *cached = NULL;
4447                 u64 dirty_bytes = 0;
4448                 u64 cur = 0;
4449                 u64 found_start;
4450                 u64 found_end;
4451
4452                 found = true;
4453                 while (!find_first_extent_bit(&trans->dirty_pages, cur,
4454                         &found_start, &found_end, EXTENT_DIRTY, &cached)) {
4455                         dirty_bytes += found_end + 1 - found_start;
4456                         cur = found_end + 1;
4457                 }
4458                 btrfs_warn(fs_info,
4459         "transaction %llu (with %llu dirty metadata bytes) is not committed",
4460                            trans->transid, dirty_bytes);
4461                 btrfs_cleanup_one_transaction(trans, fs_info);
4462
4463                 if (trans == fs_info->running_transaction)
4464                         fs_info->running_transaction = NULL;
4465                 list_del_init(&trans->list);
4466
4467                 btrfs_put_transaction(trans);
4468                 trace_btrfs_transaction_commit(fs_info);
4469         }
4470         ASSERT(!found);
4471 }
4472
4473 void __cold close_ctree(struct btrfs_fs_info *fs_info)
4474 {
4475         int ret;
4476
4477         set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags);
4478
4479         /*
4480          * We may have the reclaim task running and relocating a data block group,
4481          * in which case it may create delayed iputs. So stop it before we park
4482          * the cleaner kthread otherwise we can get new delayed iputs after
4483          * parking the cleaner, and that can make the async reclaim task to hang
4484          * if it's waiting for delayed iputs to complete, since the cleaner is
4485          * parked and can not run delayed iputs - this will make us hang when
4486          * trying to stop the async reclaim task.
4487          */
4488         cancel_work_sync(&fs_info->reclaim_bgs_work);
4489         /*
4490          * We don't want the cleaner to start new transactions, add more delayed
4491          * iputs, etc. while we're closing. We can't use kthread_stop() yet
4492          * because that frees the task_struct, and the transaction kthread might
4493          * still try to wake up the cleaner.
4494          */
4495         kthread_park(fs_info->cleaner_kthread);
4496
4497         /*
4498          * If we had UNFINISHED_DROPS we could still be processing them, so
4499          * clear that bit and wake up relocation so it can stop.
4500          */
4501         btrfs_wake_unfinished_drop(fs_info);
4502
4503         /* wait for the qgroup rescan worker to stop */
4504         btrfs_qgroup_wait_for_completion(fs_info, false);
4505
4506         /* wait for the uuid_scan task to finish */
4507         down(&fs_info->uuid_tree_rescan_sem);
4508         /* avoid complains from lockdep et al., set sem back to initial state */
4509         up(&fs_info->uuid_tree_rescan_sem);
4510
4511         /* pause restriper - we want to resume on mount */
4512         btrfs_pause_balance(fs_info);
4513
4514         btrfs_dev_replace_suspend_for_unmount(fs_info);
4515
4516         btrfs_scrub_cancel(fs_info);
4517
4518         /* wait for any defraggers to finish */
4519         wait_event(fs_info->transaction_wait,
4520                    (atomic_read(&fs_info->defrag_running) == 0));
4521
4522         /* clear out the rbtree of defraggable inodes */
4523         btrfs_cleanup_defrag_inodes(fs_info);
4524
4525         cancel_work_sync(&fs_info->async_reclaim_work);
4526         cancel_work_sync(&fs_info->async_data_reclaim_work);
4527         cancel_work_sync(&fs_info->preempt_reclaim_work);
4528
4529         /* Cancel or finish ongoing discard work */
4530         btrfs_discard_cleanup(fs_info);
4531
4532         if (!sb_rdonly(fs_info->sb)) {
4533                 /*
4534                  * The cleaner kthread is stopped, so do one final pass over
4535                  * unused block groups.
4536                  */
4537                 btrfs_delete_unused_bgs(fs_info);
4538
4539                 /*
4540                  * There might be existing delayed inode workers still running
4541                  * and holding an empty delayed inode item. We must wait for
4542                  * them to complete first because they can create a transaction.
4543                  * This happens when someone calls btrfs_balance_delayed_items()
4544                  * and then a transaction commit runs the same delayed nodes
4545                  * before any delayed worker has done something with the nodes.
4546                  * We must wait for any worker here and not at transaction
4547                  * commit time since that could cause a deadlock.
4548                  * This is a very rare case.
4549                  */
4550                 btrfs_flush_workqueue(fs_info->delayed_workers);
4551
4552                 ret = btrfs_commit_super(fs_info);
4553                 if (ret)
4554                         btrfs_err(fs_info, "commit super ret %d", ret);
4555         }
4556
4557         if (BTRFS_FS_ERROR(fs_info))
4558                 btrfs_error_commit_super(fs_info);
4559
4560         kthread_stop(fs_info->transaction_kthread);
4561         kthread_stop(fs_info->cleaner_kthread);
4562
4563         ASSERT(list_empty(&fs_info->delayed_iputs));
4564         set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags);
4565
4566         if (btrfs_check_quota_leak(fs_info)) {
4567                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4568                 btrfs_err(fs_info, "qgroup reserved space leaked");
4569         }
4570
4571         btrfs_free_qgroup_config(fs_info);
4572         ASSERT(list_empty(&fs_info->delalloc_roots));
4573
4574         if (percpu_counter_sum(&fs_info->delalloc_bytes)) {
4575                 btrfs_info(fs_info, "at unmount delalloc count %lld",
4576                        percpu_counter_sum(&fs_info->delalloc_bytes));
4577         }
4578
4579         if (percpu_counter_sum(&fs_info->ordered_bytes))
4580                 btrfs_info(fs_info, "at unmount dio bytes count %lld",
4581                            percpu_counter_sum(&fs_info->ordered_bytes));
4582
4583         btrfs_sysfs_remove_mounted(fs_info);
4584         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
4585
4586         btrfs_put_block_group_cache(fs_info);
4587
4588         /*
4589          * we must make sure there is not any read request to
4590          * submit after we stopping all workers.
4591          */
4592         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
4593         btrfs_stop_all_workers(fs_info);
4594
4595         /* We shouldn't have any transaction open at this point */
4596         warn_about_uncommitted_trans(fs_info);
4597
4598         clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
4599         free_root_pointers(fs_info, true);
4600         btrfs_free_fs_roots(fs_info);
4601
4602         /*
4603          * We must free the block groups after dropping the fs_roots as we could
4604          * have had an IO error and have left over tree log blocks that aren't
4605          * cleaned up until the fs roots are freed.  This makes the block group
4606          * accounting appear to be wrong because there's pending reserved bytes,
4607          * so make sure we do the block group cleanup afterwards.
4608          */
4609         btrfs_free_block_groups(fs_info);
4610
4611         iput(fs_info->btree_inode);
4612
4613 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4614         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY))
4615                 btrfsic_unmount(fs_info->fs_devices);
4616 #endif
4617
4618         btrfs_mapping_tree_free(&fs_info->mapping_tree);
4619         btrfs_close_devices(fs_info->fs_devices);
4620 }
4621
4622 int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
4623                           int atomic)
4624 {
4625         int ret;
4626         struct inode *btree_inode = buf->pages[0]->mapping->host;
4627
4628         ret = extent_buffer_uptodate(buf);
4629         if (!ret)
4630                 return ret;
4631
4632         ret = verify_parent_transid(&BTRFS_I(btree_inode)->io_tree, buf,
4633                                     parent_transid, atomic);
4634         if (ret == -EAGAIN)
4635                 return ret;
4636         return !ret;
4637 }
4638
4639 void btrfs_mark_buffer_dirty(struct extent_buffer *buf)
4640 {
4641         struct btrfs_fs_info *fs_info = buf->fs_info;
4642         u64 transid = btrfs_header_generation(buf);
4643         int was_dirty;
4644
4645 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4646         /*
4647          * This is a fast path so only do this check if we have sanity tests
4648          * enabled.  Normal people shouldn't be using unmapped buffers as dirty
4649          * outside of the sanity tests.
4650          */
4651         if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags)))
4652                 return;
4653 #endif
4654         btrfs_assert_tree_write_locked(buf);
4655         if (transid != fs_info->generation)
4656                 WARN(1, KERN_CRIT "btrfs transid mismatch buffer %llu, found %llu running %llu\n",
4657                         buf->start, transid, fs_info->generation);
4658         was_dirty = set_extent_buffer_dirty(buf);
4659         if (!was_dirty)
4660                 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4661                                          buf->len,
4662                                          fs_info->dirty_metadata_batch);
4663 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4664         /*
4665          * Since btrfs_mark_buffer_dirty() can be called with item pointer set
4666          * but item data not updated.
4667          * So here we should only check item pointers, not item data.
4668          */
4669         if (btrfs_header_level(buf) == 0 &&
4670             btrfs_check_leaf_relaxed(buf)) {
4671                 btrfs_print_leaf(buf);
4672                 ASSERT(0);
4673         }
4674 #endif
4675 }
4676
4677 static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info,
4678                                         int flush_delayed)
4679 {
4680         /*
4681          * looks as though older kernels can get into trouble with
4682          * this code, they end up stuck in balance_dirty_pages forever
4683          */
4684         int ret;
4685
4686         if (current->flags & PF_MEMALLOC)
4687                 return;
4688
4689         if (flush_delayed)
4690                 btrfs_balance_delayed_items(fs_info);
4691
4692         ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
4693                                      BTRFS_DIRTY_METADATA_THRESH,
4694                                      fs_info->dirty_metadata_batch);
4695         if (ret > 0) {
4696                 balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping);
4697         }
4698 }
4699
4700 void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info)
4701 {
4702         __btrfs_btree_balance_dirty(fs_info, 1);
4703 }
4704
4705 void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info)
4706 {
4707         __btrfs_btree_balance_dirty(fs_info, 0);
4708 }
4709
4710 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info)
4711 {
4712         /* cleanup FS via transaction */
4713         btrfs_cleanup_transaction(fs_info);
4714
4715         mutex_lock(&fs_info->cleaner_mutex);
4716         btrfs_run_delayed_iputs(fs_info);
4717         mutex_unlock(&fs_info->cleaner_mutex);
4718
4719         down_write(&fs_info->cleanup_work_sem);
4720         up_write(&fs_info->cleanup_work_sem);
4721 }
4722
4723 static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info)
4724 {
4725         struct btrfs_root *gang[8];
4726         u64 root_objectid = 0;
4727         int ret;
4728
4729         spin_lock(&fs_info->fs_roots_radix_lock);
4730         while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4731                                              (void **)gang, root_objectid,
4732                                              ARRAY_SIZE(gang))) != 0) {
4733                 int i;
4734
4735                 for (i = 0; i < ret; i++)
4736                         gang[i] = btrfs_grab_root(gang[i]);
4737                 spin_unlock(&fs_info->fs_roots_radix_lock);
4738
4739                 for (i = 0; i < ret; i++) {
4740                         if (!gang[i])
4741                                 continue;
4742                         root_objectid = gang[i]->root_key.objectid;
4743                         btrfs_free_log(NULL, gang[i]);
4744                         btrfs_put_root(gang[i]);
4745                 }
4746                 root_objectid++;
4747                 spin_lock(&fs_info->fs_roots_radix_lock);
4748         }
4749         spin_unlock(&fs_info->fs_roots_radix_lock);
4750         btrfs_free_log_root_tree(NULL, fs_info);
4751 }
4752
4753 static void btrfs_destroy_ordered_extents(struct btrfs_root *root)
4754 {
4755         struct btrfs_ordered_extent *ordered;
4756
4757         spin_lock(&root->ordered_extent_lock);
4758         /*
4759          * This will just short circuit the ordered completion stuff which will
4760          * make sure the ordered extent gets properly cleaned up.
4761          */
4762         list_for_each_entry(ordered, &root->ordered_extents,
4763                             root_extent_list)
4764                 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
4765         spin_unlock(&root->ordered_extent_lock);
4766 }
4767
4768 static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info)
4769 {
4770         struct btrfs_root *root;
4771         struct list_head splice;
4772
4773         INIT_LIST_HEAD(&splice);
4774
4775         spin_lock(&fs_info->ordered_root_lock);
4776         list_splice_init(&fs_info->ordered_roots, &splice);
4777         while (!list_empty(&splice)) {
4778                 root = list_first_entry(&splice, struct btrfs_root,
4779                                         ordered_root);
4780                 list_move_tail(&root->ordered_root,
4781                                &fs_info->ordered_roots);
4782
4783                 spin_unlock(&fs_info->ordered_root_lock);
4784                 btrfs_destroy_ordered_extents(root);
4785
4786                 cond_resched();
4787                 spin_lock(&fs_info->ordered_root_lock);
4788         }
4789         spin_unlock(&fs_info->ordered_root_lock);
4790
4791         /*
4792          * We need this here because if we've been flipped read-only we won't
4793          * get sync() from the umount, so we need to make sure any ordered
4794          * extents that haven't had their dirty pages IO start writeout yet
4795          * actually get run and error out properly.
4796          */
4797         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
4798 }
4799
4800 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
4801                                       struct btrfs_fs_info *fs_info)
4802 {
4803         struct rb_node *node;
4804         struct btrfs_delayed_ref_root *delayed_refs;
4805         struct btrfs_delayed_ref_node *ref;
4806         int ret = 0;
4807
4808         delayed_refs = &trans->delayed_refs;
4809
4810         spin_lock(&delayed_refs->lock);
4811         if (atomic_read(&delayed_refs->num_entries) == 0) {
4812                 spin_unlock(&delayed_refs->lock);
4813                 btrfs_debug(fs_info, "delayed_refs has NO entry");
4814                 return ret;
4815         }
4816
4817         while ((node = rb_first_cached(&delayed_refs->href_root)) != NULL) {
4818                 struct btrfs_delayed_ref_head *head;
4819                 struct rb_node *n;
4820                 bool pin_bytes = false;
4821
4822                 head = rb_entry(node, struct btrfs_delayed_ref_head,
4823                                 href_node);
4824                 if (btrfs_delayed_ref_lock(delayed_refs, head))
4825                         continue;
4826
4827                 spin_lock(&head->lock);
4828                 while ((n = rb_first_cached(&head->ref_tree)) != NULL) {
4829                         ref = rb_entry(n, struct btrfs_delayed_ref_node,
4830                                        ref_node);
4831                         ref->in_tree = 0;
4832                         rb_erase_cached(&ref->ref_node, &head->ref_tree);
4833                         RB_CLEAR_NODE(&ref->ref_node);
4834                         if (!list_empty(&ref->add_list))
4835                                 list_del(&ref->add_list);
4836                         atomic_dec(&delayed_refs->num_entries);
4837                         btrfs_put_delayed_ref(ref);
4838                 }
4839                 if (head->must_insert_reserved)
4840                         pin_bytes = true;
4841                 btrfs_free_delayed_extent_op(head->extent_op);
4842                 btrfs_delete_ref_head(delayed_refs, head);
4843                 spin_unlock(&head->lock);
4844                 spin_unlock(&delayed_refs->lock);
4845                 mutex_unlock(&head->mutex);
4846
4847                 if (pin_bytes) {
4848                         struct btrfs_block_group *cache;
4849
4850                         cache = btrfs_lookup_block_group(fs_info, head->bytenr);
4851                         BUG_ON(!cache);
4852
4853                         spin_lock(&cache->space_info->lock);
4854                         spin_lock(&cache->lock);
4855                         cache->pinned += head->num_bytes;
4856                         btrfs_space_info_update_bytes_pinned(fs_info,
4857                                 cache->space_info, head->num_bytes);
4858                         cache->reserved -= head->num_bytes;
4859                         cache->space_info->bytes_reserved -= head->num_bytes;
4860                         spin_unlock(&cache->lock);
4861                         spin_unlock(&cache->space_info->lock);
4862
4863                         btrfs_put_block_group(cache);
4864
4865                         btrfs_error_unpin_extent_range(fs_info, head->bytenr,
4866                                 head->bytenr + head->num_bytes - 1);
4867                 }
4868                 btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head);
4869                 btrfs_put_delayed_ref_head(head);
4870                 cond_resched();
4871                 spin_lock(&delayed_refs->lock);
4872         }
4873         btrfs_qgroup_destroy_extent_records(trans);
4874
4875         spin_unlock(&delayed_refs->lock);
4876
4877         return ret;
4878 }
4879
4880 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root)
4881 {
4882         struct btrfs_inode *btrfs_inode;
4883         struct list_head splice;
4884
4885         INIT_LIST_HEAD(&splice);
4886
4887         spin_lock(&root->delalloc_lock);
4888         list_splice_init(&root->delalloc_inodes, &splice);
4889
4890         while (!list_empty(&splice)) {
4891                 struct inode *inode = NULL;
4892                 btrfs_inode = list_first_entry(&splice, struct btrfs_inode,
4893                                                delalloc_inodes);
4894                 __btrfs_del_delalloc_inode(root, btrfs_inode);
4895                 spin_unlock(&root->delalloc_lock);
4896
4897                 /*
4898                  * Make sure we get a live inode and that it'll not disappear
4899                  * meanwhile.
4900                  */
4901                 inode = igrab(&btrfs_inode->vfs_inode);
4902                 if (inode) {
4903                         invalidate_inode_pages2(inode->i_mapping);
4904                         iput(inode);
4905                 }
4906                 spin_lock(&root->delalloc_lock);
4907         }
4908         spin_unlock(&root->delalloc_lock);
4909 }
4910
4911 static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info)
4912 {
4913         struct btrfs_root *root;
4914         struct list_head splice;
4915
4916         INIT_LIST_HEAD(&splice);
4917
4918         spin_lock(&fs_info->delalloc_root_lock);
4919         list_splice_init(&fs_info->delalloc_roots, &splice);
4920         while (!list_empty(&splice)) {
4921                 root = list_first_entry(&splice, struct btrfs_root,
4922                                          delalloc_root);
4923                 root = btrfs_grab_root(root);
4924                 BUG_ON(!root);
4925                 spin_unlock(&fs_info->delalloc_root_lock);
4926
4927                 btrfs_destroy_delalloc_inodes(root);
4928                 btrfs_put_root(root);
4929
4930                 spin_lock(&fs_info->delalloc_root_lock);
4931         }
4932         spin_unlock(&fs_info->delalloc_root_lock);
4933 }
4934
4935 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
4936                                         struct extent_io_tree *dirty_pages,
4937                                         int mark)
4938 {
4939         int ret;
4940         struct extent_buffer *eb;
4941         u64 start = 0;
4942         u64 end;
4943
4944         while (1) {
4945                 ret = find_first_extent_bit(dirty_pages, start, &start, &end,
4946                                             mark, NULL);
4947                 if (ret)
4948                         break;
4949
4950                 clear_extent_bits(dirty_pages, start, end, mark);
4951                 while (start <= end) {
4952                         eb = find_extent_buffer(fs_info, start);
4953                         start += fs_info->nodesize;
4954                         if (!eb)
4955                                 continue;
4956                         wait_on_extent_buffer_writeback(eb);
4957
4958                         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY,
4959                                                &eb->bflags))
4960                                 clear_extent_buffer_dirty(eb);
4961                         free_extent_buffer_stale(eb);
4962                 }
4963         }
4964
4965         return ret;
4966 }
4967
4968 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
4969                                        struct extent_io_tree *unpin)
4970 {
4971         u64 start;
4972         u64 end;
4973         int ret;
4974
4975         while (1) {
4976                 struct extent_state *cached_state = NULL;
4977
4978                 /*
4979                  * The btrfs_finish_extent_commit() may get the same range as
4980                  * ours between find_first_extent_bit and clear_extent_dirty.
4981                  * Hence, hold the unused_bg_unpin_mutex to avoid double unpin
4982                  * the same extent range.
4983                  */
4984                 mutex_lock(&fs_info->unused_bg_unpin_mutex);
4985                 ret = find_first_extent_bit(unpin, 0, &start, &end,
4986                                             EXTENT_DIRTY, &cached_state);
4987                 if (ret) {
4988                         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
4989                         break;
4990                 }
4991
4992                 clear_extent_dirty(unpin, start, end, &cached_state);
4993                 free_extent_state(cached_state);
4994                 btrfs_error_unpin_extent_range(fs_info, start, end);
4995                 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
4996                 cond_resched();
4997         }
4998
4999         return 0;
5000 }
5001
5002 static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache)
5003 {
5004         struct inode *inode;
5005
5006         inode = cache->io_ctl.inode;
5007         if (inode) {
5008                 invalidate_inode_pages2(inode->i_mapping);
5009                 BTRFS_I(inode)->generation = 0;
5010                 cache->io_ctl.inode = NULL;
5011                 iput(inode);
5012         }
5013         ASSERT(cache->io_ctl.pages == NULL);
5014         btrfs_put_block_group(cache);
5015 }
5016
5017 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
5018                              struct btrfs_fs_info *fs_info)
5019 {
5020         struct btrfs_block_group *cache;
5021
5022         spin_lock(&cur_trans->dirty_bgs_lock);
5023         while (!list_empty(&cur_trans->dirty_bgs)) {
5024                 cache = list_first_entry(&cur_trans->dirty_bgs,
5025                                          struct btrfs_block_group,
5026                                          dirty_list);
5027
5028                 if (!list_empty(&cache->io_list)) {
5029                         spin_unlock(&cur_trans->dirty_bgs_lock);
5030                         list_del_init(&cache->io_list);
5031                         btrfs_cleanup_bg_io(cache);
5032                         spin_lock(&cur_trans->dirty_bgs_lock);
5033                 }
5034
5035                 list_del_init(&cache->dirty_list);
5036                 spin_lock(&cache->lock);
5037                 cache->disk_cache_state = BTRFS_DC_ERROR;
5038                 spin_unlock(&cache->lock);
5039
5040                 spin_unlock(&cur_trans->dirty_bgs_lock);
5041                 btrfs_put_block_group(cache);
5042                 btrfs_delayed_refs_rsv_release(fs_info, 1);
5043                 spin_lock(&cur_trans->dirty_bgs_lock);
5044         }
5045         spin_unlock(&cur_trans->dirty_bgs_lock);
5046
5047         /*
5048          * Refer to the definition of io_bgs member for details why it's safe
5049          * to use it without any locking
5050          */
5051         while (!list_empty(&cur_trans->io_bgs)) {
5052                 cache = list_first_entry(&cur_trans->io_bgs,
5053                                          struct btrfs_block_group,
5054                                          io_list);
5055
5056                 list_del_init(&cache->io_list);
5057                 spin_lock(&cache->lock);
5058                 cache->disk_cache_state = BTRFS_DC_ERROR;
5059                 spin_unlock(&cache->lock);
5060                 btrfs_cleanup_bg_io(cache);
5061         }
5062 }
5063
5064 void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans,
5065                                    struct btrfs_fs_info *fs_info)
5066 {
5067         struct btrfs_device *dev, *tmp;
5068
5069         btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
5070         ASSERT(list_empty(&cur_trans->dirty_bgs));
5071         ASSERT(list_empty(&cur_trans->io_bgs));
5072
5073         list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list,
5074                                  post_commit_list) {
5075                 list_del_init(&dev->post_commit_list);
5076         }
5077
5078         btrfs_destroy_delayed_refs(cur_trans, fs_info);
5079
5080         cur_trans->state = TRANS_STATE_COMMIT_START;
5081         wake_up(&fs_info->transaction_blocked_wait);
5082
5083         cur_trans->state = TRANS_STATE_UNBLOCKED;
5084         wake_up(&fs_info->transaction_wait);
5085
5086         btrfs_destroy_delayed_inodes(fs_info);
5087
5088         btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages,
5089                                      EXTENT_DIRTY);
5090         btrfs_destroy_pinned_extent(fs_info, &cur_trans->pinned_extents);
5091
5092         btrfs_free_redirty_list(cur_trans);
5093
5094         cur_trans->state =TRANS_STATE_COMPLETED;
5095         wake_up(&cur_trans->commit_wait);
5096 }
5097
5098 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info)
5099 {
5100         struct btrfs_transaction *t;
5101
5102         mutex_lock(&fs_info->transaction_kthread_mutex);
5103
5104         spin_lock(&fs_info->trans_lock);
5105         while (!list_empty(&fs_info->trans_list)) {
5106                 t = list_first_entry(&fs_info->trans_list,
5107                                      struct btrfs_transaction, list);
5108                 if (t->state >= TRANS_STATE_COMMIT_START) {
5109                         refcount_inc(&t->use_count);
5110                         spin_unlock(&fs_info->trans_lock);
5111                         btrfs_wait_for_commit(fs_info, t->transid);
5112                         btrfs_put_transaction(t);
5113                         spin_lock(&fs_info->trans_lock);
5114                         continue;
5115                 }
5116                 if (t == fs_info->running_transaction) {
5117                         t->state = TRANS_STATE_COMMIT_DOING;
5118                         spin_unlock(&fs_info->trans_lock);
5119                         /*
5120                          * We wait for 0 num_writers since we don't hold a trans
5121                          * handle open currently for this transaction.
5122                          */
5123                         wait_event(t->writer_wait,
5124                                    atomic_read(&t->num_writers) == 0);
5125                 } else {
5126                         spin_unlock(&fs_info->trans_lock);
5127                 }
5128                 btrfs_cleanup_one_transaction(t, fs_info);
5129
5130                 spin_lock(&fs_info->trans_lock);
5131                 if (t == fs_info->running_transaction)
5132                         fs_info->running_transaction = NULL;
5133                 list_del_init(&t->list);
5134                 spin_unlock(&fs_info->trans_lock);
5135
5136                 btrfs_put_transaction(t);
5137                 trace_btrfs_transaction_commit(fs_info);
5138                 spin_lock(&fs_info->trans_lock);
5139         }
5140         spin_unlock(&fs_info->trans_lock);
5141         btrfs_destroy_all_ordered_extents(fs_info);
5142         btrfs_destroy_delayed_inodes(fs_info);
5143         btrfs_assert_delayed_root_empty(fs_info);
5144         btrfs_destroy_all_delalloc_inodes(fs_info);
5145         btrfs_drop_all_logs(fs_info);
5146         mutex_unlock(&fs_info->transaction_kthread_mutex);
5147
5148         return 0;
5149 }
5150
5151 int btrfs_init_root_free_objectid(struct btrfs_root *root)
5152 {
5153         struct btrfs_path *path;
5154         int ret;
5155         struct extent_buffer *l;
5156         struct btrfs_key search_key;
5157         struct btrfs_key found_key;
5158         int slot;
5159
5160         path = btrfs_alloc_path();
5161         if (!path)
5162                 return -ENOMEM;
5163
5164         search_key.objectid = BTRFS_LAST_FREE_OBJECTID;
5165         search_key.type = -1;
5166         search_key.offset = (u64)-1;
5167         ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0);
5168         if (ret < 0)
5169                 goto error;
5170         BUG_ON(ret == 0); /* Corruption */
5171         if (path->slots[0] > 0) {
5172                 slot = path->slots[0] - 1;
5173                 l = path->nodes[0];
5174                 btrfs_item_key_to_cpu(l, &found_key, slot);
5175                 root->free_objectid = max_t(u64, found_key.objectid + 1,
5176                                             BTRFS_FIRST_FREE_OBJECTID);
5177         } else {
5178                 root->free_objectid = BTRFS_FIRST_FREE_OBJECTID;
5179         }
5180         ret = 0;
5181 error:
5182         btrfs_free_path(path);
5183         return ret;
5184 }
5185
5186 int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid)
5187 {
5188         int ret;
5189         mutex_lock(&root->objectid_mutex);
5190
5191         if (unlikely(root->free_objectid >= BTRFS_LAST_FREE_OBJECTID)) {
5192                 btrfs_warn(root->fs_info,
5193                            "the objectid of root %llu reaches its highest value",
5194                            root->root_key.objectid);
5195                 ret = -ENOSPC;
5196                 goto out;
5197         }
5198
5199         *objectid = root->free_objectid++;
5200         ret = 0;
5201 out:
5202         mutex_unlock(&root->objectid_mutex);
5203         return ret;
5204 }