fs/btrfs/disk-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/blkdev.h>
   8 #include <linux/radix-tree.h>
   9 #include <linux/writeback.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/kthread.h>
  12 #include <linux/slab.h>
  13 #include <linux/migrate.h>
  14 #include <linux/ratelimit.h>
  15 #include <linux/uuid.h>
  16 #include <linux/semaphore.h>
  17 #include <linux/error-injection.h>
  18 #include <linux/crc32c.h>
  19 #include <linux/sched/mm.h>
  20 #include <asm/unaligned.h>
  21 #include <crypto/hash.h>
  22 #include "ctree.h"
  23 #include "disk-io.h"
  24 #include "transaction.h"
  25 #include "btrfs_inode.h"
  26 #include "volumes.h"
  27 #include "print-tree.h"
  28 #include "locking.h"
  29 #include "tree-log.h"
  30 #include "free-space-cache.h"
  31 #include "free-space-tree.h"
  32 #include "check-integrity.h"
  33 #include "rcu-string.h"
  34 #include "dev-replace.h"
  35 #include "raid56.h"
  36 #include "sysfs.h"
  37 #include "qgroup.h"
  38 #include "compression.h"
  39 #include "tree-checker.h"
  40 #include "ref-verify.h"
  41 #include "block-group.h"
  42 #include "discard.h"
  43 #include "space-info.h"
  44 #include "zoned.h"
  45 #include "subpage.h"
  46
  47 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
  48                                  BTRFS_HEADER_FLAG_RELOC |\
  49                                  BTRFS_SUPER_FLAG_ERROR |\
  50                                  BTRFS_SUPER_FLAG_SEEDING |\
  51                                  BTRFS_SUPER_FLAG_METADUMP |\
  52                                  BTRFS_SUPER_FLAG_METADUMP_V2)
  53
  54 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
  55 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
  56                                       struct btrfs_fs_info *fs_info);
  57 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
  58 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
  59                                         struct extent_io_tree *dirty_pages,
  60                                         int mark);
  61 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
  62                                        struct extent_io_tree *pinned_extents);
  63 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
  64 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
  65
  66 static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
  67 {
  68         if (fs_info->csum_shash)
  69                 crypto_free_shash(fs_info->csum_shash);
  70 }
  71
  72 /*
  73  * async submit bios are used to offload expensive checksumming
  74  * onto the worker threads.  They checksum file and metadata bios
  75  * just before they are sent down the IO stack.
  76  */
  77 struct async_submit_bio {
  78         struct inode *inode;
  79         struct bio *bio;
  80         extent_submit_bio_start_t *submit_bio_start;
  81         int mirror_num;
  82
  83         /* Optional parameter for submit_bio_start used by direct io */
  84         u64 dio_file_offset;
  85         struct btrfs_work work;
  86         blk_status_t status;
  87 };
  88
  89 /*
  90  * Compute the csum of a btree block and store the result to provided buffer.
  91  */
  92 static void csum_tree_block(struct extent_buffer *buf, u8 *result)
  93 {
  94         struct btrfs_fs_info *fs_info = buf->fs_info;
  95         const int num_pages = num_extent_pages(buf);
  96         const int first_page_part = min_t(u32, PAGE_SIZE, fs_info->nodesize);
  97         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
  98         char *kaddr;
  99         int i;
 100
 101         shash->tfm = fs_info->csum_shash;
 102         crypto_shash_init(shash);
 103         kaddr = page_address(buf->pages[0]) + offset_in_page(buf->start);
 104         crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
 105                             first_page_part - BTRFS_CSUM_SIZE);
 106
 107         for (i = 1; i < num_pages; i++) {
 108                 kaddr = page_address(buf->pages[i]);
 109                 crypto_shash_update(shash, kaddr, PAGE_SIZE);
 110         }
 111         memset(result, 0, BTRFS_CSUM_SIZE);
 112         crypto_shash_final(shash, result);
 113 }
 114
 115 /*
 116  * we can't consider a given block up to date unless the transid of the
 117  * block matches the transid in the parent node's pointer.  This is how we
 118  * detect blocks that either didn't get written at all or got written
 119  * in the wrong place.
 120  */
 121 static int verify_parent_transid(struct extent_io_tree *io_tree,
 122                                  struct extent_buffer *eb, u64 parent_transid,
 123                                  int atomic)
 124 {
 125         struct extent_state *cached_state = NULL;
 126         int ret;
 127
 128         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
 129                 return 0;
 130
 131         if (atomic)
 132                 return -EAGAIN;
 133
 134         lock_extent(io_tree, eb->start, eb->start + eb->len - 1, &cached_state);
 135         if (extent_buffer_uptodate(eb) &&
 136             btrfs_header_generation(eb) == parent_transid) {
 137                 ret = 0;
 138                 goto out;
 139         }
 140         btrfs_err_rl(eb->fs_info,
 141 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
 142                         eb->start, eb->read_mirror,
 143                         parent_transid, btrfs_header_generation(eb));
 144         ret = 1;
 145         clear_extent_buffer_uptodate(eb);
 146 out:
 147         unlock_extent(io_tree, eb->start, eb->start + eb->len - 1,
 148                       &cached_state);
 149         return ret;
 150 }
 151
 152 static bool btrfs_supported_super_csum(u16 csum_type)
 153 {
 154         switch (csum_type) {
 155         case BTRFS_CSUM_TYPE_CRC32:
 156         case BTRFS_CSUM_TYPE_XXHASH:
 157         case BTRFS_CSUM_TYPE_SHA256:
 158         case BTRFS_CSUM_TYPE_BLAKE2:
 159                 return true;
 160         default:
 161                 return false;
 162         }
 163 }
 164
 165 /*
 166  * Return 0 if the superblock checksum type matches the checksum value of that
 167  * algorithm. Pass the raw disk superblock data.
 168  */
 169 static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 170                                   char *raw_disk_sb)
 171 {
 172         struct btrfs_super_block *disk_sb =
 173                 (struct btrfs_super_block *)raw_disk_sb;
 174         char result[BTRFS_CSUM_SIZE];
 175         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 176
 177         shash->tfm = fs_info->csum_shash;
 178
 179         /*
 180          * The super_block structure does not span the whole
 181          * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
 182          * filled with zeros and is included in the checksum.
 183          */
 184         crypto_shash_digest(shash, raw_disk_sb + BTRFS_CSUM_SIZE,
 185                             BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
 186
 187         if (memcmp(disk_sb->csum, result, fs_info->csum_size))
 188                 return 1;
 189
 190         return 0;
 191 }
 192
 193 int btrfs_verify_level_key(struct extent_buffer *eb, int level,
 194                            struct btrfs_key *first_key, u64 parent_transid)
 195 {
 196         struct btrfs_fs_info *fs_info = eb->fs_info;
 197         int found_level;
 198         struct btrfs_key found_key;
 199         int ret;
 200
 201         found_level = btrfs_header_level(eb);
 202         if (found_level != level) {
 203                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 204                      KERN_ERR "BTRFS: tree level check failed\n");
 205                 btrfs_err(fs_info,
 206 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
 207                           eb->start, level, found_level);
 208                 return -EIO;
 209         }
 210
 211         if (!first_key)
 212                 return 0;
 213
 214         /*
 215          * For live tree block (new tree blocks in current transaction),
 216          * we need proper lock context to avoid race, which is impossible here.
 217          * So we only checks tree blocks which is read from disk, whose
 218          * generation <= fs_info->last_trans_committed.
 219          */
 220         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
 221                 return 0;
 222
 223         /* We have @first_key, so this @eb must have at least one item */
 224         if (btrfs_header_nritems(eb) == 0) {
 225                 btrfs_err(fs_info,
 226                 "invalid tree nritems, bytenr=%llu nritems=0 expect >0",
 227                           eb->start);
 228                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 229                 return -EUCLEAN;
 230         }
 231
 232         if (found_level)
 233                 btrfs_node_key_to_cpu(eb, &found_key, 0);
 234         else
 235                 btrfs_item_key_to_cpu(eb, &found_key, 0);
 236         ret = btrfs_comp_cpu_keys(first_key, &found_key);
 237
 238         if (ret) {
 239                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 240                      KERN_ERR "BTRFS: tree first key check failed\n");
 241                 btrfs_err(fs_info,
 242 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 243                           eb->start, parent_transid, first_key->objectid,
 244                           first_key->type, first_key->offset,
 245                           found_key.objectid, found_key.type,
 246                           found_key.offset);
 247         }
 248         return ret;
 249 }
 250
 251 /*
 252  * helper to read a given tree block, doing retries as required when
 253  * the checksums don't match and we have alternate mirrors to try.
 254  *
 255  * @parent_transid:     expected transid, skip check if 0
 256  * @level:              expected level, mandatory check
 257  * @first_key:          expected key of first slot, skip check if NULL
 258  */
 259 int btrfs_read_extent_buffer(struct extent_buffer *eb,
 260                              u64 parent_transid, int level,
 261                              struct btrfs_key *first_key)
 262 {
 263         struct btrfs_fs_info *fs_info = eb->fs_info;
 264         struct extent_io_tree *io_tree;
 265         int failed = 0;
 266         int ret;
 267         int num_copies = 0;
 268         int mirror_num = 0;
 269         int failed_mirror = 0;
 270
 271         io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
 272         while (1) {
 273                 clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 274                 ret = read_extent_buffer_pages(eb, WAIT_COMPLETE, mirror_num);
 275                 if (!ret) {
 276                         if (verify_parent_transid(io_tree, eb,
 277                                                    parent_transid, 0))
 278                                 ret = -EIO;
 279                         else if (btrfs_verify_level_key(eb, level,
 280                                                 first_key, parent_transid))
 281                                 ret = -EUCLEAN;
 282                         else
 283                                 break;
 284                 }
 285
 286                 num_copies = btrfs_num_copies(fs_info,
 287                                               eb->start, eb->len);
 288                 if (num_copies == 1)
 289                         break;
 290
 291                 if (!failed_mirror) {
 292                         failed = 1;
 293                         failed_mirror = eb->read_mirror;
 294                 }
 295
 296                 mirror_num++;
 297                 if (mirror_num == failed_mirror)
 298                         mirror_num++;
 299
 300                 if (mirror_num > num_copies)
 301                         break;
 302         }
 303
 304         if (failed && !ret && failed_mirror)
 305                 btrfs_repair_eb_io_failure(eb, failed_mirror);
 306
 307         return ret;
 308 }
 309
 310 static int csum_one_extent_buffer(struct extent_buffer *eb)
 311 {
 312         struct btrfs_fs_info *fs_info = eb->fs_info;
 313         u8 result[BTRFS_CSUM_SIZE];
 314         int ret;
 315
 316         ASSERT(memcmp_extent_buffer(eb, fs_info->fs_devices->metadata_uuid,
 317                                     offsetof(struct btrfs_header, fsid),
 318                                     BTRFS_FSID_SIZE) == 0);
 319         csum_tree_block(eb, result);
 320
 321         if (btrfs_header_level(eb))
 322                 ret = btrfs_check_node(eb);
 323         else
 324                 ret = btrfs_check_leaf_full(eb);
 325
 326         if (ret < 0)
 327                 goto error;
 328
 329         /*
 330          * Also check the generation, the eb reached here must be newer than
 331          * last committed. Or something seriously wrong happened.
 332          */
 333         if (unlikely(btrfs_header_generation(eb) <= fs_info->last_trans_committed)) {
 334                 ret = -EUCLEAN;
 335                 btrfs_err(fs_info,
 336                         "block=%llu bad generation, have %llu expect > %llu",
 337                           eb->start, btrfs_header_generation(eb),
 338                           fs_info->last_trans_committed);
 339                 goto error;
 340         }
 341         write_extent_buffer(eb, result, 0, fs_info->csum_size);
 342
 343         return 0;
 344
 345 error:
 346         btrfs_print_tree(eb, 0);
 347         btrfs_err(fs_info, "block=%llu write time tree block corruption detected",
 348                   eb->start);
 349         WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 350         return ret;
 351 }
 352
 353 /* Checksum all dirty extent buffers in one bio_vec */
 354 static int csum_dirty_subpage_buffers(struct btrfs_fs_info *fs_info,
 355                                       struct bio_vec *bvec)
 356 {
 357         struct page *page = bvec->bv_page;
 358         u64 bvec_start = page_offset(page) + bvec->bv_offset;
 359         u64 cur;
 360         int ret = 0;
 361
 362         for (cur = bvec_start; cur < bvec_start + bvec->bv_len;
 363              cur += fs_info->nodesize) {
 364                 struct extent_buffer *eb;
 365                 bool uptodate;
 366
 367                 eb = find_extent_buffer(fs_info, cur);
 368                 uptodate = btrfs_subpage_test_uptodate(fs_info, page, cur,
 369                                                        fs_info->nodesize);
 370
 371                 /* A dirty eb shouldn't disappear from buffer_radix */
 372                 if (WARN_ON(!eb))
 373                         return -EUCLEAN;
 374
 375                 if (WARN_ON(cur != btrfs_header_bytenr(eb))) {
 376                         free_extent_buffer(eb);
 377                         return -EUCLEAN;
 378                 }
 379                 if (WARN_ON(!uptodate)) {
 380                         free_extent_buffer(eb);
 381                         return -EUCLEAN;
 382                 }
 383
 384                 ret = csum_one_extent_buffer(eb);
 385                 free_extent_buffer(eb);
 386                 if (ret < 0)
 387                         return ret;
 388         }
 389         return ret;
 390 }
 391
 392 /*
 393  * Checksum a dirty tree block before IO.  This has extra checks to make sure
 394  * we only fill in the checksum field in the first page of a multi-page block.
 395  * For subpage extent buffers we need bvec to also read the offset in the page.
 396  */
 397 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec)
 398 {
 399         struct page *page = bvec->bv_page;
 400         u64 start = page_offset(page);
 401         u64 found_start;
 402         struct extent_buffer *eb;
 403
 404         if (fs_info->nodesize < PAGE_SIZE)
 405                 return csum_dirty_subpage_buffers(fs_info, bvec);
 406
 407         eb = (struct extent_buffer *)page->private;
 408         if (page != eb->pages[0])
 409                 return 0;
 410
 411         found_start = btrfs_header_bytenr(eb);
 412
 413         if (test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)) {
 414                 WARN_ON(found_start != 0);
 415                 return 0;
 416         }
 417
 418         /*
 419          * Please do not consolidate these warnings into a single if.
 420          * It is useful to know what went wrong.
 421          */
 422         if (WARN_ON(found_start != start))
 423                 return -EUCLEAN;
 424         if (WARN_ON(!PageUptodate(page)))
 425                 return -EUCLEAN;
 426
 427         return csum_one_extent_buffer(eb);
 428 }
 429
 430 static int check_tree_block_fsid(struct extent_buffer *eb)
 431 {
 432         struct btrfs_fs_info *fs_info = eb->fs_info;
 433         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
 434         u8 fsid[BTRFS_FSID_SIZE];
 435         u8 *metadata_uuid;
 436
 437         read_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid),
 438                            BTRFS_FSID_SIZE);
 439         /*
 440          * Checking the incompat flag is only valid for the current fs. For
 441          * seed devices it's forbidden to have their uuid changed so reading
 442          * ->fsid in this case is fine
 443          */
 444         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
 445                 metadata_uuid = fs_devices->metadata_uuid;
 446         else
 447                 metadata_uuid = fs_devices->fsid;
 448
 449         if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE))
 450                 return 0;
 451
 452         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list)
 453                 if (!memcmp(fsid, seed_devs->fsid, BTRFS_FSID_SIZE))
 454                         return 0;
 455
 456         return 1;
 457 }
 458
 459 /* Do basic extent buffer checks at read time */
 460 static int validate_extent_buffer(struct extent_buffer *eb)
 461 {
 462         struct btrfs_fs_info *fs_info = eb->fs_info;
 463         u64 found_start;
 464         const u32 csum_size = fs_info->csum_size;
 465         u8 found_level;
 466         u8 result[BTRFS_CSUM_SIZE];
 467         const u8 *header_csum;
 468         int ret = 0;
 469
 470         found_start = btrfs_header_bytenr(eb);
 471         if (found_start != eb->start) {
 472                 btrfs_err_rl(fs_info,
 473                         "bad tree block start, mirror %u want %llu have %llu",
 474                              eb->read_mirror, eb->start, found_start);
 475                 ret = -EIO;
 476                 goto out;
 477         }
 478         if (check_tree_block_fsid(eb)) {
 479                 btrfs_err_rl(fs_info, "bad fsid on logical %llu mirror %u",
 480                              eb->start, eb->read_mirror);
 481                 ret = -EIO;
 482                 goto out;
 483         }
 484         found_level = btrfs_header_level(eb);
 485         if (found_level >= BTRFS_MAX_LEVEL) {
 486                 btrfs_err(fs_info,
 487                         "bad tree block level, mirror %u level %d on logical %llu",
 488                         eb->read_mirror, btrfs_header_level(eb), eb->start);
 489                 ret = -EIO;
 490                 goto out;
 491         }
 492
 493         csum_tree_block(eb, result);
 494         header_csum = page_address(eb->pages[0]) +
 495                 get_eb_offset_in_page(eb, offsetof(struct btrfs_header, csum));
 496
 497         if (memcmp(result, header_csum, csum_size) != 0) {
 498                 btrfs_warn_rl(fs_info,
 499 "checksum verify failed on logical %llu mirror %u wanted " CSUM_FMT " found " CSUM_FMT " level %d",
 500                               eb->start, eb->read_mirror,
 501                               CSUM_FMT_VALUE(csum_size, header_csum),
 502                               CSUM_FMT_VALUE(csum_size, result),
 503                               btrfs_header_level(eb));
 504                 ret = -EUCLEAN;
 505                 goto out;
 506         }
 507
 508         /*
 509          * If this is a leaf block and it is corrupt, set the corrupt bit so
 510          * that we don't try and read the other copies of this block, just
 511          * return -EIO.
 512          */
 513         if (found_level == 0 && btrfs_check_leaf_full(eb)) {
 514                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 515                 ret = -EIO;
 516         }
 517
 518         if (found_level > 0 && btrfs_check_node(eb))
 519                 ret = -EIO;
 520
 521         if (!ret)
 522                 set_extent_buffer_uptodate(eb);
 523         else
 524                 btrfs_err(fs_info,
 525                 "read time tree block corruption detected on logical %llu mirror %u",
 526                           eb->start, eb->read_mirror);
 527 out:
 528         return ret;
 529 }
 530
 531 static int validate_subpage_buffer(struct page *page, u64 start, u64 end,
 532                                    int mirror)
 533 {
 534         struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
 535         struct extent_buffer *eb;
 536         bool reads_done;
 537         int ret = 0;
 538
 539         /*
 540          * We don't allow bio merge for subpage metadata read, so we should
 541          * only get one eb for each endio hook.
 542          */
 543         ASSERT(end == start + fs_info->nodesize - 1);
 544         ASSERT(PagePrivate(page));
 545
 546         eb = find_extent_buffer(fs_info, start);
 547         /*
 548          * When we are reading one tree block, eb must have been inserted into
 549          * the radix tree. If not, something is wrong.
 550          */
 551         ASSERT(eb);
 552
 553         reads_done = atomic_dec_and_test(&eb->io_pages);
 554         /* Subpage read must finish in page read */
 555         ASSERT(reads_done);
 556
 557         eb->read_mirror = mirror;
 558         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 559                 ret = -EIO;
 560                 goto err;
 561         }
 562         ret = validate_extent_buffer(eb);
 563         if (ret < 0)
 564                 goto err;
 565
 566         set_extent_buffer_uptodate(eb);
 567
 568         free_extent_buffer(eb);
 569         return ret;
 570 err:
 571         /*
 572          * end_bio_extent_readpage decrements io_pages in case of error,
 573          * make sure it has something to decrement.
 574          */
 575         atomic_inc(&eb->io_pages);
 576         clear_extent_buffer_uptodate(eb);
 577         free_extent_buffer(eb);
 578         return ret;
 579 }
 580
 581 int btrfs_validate_metadata_buffer(struct btrfs_bio *bbio,
 582                                    struct page *page, u64 start, u64 end,
 583                                    int mirror)
 584 {
 585         struct extent_buffer *eb;
 586         int ret = 0;
 587         int reads_done;
 588
 589         ASSERT(page->private);
 590
 591         if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
 592                 return validate_subpage_buffer(page, start, end, mirror);
 593
 594         eb = (struct extent_buffer *)page->private;
 595
 596         /*
 597          * The pending IO might have been the only thing that kept this buffer
 598          * in memory.  Make sure we have a ref for all this other checks
 599          */
 600         atomic_inc(&eb->refs);
 601
 602         reads_done = atomic_dec_and_test(&eb->io_pages);
 603         if (!reads_done)
 604                 goto err;
 605
 606         eb->read_mirror = mirror;
 607         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 608                 ret = -EIO;
 609                 goto err;
 610         }
 611         ret = validate_extent_buffer(eb);
 612 err:
 613         if (ret) {
 614                 /*
 615                  * our io error hook is going to dec the io pages
 616                  * again, we have to make sure it has something
 617                  * to decrement
 618                  */
 619                 atomic_inc(&eb->io_pages);
 620                 clear_extent_buffer_uptodate(eb);
 621         }
 622         free_extent_buffer(eb);
 623
 624         return ret;
 625 }
 626
 627 static void run_one_async_start(struct btrfs_work *work)
 628 {
 629         struct async_submit_bio *async;
 630         blk_status_t ret;
 631
 632         async = container_of(work, struct  async_submit_bio, work);
 633         ret = async->submit_bio_start(async->inode, async->bio,
 634                                       async->dio_file_offset);
 635         if (ret)
 636                 async->status = ret;
 637 }
 638
 639 /*
 640  * In order to insert checksums into the metadata in large chunks, we wait
 641  * until bio submission time.   All the pages in the bio are checksummed and
 642  * sums are attached onto the ordered extent record.
 643  *
 644  * At IO completion time the csums attached on the ordered extent record are
 645  * inserted into the tree.
 646  */
 647 static void run_one_async_done(struct btrfs_work *work)
 648 {
 649         struct async_submit_bio *async =
 650                 container_of(work, struct  async_submit_bio, work);
 651         struct inode *inode = async->inode;
 652         struct btrfs_bio *bbio = btrfs_bio(async->bio);
 653
 654         /* If an error occurred we just want to clean up the bio and move on */
 655         if (async->status) {
 656                 btrfs_bio_end_io(bbio, async->status);
 657                 return;
 658         }
 659
 660         /*
 661          * All of the bios that pass through here are from async helpers.
 662          * Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context.
 663          * This changes nothing when cgroups aren't in use.
 664          */
 665         async->bio->bi_opf |= REQ_CGROUP_PUNT;
 666         btrfs_submit_bio(btrfs_sb(inode->i_sb), async->bio, async->mirror_num);
 667 }
 668
 669 static void run_one_async_free(struct btrfs_work *work)
 670 {
 671         struct async_submit_bio *async;
 672
 673         async = container_of(work, struct  async_submit_bio, work);
 674         kfree(async);
 675 }
 676
 677 /*
 678  * Submit bio to an async queue.
 679  *
 680  * Retrun:
 681  * - true if the work has been succesfuly submitted
 682  * - false in case of error
 683  */
 684 bool btrfs_wq_submit_bio(struct inode *inode, struct bio *bio, int mirror_num,
 685                          u64 dio_file_offset,
 686                          extent_submit_bio_start_t *submit_bio_start)
 687 {
 688         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
 689         struct async_submit_bio *async;
 690
 691         async = kmalloc(sizeof(*async), GFP_NOFS);
 692         if (!async)
 693                 return false;
 694
 695         async->inode = inode;
 696         async->bio = bio;
 697         async->mirror_num = mirror_num;
 698         async->submit_bio_start = submit_bio_start;
 699
 700         btrfs_init_work(&async->work, run_one_async_start, run_one_async_done,
 701                         run_one_async_free);
 702
 703         async->dio_file_offset = dio_file_offset;
 704
 705         async->status = 0;
 706
 707         if (op_is_sync(bio->bi_opf))
 708                 btrfs_queue_work(fs_info->hipri_workers, &async->work);
 709         else
 710                 btrfs_queue_work(fs_info->workers, &async->work);
 711         return true;
 712 }
 713
 714 static blk_status_t btree_csum_one_bio(struct bio *bio)
 715 {
 716         struct bio_vec *bvec;
 717         struct btrfs_root *root;
 718         int ret = 0;
 719         struct bvec_iter_all iter_all;
 720
 721         ASSERT(!bio_flagged(bio, BIO_CLONED));
 722         bio_for_each_segment_all(bvec, bio, iter_all) {
 723                 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
 724                 ret = csum_dirty_buffer(root->fs_info, bvec);
 725                 if (ret)
 726                         break;
 727         }
 728
 729         return errno_to_blk_status(ret);
 730 }
 731
 732 static blk_status_t btree_submit_bio_start(struct inode *inode, struct bio *bio,
 733                                            u64 dio_file_offset)
 734 {
 735         /*
 736          * when we're called for a write, we're already in the async
 737          * submission context.  Just jump into btrfs_submit_bio.
 738          */
 739         return btree_csum_one_bio(bio);
 740 }
 741
 742 static bool should_async_write(struct btrfs_fs_info *fs_info,
 743                              struct btrfs_inode *bi)
 744 {
 745         if (btrfs_is_zoned(fs_info))
 746                 return false;
 747         if (atomic_read(&bi->sync_writers))
 748                 return false;
 749         if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags))
 750                 return false;
 751         return true;
 752 }
 753
 754 void btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int mirror_num)
 755 {
 756         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 757         struct btrfs_bio *bbio = btrfs_bio(bio);
 758         blk_status_t ret;
 759
 760         bio->bi_opf |= REQ_META;
 761
 762         if (btrfs_op(bio) != BTRFS_MAP_WRITE) {
 763                 btrfs_submit_bio(fs_info, bio, mirror_num);
 764                 return;
 765         }
 766
 767         /*
 768          * Kthread helpers are used to submit writes so that checksumming can
 769          * happen in parallel across all CPUs.
 770          */
 771         if (should_async_write(fs_info, BTRFS_I(inode)) &&
 772             btrfs_wq_submit_bio(inode, bio, mirror_num, 0, btree_submit_bio_start))
 773                 return;
 774
 775         ret = btree_csum_one_bio(bio);
 776         if (ret) {
 777                 btrfs_bio_end_io(bbio, ret);
 778                 return;
 779         }
 780
 781         btrfs_submit_bio(fs_info, bio, mirror_num);
 782 }
 783
 784 #ifdef CONFIG_MIGRATION
 785 static int btree_migrate_folio(struct address_space *mapping,
 786                 struct folio *dst, struct folio *src, enum migrate_mode mode)
 787 {
 788         /*
 789          * we can't safely write a btree page from here,
 790          * we haven't done the locking hook
 791          */
 792         if (folio_test_dirty(src))
 793                 return -EAGAIN;
 794         /*
 795          * Buffers may be managed in a filesystem specific way.
 796          * We must have no buffers or drop them.
 797          */
 798         if (folio_get_private(src) &&
 799             !filemap_release_folio(src, GFP_KERNEL))
 800                 return -EAGAIN;
 801         return migrate_folio(mapping, dst, src, mode);
 802 }
 803 #else
 804 #define btree_migrate_folio NULL
 805 #endif
 806
 807 static int btree_writepages(struct address_space *mapping,
 808                             struct writeback_control *wbc)
 809 {
 810         struct btrfs_fs_info *fs_info;
 811         int ret;
 812
 813         if (wbc->sync_mode == WB_SYNC_NONE) {
 814
 815                 if (wbc->for_kupdate)
 816                         return 0;
 817
 818                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
 819                 /* this is a bit racy, but that's ok */
 820                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
 821                                              BTRFS_DIRTY_METADATA_THRESH,
 822                                              fs_info->dirty_metadata_batch);
 823                 if (ret < 0)
 824                         return 0;
 825         }
 826         return btree_write_cache_pages(mapping, wbc);
 827 }
 828
 829 static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags)
 830 {
 831         if (folio_test_writeback(folio) || folio_test_dirty(folio))
 832                 return false;
 833
 834         return try_release_extent_buffer(&folio->page);
 835 }
 836
 837 static void btree_invalidate_folio(struct folio *folio, size_t offset,
 838                                  size_t length)
 839 {
 840         struct extent_io_tree *tree;
 841         tree = &BTRFS_I(folio->mapping->host)->io_tree;
 842         extent_invalidate_folio(tree, folio, offset);
 843         btree_release_folio(folio, GFP_NOFS);
 844         if (folio_get_private(folio)) {
 845                 btrfs_warn(BTRFS_I(folio->mapping->host)->root->fs_info,
 846                            "folio private not zero on folio %llu",
 847                            (unsigned long long)folio_pos(folio));
 848                 folio_detach_private(folio);
 849         }
 850 }
 851
 852 #ifdef DEBUG
 853 static bool btree_dirty_folio(struct address_space *mapping,
 854                 struct folio *folio)
 855 {
 856         struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
 857         struct btrfs_subpage *subpage;
 858         struct extent_buffer *eb;
 859         int cur_bit = 0;
 860         u64 page_start = folio_pos(folio);
 861
 862         if (fs_info->sectorsize == PAGE_SIZE) {
 863                 eb = folio_get_private(folio);
 864                 BUG_ON(!eb);
 865                 BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 866                 BUG_ON(!atomic_read(&eb->refs));
 867                 btrfs_assert_tree_write_locked(eb);
 868                 return filemap_dirty_folio(mapping, folio);
 869         }
 870         subpage = folio_get_private(folio);
 871
 872         ASSERT(subpage->dirty_bitmap);
 873         while (cur_bit < BTRFS_SUBPAGE_BITMAP_SIZE) {
 874                 unsigned long flags;
 875                 u64 cur;
 876                 u16 tmp = (1 << cur_bit);
 877
 878                 spin_lock_irqsave(&subpage->lock, flags);
 879                 if (!(tmp & subpage->dirty_bitmap)) {
 880                         spin_unlock_irqrestore(&subpage->lock, flags);
 881                         cur_bit++;
 882                         continue;
 883                 }
 884                 spin_unlock_irqrestore(&subpage->lock, flags);
 885                 cur = page_start + cur_bit * fs_info->sectorsize;
 886
 887                 eb = find_extent_buffer(fs_info, cur);
 888                 ASSERT(eb);
 889                 ASSERT(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 890                 ASSERT(atomic_read(&eb->refs));
 891                 btrfs_assert_tree_write_locked(eb);
 892                 free_extent_buffer(eb);
 893
 894                 cur_bit += (fs_info->nodesize >> fs_info->sectorsize_bits);
 895         }
 896         return filemap_dirty_folio(mapping, folio);
 897 }
 898 #else
 899 #define btree_dirty_folio filemap_dirty_folio
 900 #endif
 901
 902 static const struct address_space_operations btree_aops = {
 903         .writepages     = btree_writepages,
 904         .release_folio  = btree_release_folio,
 905         .invalidate_folio = btree_invalidate_folio,
 906         .migrate_folio  = btree_migrate_folio,
 907         .dirty_folio    = btree_dirty_folio,
 908 };
 909
 910 struct extent_buffer *btrfs_find_create_tree_block(
 911                                                 struct btrfs_fs_info *fs_info,
 912                                                 u64 bytenr, u64 owner_root,
 913                                                 int level)
 914 {
 915         if (btrfs_is_testing(fs_info))
 916                 return alloc_test_extent_buffer(fs_info, bytenr);
 917         return alloc_extent_buffer(fs_info, bytenr, owner_root, level);
 918 }
 919
 920 /*
 921  * Read tree block at logical address @bytenr and do variant basic but critical
 922  * verification.
 923  *
 924  * @owner_root:         the objectid of the root owner for this block.
 925  * @parent_transid:     expected transid of this tree block, skip check if 0
 926  * @level:              expected level, mandatory check
 927  * @first_key:          expected key in slot 0, skip check if NULL
 928  */
 929 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
 930                                       u64 owner_root, u64 parent_transid,
 931                                       int level, struct btrfs_key *first_key)
 932 {
 933         struct extent_buffer *buf = NULL;
 934         int ret;
 935
 936         buf = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
 937         if (IS_ERR(buf))
 938                 return buf;
 939
 940         ret = btrfs_read_extent_buffer(buf, parent_transid, level, first_key);
 941         if (ret) {
 942                 free_extent_buffer_stale(buf);
 943                 return ERR_PTR(ret);
 944         }
 945         if (btrfs_check_eb_owner(buf, owner_root)) {
 946                 free_extent_buffer_stale(buf);
 947                 return ERR_PTR(-EUCLEAN);
 948         }
 949         return buf;
 950
 951 }
 952
 953 void btrfs_clean_tree_block(struct extent_buffer *buf)
 954 {
 955         struct btrfs_fs_info *fs_info = buf->fs_info;
 956         if (btrfs_header_generation(buf) ==
 957             fs_info->running_transaction->transid) {
 958                 btrfs_assert_tree_write_locked(buf);
 959
 960                 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) {
 961                         percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
 962                                                  -buf->len,
 963                                                  fs_info->dirty_metadata_batch);
 964                         clear_extent_buffer_dirty(buf);
 965                 }
 966         }
 967 }
 968
 969 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
 970                          u64 objectid)
 971 {
 972         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
 973
 974         memset(&root->root_key, 0, sizeof(root->root_key));
 975         memset(&root->root_item, 0, sizeof(root->root_item));
 976         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
 977         root->fs_info = fs_info;
 978         root->root_key.objectid = objectid;
 979         root->node = NULL;
 980         root->commit_root = NULL;
 981         root->state = 0;
 982         RB_CLEAR_NODE(&root->rb_node);
 983
 984         root->last_trans = 0;
 985         root->free_objectid = 0;
 986         root->nr_delalloc_inodes = 0;
 987         root->nr_ordered_extents = 0;
 988         root->inode_tree = RB_ROOT;
 989         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
 990
 991         btrfs_init_root_block_rsv(root);
 992
 993         INIT_LIST_HEAD(&root->dirty_list);
 994         INIT_LIST_HEAD(&root->root_list);
 995         INIT_LIST_HEAD(&root->delalloc_inodes);
 996         INIT_LIST_HEAD(&root->delalloc_root);
 997         INIT_LIST_HEAD(&root->ordered_extents);
 998         INIT_LIST_HEAD(&root->ordered_root);
 999         INIT_LIST_HEAD(&root->reloc_dirty_list);
1000         INIT_LIST_HEAD(&root->logged_list[0]);
1001         INIT_LIST_HEAD(&root->logged_list[1]);
1002         spin_lock_init(&root->inode_lock);
1003         spin_lock_init(&root->delalloc_lock);
1004         spin_lock_init(&root->ordered_extent_lock);
1005         spin_lock_init(&root->accounting_lock);
1006         spin_lock_init(&root->log_extents_lock[0]);
1007         spin_lock_init(&root->log_extents_lock[1]);
1008         spin_lock_init(&root->qgroup_meta_rsv_lock);
1009         mutex_init(&root->objectid_mutex);
1010         mutex_init(&root->log_mutex);
1011         mutex_init(&root->ordered_extent_mutex);
1012         mutex_init(&root->delalloc_mutex);
1013         init_waitqueue_head(&root->qgroup_flush_wait);
1014         init_waitqueue_head(&root->log_writer_wait);
1015         init_waitqueue_head(&root->log_commit_wait[0]);
1016         init_waitqueue_head(&root->log_commit_wait[1]);
1017         INIT_LIST_HEAD(&root->log_ctxs[0]);
1018         INIT_LIST_HEAD(&root->log_ctxs[1]);
1019         atomic_set(&root->log_commit[0], 0);
1020         atomic_set(&root->log_commit[1], 0);
1021         atomic_set(&root->log_writers, 0);
1022         atomic_set(&root->log_batch, 0);
1023         refcount_set(&root->refs, 1);
1024         atomic_set(&root->snapshot_force_cow, 0);
1025         atomic_set(&root->nr_swapfiles, 0);
1026         root->log_transid = 0;
1027         root->log_transid_committed = -1;
1028         root->last_log_commit = 0;
1029         root->anon_dev = 0;
1030         if (!dummy) {
1031                 extent_io_tree_init(fs_info, &root->dirty_log_pages,
1032                                     IO_TREE_ROOT_DIRTY_LOG_PAGES, NULL);
1033                 extent_io_tree_init(fs_info, &root->log_csum_range,
1034                                     IO_TREE_LOG_CSUM_RANGE, NULL);
1035         }
1036
1037         spin_lock_init(&root->root_item_lock);
1038         btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks);
1039 #ifdef CONFIG_BTRFS_DEBUG
1040         INIT_LIST_HEAD(&root->leak_list);
1041         spin_lock(&fs_info->fs_roots_radix_lock);
1042         list_add_tail(&root->leak_list, &fs_info->allocated_roots);
1043         spin_unlock(&fs_info->fs_roots_radix_lock);
1044 #endif
1045 }
1046
1047 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
1048                                            u64 objectid, gfp_t flags)
1049 {
1050         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
1051         if (root)
1052                 __setup_root(root, fs_info, objectid);
1053         return root;
1054 }
1055
1056 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1057 /* Should only be used by the testing infrastructure */
1058 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
1059 {
1060         struct btrfs_root *root;
1061
1062         if (!fs_info)
1063                 return ERR_PTR(-EINVAL);
1064
1065         root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, GFP_KERNEL);
1066         if (!root)
1067                 return ERR_PTR(-ENOMEM);
1068
1069         /* We don't use the stripesize in selftest, set it as sectorsize */
1070         root->alloc_bytenr = 0;
1071
1072         return root;
1073 }
1074 #endif
1075
1076 static int global_root_cmp(struct rb_node *a_node, const struct rb_node *b_node)
1077 {
1078         const struct btrfs_root *a = rb_entry(a_node, struct btrfs_root, rb_node);
1079         const struct btrfs_root *b = rb_entry(b_node, struct btrfs_root, rb_node);
1080
1081         return btrfs_comp_cpu_keys(&a->root_key, &b->root_key);
1082 }
1083
1084 static int global_root_key_cmp(const void *k, const struct rb_node *node)
1085 {
1086         const struct btrfs_key *key = k;
1087         const struct btrfs_root *root = rb_entry(node, struct btrfs_root, rb_node);
1088
1089         return btrfs_comp_cpu_keys(key, &root->root_key);
1090 }
1091
1092 int btrfs_global_root_insert(struct btrfs_root *root)
1093 {
1094         struct btrfs_fs_info *fs_info = root->fs_info;
1095         struct rb_node *tmp;
1096
1097         write_lock(&fs_info->global_root_lock);
1098         tmp = rb_find_add(&root->rb_node, &fs_info->global_root_tree, global_root_cmp);
1099         write_unlock(&fs_info->global_root_lock);
1100         ASSERT(!tmp);
1101
1102         return tmp ? -EEXIST : 0;
1103 }
1104
1105 void btrfs_global_root_delete(struct btrfs_root *root)
1106 {
1107         struct btrfs_fs_info *fs_info = root->fs_info;
1108
1109         write_lock(&fs_info->global_root_lock);
1110         rb_erase(&root->rb_node, &fs_info->global_root_tree);
1111         write_unlock(&fs_info->global_root_lock);
1112 }
1113
1114 struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info,
1115                                      struct btrfs_key *key)
1116 {
1117         struct rb_node *node;
1118         struct btrfs_root *root = NULL;
1119
1120         read_lock(&fs_info->global_root_lock);
1121         node = rb_find(key, &fs_info->global_root_tree, global_root_key_cmp);
1122         if (node)
1123                 root = container_of(node, struct btrfs_root, rb_node);
1124         read_unlock(&fs_info->global_root_lock);
1125
1126         return root;
1127 }
1128
1129 static u64 btrfs_global_root_id(struct btrfs_fs_info *fs_info, u64 bytenr)
1130 {
1131         struct btrfs_block_group *block_group;
1132         u64 ret;
1133
1134         if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))
1135                 return 0;
1136
1137         if (bytenr)
1138                 block_group = btrfs_lookup_block_group(fs_info, bytenr);
1139         else
1140                 block_group = btrfs_lookup_first_block_group(fs_info, bytenr);
1141         ASSERT(block_group);
1142         if (!block_group)
1143                 return 0;
1144         ret = block_group->global_root_id;
1145         btrfs_put_block_group(block_group);
1146
1147         return ret;
1148 }
1149
1150 struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1151 {
1152         struct btrfs_key key = {
1153                 .objectid = BTRFS_CSUM_TREE_OBJECTID,
1154                 .type = BTRFS_ROOT_ITEM_KEY,
1155                 .offset = btrfs_global_root_id(fs_info, bytenr),
1156         };
1157
1158         return btrfs_global_root(fs_info, &key);
1159 }
1160
1161 struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1162 {
1163         struct btrfs_key key = {
1164                 .objectid = BTRFS_EXTENT_TREE_OBJECTID,
1165                 .type = BTRFS_ROOT_ITEM_KEY,
1166                 .offset = btrfs_global_root_id(fs_info, bytenr),
1167         };
1168
1169         return btrfs_global_root(fs_info, &key);
1170 }
1171
1172 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1173                                      u64 objectid)
1174 {
1175         struct btrfs_fs_info *fs_info = trans->fs_info;
1176         struct extent_buffer *leaf;
1177         struct btrfs_root *tree_root = fs_info->tree_root;
1178         struct btrfs_root *root;
1179         struct btrfs_key key;
1180         unsigned int nofs_flag;
1181         int ret = 0;
1182
1183         /*
1184          * We're holding a transaction handle, so use a NOFS memory allocation
1185          * context to avoid deadlock if reclaim happens.
1186          */
1187         nofs_flag = memalloc_nofs_save();
1188         root = btrfs_alloc_root(fs_info, objectid, GFP_KERNEL);
1189         memalloc_nofs_restore(nofs_flag);
1190         if (!root)
1191                 return ERR_PTR(-ENOMEM);
1192
1193         root->root_key.objectid = objectid;
1194         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1195         root->root_key.offset = 0;
1196
1197         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0,
1198                                       BTRFS_NESTING_NORMAL);
1199         if (IS_ERR(leaf)) {
1200                 ret = PTR_ERR(leaf);
1201                 leaf = NULL;
1202                 goto fail_unlock;
1203         }
1204
1205         root->node = leaf;
1206         btrfs_mark_buffer_dirty(leaf);
1207
1208         root->commit_root = btrfs_root_node(root);
1209         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1210
1211         btrfs_set_root_flags(&root->root_item, 0);
1212         btrfs_set_root_limit(&root->root_item, 0);
1213         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1214         btrfs_set_root_generation(&root->root_item, trans->transid);
1215         btrfs_set_root_level(&root->root_item, 0);
1216         btrfs_set_root_refs(&root->root_item, 1);
1217         btrfs_set_root_used(&root->root_item, leaf->len);
1218         btrfs_set_root_last_snapshot(&root->root_item, 0);
1219         btrfs_set_root_dirid(&root->root_item, 0);
1220         if (is_fstree(objectid))
1221                 generate_random_guid(root->root_item.uuid);
1222         else
1223                 export_guid(root->root_item.uuid, &guid_null);
1224         btrfs_set_root_drop_level(&root->root_item, 0);
1225
1226         btrfs_tree_unlock(leaf);
1227
1228         key.objectid = objectid;
1229         key.type = BTRFS_ROOT_ITEM_KEY;
1230         key.offset = 0;
1231         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1232         if (ret)
1233                 goto fail;
1234
1235         return root;
1236
1237 fail_unlock:
1238         if (leaf)
1239                 btrfs_tree_unlock(leaf);
1240 fail:
1241         btrfs_put_root(root);
1242
1243         return ERR_PTR(ret);
1244 }
1245
1246 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1247                                          struct btrfs_fs_info *fs_info)
1248 {
1249         struct btrfs_root *root;
1250
1251         root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS);
1252         if (!root)
1253                 return ERR_PTR(-ENOMEM);
1254
1255         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1256         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1257         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1258
1259         return root;
1260 }
1261
1262 int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans,
1263                               struct btrfs_root *root)
1264 {
1265         struct extent_buffer *leaf;
1266
1267         /*
1268          * DON'T set SHAREABLE bit for log trees.
1269          *
1270          * Log trees are not exposed to user space thus can't be snapshotted,
1271          * and they go away before a real commit is actually done.
1272          *
1273          * They do store pointers to file data extents, and those reference
1274          * counts still get updated (along with back refs to the log tree).
1275          */
1276
1277         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1278                         NULL, 0, 0, 0, BTRFS_NESTING_NORMAL);
1279         if (IS_ERR(leaf))
1280                 return PTR_ERR(leaf);
1281
1282         root->node = leaf;
1283
1284         btrfs_mark_buffer_dirty(root->node);
1285         btrfs_tree_unlock(root->node);
1286
1287         return 0;
1288 }
1289
1290 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1291                              struct btrfs_fs_info *fs_info)
1292 {
1293         struct btrfs_root *log_root;
1294
1295         log_root = alloc_log_tree(trans, fs_info);
1296         if (IS_ERR(log_root))
1297                 return PTR_ERR(log_root);
1298
1299         if (!btrfs_is_zoned(fs_info)) {
1300                 int ret = btrfs_alloc_log_tree_node(trans, log_root);
1301
1302                 if (ret) {
1303                         btrfs_put_root(log_root);
1304                         return ret;
1305                 }
1306         }
1307
1308         WARN_ON(fs_info->log_root_tree);
1309         fs_info->log_root_tree = log_root;
1310         return 0;
1311 }
1312
1313 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1314                        struct btrfs_root *root)
1315 {
1316         struct btrfs_fs_info *fs_info = root->fs_info;
1317         struct btrfs_root *log_root;
1318         struct btrfs_inode_item *inode_item;
1319         int ret;
1320
1321         log_root = alloc_log_tree(trans, fs_info);
1322         if (IS_ERR(log_root))
1323                 return PTR_ERR(log_root);
1324
1325         ret = btrfs_alloc_log_tree_node(trans, log_root);
1326         if (ret) {
1327                 btrfs_put_root(log_root);
1328                 return ret;
1329         }
1330
1331         log_root->last_trans = trans->transid;
1332         log_root->root_key.offset = root->root_key.objectid;
1333
1334         inode_item = &log_root->root_item.inode;
1335         btrfs_set_stack_inode_generation(inode_item, 1);
1336         btrfs_set_stack_inode_size(inode_item, 3);
1337         btrfs_set_stack_inode_nlink(inode_item, 1);
1338         btrfs_set_stack_inode_nbytes(inode_item,
1339                                      fs_info->nodesize);
1340         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1341
1342         btrfs_set_root_node(&log_root->root_item, log_root->node);
1343
1344         WARN_ON(root->log_root);
1345         root->log_root = log_root;
1346         root->log_transid = 0;
1347         root->log_transid_committed = -1;
1348         root->last_log_commit = 0;
1349         return 0;
1350 }
1351
1352 static struct btrfs_root *read_tree_root_path(struct btrfs_root *tree_root,
1353                                               struct btrfs_path *path,
1354                                               struct btrfs_key *key)
1355 {
1356         struct btrfs_root *root;
1357         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1358         u64 generation;
1359         int ret;
1360         int level;
1361
1362         root = btrfs_alloc_root(fs_info, key->objectid, GFP_NOFS);
1363         if (!root)
1364                 return ERR_PTR(-ENOMEM);
1365
1366         ret = btrfs_find_root(tree_root, key, path,
1367                               &root->root_item, &root->root_key);
1368         if (ret) {
1369                 if (ret > 0)
1370                         ret = -ENOENT;
1371                 goto fail;
1372         }
1373
1374         generation = btrfs_root_generation(&root->root_item);
1375         level = btrfs_root_level(&root->root_item);
1376         root->node = read_tree_block(fs_info,
1377                                      btrfs_root_bytenr(&root->root_item),
1378                                      key->objectid, generation, level, NULL);
1379         if (IS_ERR(root->node)) {
1380                 ret = PTR_ERR(root->node);
1381                 root->node = NULL;
1382                 goto fail;
1383         }
1384         if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1385                 ret = -EIO;
1386                 goto fail;
1387         }
1388
1389         /*
1390          * For real fs, and not log/reloc trees, root owner must
1391          * match its root node owner
1392          */
1393         if (!test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state) &&
1394             root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1395             root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
1396             root->root_key.objectid != btrfs_header_owner(root->node)) {
1397                 btrfs_crit(fs_info,
1398 "root=%llu block=%llu, tree root owner mismatch, have %llu expect %llu",
1399                            root->root_key.objectid, root->node->start,
1400                            btrfs_header_owner(root->node),
1401                            root->root_key.objectid);
1402                 ret = -EUCLEAN;
1403                 goto fail;
1404         }
1405         root->commit_root = btrfs_root_node(root);
1406         return root;
1407 fail:
1408         btrfs_put_root(root);
1409         return ERR_PTR(ret);
1410 }
1411
1412 struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1413                                         struct btrfs_key *key)
1414 {
1415         struct btrfs_root *root;
1416         struct btrfs_path *path;
1417
1418         path = btrfs_alloc_path();
1419         if (!path)
1420                 return ERR_PTR(-ENOMEM);
1421         root = read_tree_root_path(tree_root, path, key);
1422         btrfs_free_path(path);
1423
1424         return root;
1425 }
1426
1427 /*
1428  * Initialize subvolume root in-memory structure
1429  *
1430  * @anon_dev:   anonymous device to attach to the root, if zero, allocate new
1431  */
1432 static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev)
1433 {
1434         int ret;
1435         unsigned int nofs_flag;
1436
1437         /*
1438          * We might be called under a transaction (e.g. indirect backref
1439          * resolution) which could deadlock if it triggers memory reclaim
1440          */
1441         nofs_flag = memalloc_nofs_save();
1442         ret = btrfs_drew_lock_init(&root->snapshot_lock);
1443         memalloc_nofs_restore(nofs_flag);
1444         if (ret)
1445                 goto fail;
1446
1447         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1448             !btrfs_is_data_reloc_root(root)) {
1449                 set_bit(BTRFS_ROOT_SHAREABLE, &root->state);
1450                 btrfs_check_and_init_root_item(&root->root_item);
1451         }
1452
1453         /*
1454          * Don't assign anonymous block device to roots that are not exposed to
1455          * userspace, the id pool is limited to 1M
1456          */
1457         if (is_fstree(root->root_key.objectid) &&
1458             btrfs_root_refs(&root->root_item) > 0) {
1459                 if (!anon_dev) {
1460                         ret = get_anon_bdev(&root->anon_dev);
1461                         if (ret)
1462                                 goto fail;
1463                 } else {
1464                         root->anon_dev = anon_dev;
1465                 }
1466         }
1467
1468         mutex_lock(&root->objectid_mutex);
1469         ret = btrfs_init_root_free_objectid(root);
1470         if (ret) {
1471                 mutex_unlock(&root->objectid_mutex);
1472                 goto fail;
1473         }
1474
1475         ASSERT(root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
1476
1477         mutex_unlock(&root->objectid_mutex);
1478
1479         return 0;
1480 fail:
1481         /* The caller is responsible to call btrfs_free_fs_root */
1482         return ret;
1483 }
1484
1485 static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1486                                                u64 root_id)
1487 {
1488         struct btrfs_root *root;
1489
1490         spin_lock(&fs_info->fs_roots_radix_lock);
1491         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1492                                  (unsigned long)root_id);
1493         if (root)
1494                 root = btrfs_grab_root(root);
1495         spin_unlock(&fs_info->fs_roots_radix_lock);
1496         return root;
1497 }
1498
1499 static struct btrfs_root *btrfs_get_global_root(struct btrfs_fs_info *fs_info,
1500                                                 u64 objectid)
1501 {
1502         struct btrfs_key key = {
1503                 .objectid = objectid,
1504                 .type = BTRFS_ROOT_ITEM_KEY,
1505                 .offset = 0,
1506         };
1507
1508         if (objectid == BTRFS_ROOT_TREE_OBJECTID)
1509                 return btrfs_grab_root(fs_info->tree_root);
1510         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
1511                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1512         if (objectid == BTRFS_CHUNK_TREE_OBJECTID)
1513                 return btrfs_grab_root(fs_info->chunk_root);
1514         if (objectid == BTRFS_DEV_TREE_OBJECTID)
1515                 return btrfs_grab_root(fs_info->dev_root);
1516         if (objectid == BTRFS_CSUM_TREE_OBJECTID)
1517                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1518         if (objectid == BTRFS_QUOTA_TREE_OBJECTID)
1519                 return btrfs_grab_root(fs_info->quota_root) ?
1520                         fs_info->quota_root : ERR_PTR(-ENOENT);
1521         if (objectid == BTRFS_UUID_TREE_OBJECTID)
1522                 return btrfs_grab_root(fs_info->uuid_root) ?
1523                         fs_info->uuid_root : ERR_PTR(-ENOENT);
1524         if (objectid == BTRFS_BLOCK_GROUP_TREE_OBJECTID)
1525                 return btrfs_grab_root(fs_info->block_group_root) ?
1526                         fs_info->block_group_root : ERR_PTR(-ENOENT);
1527         if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) {
1528                 struct btrfs_root *root = btrfs_global_root(fs_info, &key);
1529
1530                 return btrfs_grab_root(root) ? root : ERR_PTR(-ENOENT);
1531         }
1532         return NULL;
1533 }
1534
1535 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1536                          struct btrfs_root *root)
1537 {
1538         int ret;
1539
1540         ret = radix_tree_preload(GFP_NOFS);
1541         if (ret)
1542                 return ret;
1543
1544         spin_lock(&fs_info->fs_roots_radix_lock);
1545         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1546                                 (unsigned long)root->root_key.objectid,
1547                                 root);
1548         if (ret == 0) {
1549                 btrfs_grab_root(root);
1550                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1551         }
1552         spin_unlock(&fs_info->fs_roots_radix_lock);
1553         radix_tree_preload_end();
1554
1555         return ret;
1556 }
1557
1558 void btrfs_check_leaked_roots(struct btrfs_fs_info *fs_info)
1559 {
1560 #ifdef CONFIG_BTRFS_DEBUG
1561         struct btrfs_root *root;
1562
1563         while (!list_empty(&fs_info->allocated_roots)) {
1564                 char buf[BTRFS_ROOT_NAME_BUF_LEN];
1565
1566                 root = list_first_entry(&fs_info->allocated_roots,
1567                                         struct btrfs_root, leak_list);
1568                 btrfs_err(fs_info, "leaked root %s refcount %d",
1569                           btrfs_root_name(&root->root_key, buf),
1570                           refcount_read(&root->refs));
1571                 while (refcount_read(&root->refs) > 1)
1572                         btrfs_put_root(root);
1573                 btrfs_put_root(root);
1574         }
1575 #endif
1576 }
1577
1578 static void free_global_roots(struct btrfs_fs_info *fs_info)
1579 {
1580         struct btrfs_root *root;
1581         struct rb_node *node;
1582
1583         while ((node = rb_first_postorder(&fs_info->global_root_tree)) != NULL) {
1584                 root = rb_entry(node, struct btrfs_root, rb_node);
1585                 rb_erase(&root->rb_node, &fs_info->global_root_tree);
1586                 btrfs_put_root(root);
1587         }
1588 }
1589
1590 void btrfs_free_fs_info(struct btrfs_fs_info *fs_info)
1591 {
1592         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
1593         percpu_counter_destroy(&fs_info->delalloc_bytes);
1594         percpu_counter_destroy(&fs_info->ordered_bytes);
1595         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
1596         btrfs_free_csum_hash(fs_info);
1597         btrfs_free_stripe_hash_table(fs_info);
1598         btrfs_free_ref_cache(fs_info);
1599         kfree(fs_info->balance_ctl);
1600         kfree(fs_info->delayed_root);
1601         free_global_roots(fs_info);
1602         btrfs_put_root(fs_info->tree_root);
1603         btrfs_put_root(fs_info->chunk_root);
1604         btrfs_put_root(fs_info->dev_root);
1605         btrfs_put_root(fs_info->quota_root);
1606         btrfs_put_root(fs_info->uuid_root);
1607         btrfs_put_root(fs_info->fs_root);
1608         btrfs_put_root(fs_info->data_reloc_root);
1609         btrfs_put_root(fs_info->block_group_root);
1610         btrfs_check_leaked_roots(fs_info);
1611         btrfs_extent_buffer_leak_debug_check(fs_info);
1612         kfree(fs_info->super_copy);
1613         kfree(fs_info->super_for_commit);
1614         kfree(fs_info->subpage_info);
1615         kvfree(fs_info);
1616 }
1617
1618
1619 /*
1620  * Get an in-memory reference of a root structure.
1621  *
1622  * For essential trees like root/extent tree, we grab it from fs_info directly.
1623  * For subvolume trees, we check the cached filesystem roots first. If not
1624  * found, then read it from disk and add it to cached fs roots.
1625  *
1626  * Caller should release the root by calling btrfs_put_root() after the usage.
1627  *
1628  * NOTE: Reloc and log trees can't be read by this function as they share the
1629  *       same root objectid.
1630  *
1631  * @objectid:   root id
1632  * @anon_dev:   preallocated anonymous block device number for new roots,
1633  *              pass 0 for new allocation.
1634  * @check_ref:  whether to check root item references, If true, return -ENOENT
1635  *              for orphan roots
1636  */
1637 static struct btrfs_root *btrfs_get_root_ref(struct btrfs_fs_info *fs_info,
1638                                              u64 objectid, dev_t anon_dev,
1639                                              bool check_ref)
1640 {
1641         struct btrfs_root *root;
1642         struct btrfs_path *path;
1643         struct btrfs_key key;
1644         int ret;
1645
1646         root = btrfs_get_global_root(fs_info, objectid);
1647         if (root)
1648                 return root;
1649 again:
1650         root = btrfs_lookup_fs_root(fs_info, objectid);
1651         if (root) {
1652                 /* Shouldn't get preallocated anon_dev for cached roots */
1653                 ASSERT(!anon_dev);
1654                 if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1655                         btrfs_put_root(root);
1656                         return ERR_PTR(-ENOENT);
1657                 }
1658                 return root;
1659         }
1660
1661         key.objectid = objectid;
1662         key.type = BTRFS_ROOT_ITEM_KEY;
1663         key.offset = (u64)-1;
1664         root = btrfs_read_tree_root(fs_info->tree_root, &key);
1665         if (IS_ERR(root))
1666                 return root;
1667
1668         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1669                 ret = -ENOENT;
1670                 goto fail;
1671         }
1672
1673         ret = btrfs_init_fs_root(root, anon_dev);
1674         if (ret)
1675                 goto fail;
1676
1677         path = btrfs_alloc_path();
1678         if (!path) {
1679                 ret = -ENOMEM;
1680                 goto fail;
1681         }
1682         key.objectid = BTRFS_ORPHAN_OBJECTID;
1683         key.type = BTRFS_ORPHAN_ITEM_KEY;
1684         key.offset = objectid;
1685
1686         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1687         btrfs_free_path(path);
1688         if (ret < 0)
1689                 goto fail;
1690         if (ret == 0)
1691                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1692
1693         ret = btrfs_insert_fs_root(fs_info, root);
1694         if (ret) {
1695                 if (ret == -EEXIST) {
1696                         btrfs_put_root(root);
1697                         goto again;
1698                 }
1699                 goto fail;
1700         }
1701         return root;
1702 fail:
1703         /*
1704          * If our caller provided us an anonymous device, then it's his
1705          * responsibility to free it in case we fail. So we have to set our
1706          * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root()
1707          * and once again by our caller.
1708          */
1709         if (anon_dev)
1710                 root->anon_dev = 0;
1711         btrfs_put_root(root);
1712         return ERR_PTR(ret);
1713 }
1714
1715 /*
1716  * Get in-memory reference of a root structure
1717  *
1718  * @objectid:   tree objectid
1719  * @check_ref:  if set, verify that the tree exists and the item has at least
1720  *              one reference
1721  */
1722 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1723                                      u64 objectid, bool check_ref)
1724 {
1725         return btrfs_get_root_ref(fs_info, objectid, 0, check_ref);
1726 }
1727
1728 /*
1729  * Get in-memory reference of a root structure, created as new, optionally pass
1730  * the anonymous block device id
1731  *
1732  * @objectid:   tree objectid
1733  * @anon_dev:   if zero, allocate a new anonymous block device or use the
1734  *              parameter value
1735  */
1736 struct btrfs_root *btrfs_get_new_fs_root(struct btrfs_fs_info *fs_info,
1737                                          u64 objectid, dev_t anon_dev)
1738 {
1739         return btrfs_get_root_ref(fs_info, objectid, anon_dev, true);
1740 }
1741
1742 /*
1743  * btrfs_get_fs_root_commit_root - return a root for the given objectid
1744  * @fs_info:    the fs_info
1745  * @objectid:   the objectid we need to lookup
1746  *
1747  * This is exclusively used for backref walking, and exists specifically because
1748  * of how qgroups does lookups.  Qgroups will do a backref lookup at delayed ref
1749  * creation time, which means we may have to read the tree_root in order to look
1750  * up a fs root that is not in memory.  If the root is not in memory we will
1751  * read the tree root commit root and look up the fs root from there.  This is a
1752  * temporary root, it will not be inserted into the radix tree as it doesn't
1753  * have the most uptodate information, it'll simply be discarded once the
1754  * backref code is finished using the root.
1755  */
1756 struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info,
1757                                                  struct btrfs_path *path,
1758                                                  u64 objectid)
1759 {
1760         struct btrfs_root *root;
1761         struct btrfs_key key;
1762
1763         ASSERT(path->search_commit_root && path->skip_locking);
1764
1765         /*
1766          * This can return -ENOENT if we ask for a root that doesn't exist, but
1767          * since this is called via the backref walking code we won't be looking
1768          * up a root that doesn't exist, unless there's corruption.  So if root
1769          * != NULL just return it.
1770          */
1771         root = btrfs_get_global_root(fs_info, objectid);
1772         if (root)
1773                 return root;
1774
1775         root = btrfs_lookup_fs_root(fs_info, objectid);
1776         if (root)
1777                 return root;
1778
1779         key.objectid = objectid;
1780         key.type = BTRFS_ROOT_ITEM_KEY;
1781         key.offset = (u64)-1;
1782         root = read_tree_root_path(fs_info->tree_root, path, &key);
1783         btrfs_release_path(path);
1784
1785         return root;
1786 }
1787
1788 static int cleaner_kthread(void *arg)
1789 {
1790         struct btrfs_fs_info *fs_info = arg;
1791         int again;
1792
1793         while (1) {
1794                 again = 0;
1795
1796                 set_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1797
1798                 /* Make the cleaner go to sleep early. */
1799                 if (btrfs_need_cleaner_sleep(fs_info))
1800                         goto sleep;
1801
1802                 /*
1803                  * Do not do anything if we might cause open_ctree() to block
1804                  * before we have finished mounting the filesystem.
1805                  */
1806                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1807                         goto sleep;
1808
1809                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1810                         goto sleep;
1811
1812                 /*
1813                  * Avoid the problem that we change the status of the fs
1814                  * during the above check and trylock.
1815                  */
1816                 if (btrfs_need_cleaner_sleep(fs_info)) {
1817                         mutex_unlock(&fs_info->cleaner_mutex);
1818                         goto sleep;
1819                 }
1820
1821                 btrfs_run_delayed_iputs(fs_info);
1822
1823                 again = btrfs_clean_one_deleted_snapshot(fs_info);
1824                 mutex_unlock(&fs_info->cleaner_mutex);
1825
1826                 /*
1827                  * The defragger has dealt with the R/O remount and umount,
1828                  * needn't do anything special here.
1829                  */
1830                 btrfs_run_defrag_inodes(fs_info);
1831
1832                 /*
1833                  * Acquires fs_info->reclaim_bgs_lock to avoid racing
1834                  * with relocation (btrfs_relocate_chunk) and relocation
1835                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1836                  * after acquiring fs_info->reclaim_bgs_lock. So we
1837                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1838                  * unused block groups.
1839                  */
1840                 btrfs_delete_unused_bgs(fs_info);
1841
1842                 /*
1843                  * Reclaim block groups in the reclaim_bgs list after we deleted
1844                  * all unused block_groups. This possibly gives us some more free
1845                  * space.
1846                  */
1847                 btrfs_reclaim_bgs(fs_info);
1848 sleep:
1849                 clear_and_wake_up_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1850                 if (kthread_should_park())
1851                         kthread_parkme();
1852                 if (kthread_should_stop())
1853                         return 0;
1854                 if (!again) {
1855                         set_current_state(TASK_INTERRUPTIBLE);
1856                         schedule();
1857                         __set_current_state(TASK_RUNNING);
1858                 }
1859         }
1860 }
1861
1862 static int transaction_kthread(void *arg)
1863 {
1864         struct btrfs_root *root = arg;
1865         struct btrfs_fs_info *fs_info = root->fs_info;
1866         struct btrfs_trans_handle *trans;
1867         struct btrfs_transaction *cur;
1868         u64 transid;
1869         time64_t delta;
1870         unsigned long delay;
1871         bool cannot_commit;
1872
1873         do {
1874                 cannot_commit = false;
1875                 delay = msecs_to_jiffies(fs_info->commit_interval * 1000);
1876                 mutex_lock(&fs_info->transaction_kthread_mutex);
1877
1878                 spin_lock(&fs_info->trans_lock);
1879                 cur = fs_info->running_transaction;
1880                 if (!cur) {
1881                         spin_unlock(&fs_info->trans_lock);
1882                         goto sleep;
1883                 }
1884
1885                 delta = ktime_get_seconds() - cur->start_time;
1886                 if (!test_and_clear_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags) &&
1887                     cur->state < TRANS_STATE_COMMIT_START &&
1888                     delta < fs_info->commit_interval) {
1889                         spin_unlock(&fs_info->trans_lock);
1890                         delay -= msecs_to_jiffies((delta - 1) * 1000);
1891                         delay = min(delay,
1892                                     msecs_to_jiffies(fs_info->commit_interval * 1000));
1893                         goto sleep;
1894                 }
1895                 transid = cur->transid;
1896                 spin_unlock(&fs_info->trans_lock);
1897
1898                 /* If the file system is aborted, this will always fail. */
1899                 trans = btrfs_attach_transaction(root);
1900                 if (IS_ERR(trans)) {
1901                         if (PTR_ERR(trans) != -ENOENT)
1902                                 cannot_commit = true;
1903                         goto sleep;
1904                 }
1905                 if (transid == trans->transid) {
1906                         btrfs_commit_transaction(trans);
1907                 } else {
1908                         btrfs_end_transaction(trans);
1909                 }
1910 sleep:
1911                 wake_up_process(fs_info->cleaner_kthread);
1912                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1913
1914                 if (BTRFS_FS_ERROR(fs_info))
1915                         btrfs_cleanup_transaction(fs_info);
1916                 if (!kthread_should_stop() &&
1917                                 (!btrfs_transaction_blocked(fs_info) ||
1918                                  cannot_commit))
1919                         schedule_timeout_interruptible(delay);
1920         } while (!kthread_should_stop());
1921         return 0;
1922 }
1923
1924 /*
1925  * This will find the highest generation in the array of root backups.  The
1926  * index of the highest array is returned, or -EINVAL if we can't find
1927  * anything.
1928  *
1929  * We check to make sure the array is valid by comparing the
1930  * generation of the latest  root in the array with the generation
1931  * in the super block.  If they don't match we pitch it.
1932  */
1933 static int find_newest_super_backup(struct btrfs_fs_info *info)
1934 {
1935         const u64 newest_gen = btrfs_super_generation(info->super_copy);
1936         u64 cur;
1937         struct btrfs_root_backup *root_backup;
1938         int i;
1939
1940         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1941                 root_backup = info->super_copy->super_roots + i;
1942                 cur = btrfs_backup_tree_root_gen(root_backup);
1943                 if (cur == newest_gen)
1944                         return i;
1945         }
1946
1947         return -EINVAL;
1948 }
1949
1950 /*
1951  * copy all the root pointers into the super backup array.
1952  * this will bump the backup pointer by one when it is
1953  * done
1954  */
1955 static void backup_super_roots(struct btrfs_fs_info *info)
1956 {
1957         const int next_backup = info->backup_root_index;
1958         struct btrfs_root_backup *root_backup;
1959
1960         root_backup = info->super_for_commit->super_roots + next_backup;
1961
1962         /*
1963          * make sure all of our padding and empty slots get zero filled
1964          * regardless of which ones we use today
1965          */
1966         memset(root_backup, 0, sizeof(*root_backup));
1967
1968         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1969
1970         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1971         btrfs_set_backup_tree_root_gen(root_backup,
1972                                btrfs_header_generation(info->tree_root->node));
1973
1974         btrfs_set_backup_tree_root_level(root_backup,
1975                                btrfs_header_level(info->tree_root->node));
1976
1977         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1978         btrfs_set_backup_chunk_root_gen(root_backup,
1979                                btrfs_header_generation(info->chunk_root->node));
1980         btrfs_set_backup_chunk_root_level(root_backup,
1981                                btrfs_header_level(info->chunk_root->node));
1982
1983         if (!btrfs_fs_compat_ro(info, BLOCK_GROUP_TREE)) {
1984                 struct btrfs_root *extent_root = btrfs_extent_root(info, 0);
1985                 struct btrfs_root *csum_root = btrfs_csum_root(info, 0);
1986
1987                 btrfs_set_backup_extent_root(root_backup,
1988                                              extent_root->node->start);
1989                 btrfs_set_backup_extent_root_gen(root_backup,
1990                                 btrfs_header_generation(extent_root->node));
1991                 btrfs_set_backup_extent_root_level(root_backup,
1992                                         btrfs_header_level(extent_root->node));
1993
1994                 btrfs_set_backup_csum_root(root_backup, csum_root->node->start);
1995                 btrfs_set_backup_csum_root_gen(root_backup,
1996                                                btrfs_header_generation(csum_root->node));
1997                 btrfs_set_backup_csum_root_level(root_backup,
1998                                                  btrfs_header_level(csum_root->node));
1999         }
2000
2001         /*
2002          * we might commit during log recovery, which happens before we set
2003          * the fs_root.  Make sure it is valid before we fill it in.
2004          */
2005         if (info->fs_root && info->fs_root->node) {
2006                 btrfs_set_backup_fs_root(root_backup,
2007                                          info->fs_root->node->start);
2008                 btrfs_set_backup_fs_root_gen(root_backup,
2009                                btrfs_header_generation(info->fs_root->node));
2010                 btrfs_set_backup_fs_root_level(root_backup,
2011                                btrfs_header_level(info->fs_root->node));
2012         }
2013
2014         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
2015         btrfs_set_backup_dev_root_gen(root_backup,
2016                                btrfs_header_generation(info->dev_root->node));
2017         btrfs_set_backup_dev_root_level(root_backup,
2018                                        btrfs_header_level(info->dev_root->node));
2019
2020         btrfs_set_backup_total_bytes(root_backup,
2021                              btrfs_super_total_bytes(info->super_copy));
2022         btrfs_set_backup_bytes_used(root_backup,
2023                              btrfs_super_bytes_used(info->super_copy));
2024         btrfs_set_backup_num_devices(root_backup,
2025                              btrfs_super_num_devices(info->super_copy));
2026
2027         /*
2028          * if we don't copy this out to the super_copy, it won't get remembered
2029          * for the next commit
2030          */
2031         memcpy(&info->super_copy->super_roots,
2032                &info->super_for_commit->super_roots,
2033                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
2034 }
2035
2036 /*
2037  * read_backup_root - Reads a backup root based on the passed priority. Prio 0
2038  * is the newest, prio 1/2/3 are 2nd newest/3rd newest/4th (oldest) backup roots
2039  *
2040  * fs_info - filesystem whose backup roots need to be read
2041  * priority - priority of backup root required
2042  *
2043  * Returns backup root index on success and -EINVAL otherwise.
2044  */
2045 static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority)
2046 {
2047         int backup_index = find_newest_super_backup(fs_info);
2048         struct btrfs_super_block *super = fs_info->super_copy;
2049         struct btrfs_root_backup *root_backup;
2050
2051         if (priority < BTRFS_NUM_BACKUP_ROOTS && backup_index >= 0) {
2052                 if (priority == 0)
2053                         return backup_index;
2054
2055                 backup_index = backup_index + BTRFS_NUM_BACKUP_ROOTS - priority;
2056                 backup_index %= BTRFS_NUM_BACKUP_ROOTS;
2057         } else {
2058                 return -EINVAL;
2059         }
2060
2061         root_backup = super->super_roots + backup_index;
2062
2063         btrfs_set_super_generation(super,
2064                                    btrfs_backup_tree_root_gen(root_backup));
2065         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
2066         btrfs_set_super_root_level(super,
2067                                    btrfs_backup_tree_root_level(root_backup));
2068         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
2069
2070         /*
2071          * Fixme: the total bytes and num_devices need to match or we should
2072          * need a fsck
2073          */
2074         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
2075         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
2076
2077         return backup_index;
2078 }
2079
2080 /* helper to cleanup workers */
2081 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
2082 {
2083         btrfs_destroy_workqueue(fs_info->fixup_workers);
2084         btrfs_destroy_workqueue(fs_info->delalloc_workers);
2085         btrfs_destroy_workqueue(fs_info->hipri_workers);
2086         btrfs_destroy_workqueue(fs_info->workers);
2087         if (fs_info->endio_workers)
2088                 destroy_workqueue(fs_info->endio_workers);
2089         if (fs_info->endio_raid56_workers)
2090                 destroy_workqueue(fs_info->endio_raid56_workers);
2091         if (fs_info->rmw_workers)
2092                 destroy_workqueue(fs_info->rmw_workers);
2093         if (fs_info->compressed_write_workers)
2094                 destroy_workqueue(fs_info->compressed_write_workers);
2095         btrfs_destroy_workqueue(fs_info->endio_write_workers);
2096         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
2097         btrfs_destroy_workqueue(fs_info->delayed_workers);
2098         btrfs_destroy_workqueue(fs_info->caching_workers);
2099         btrfs_destroy_workqueue(fs_info->flush_workers);
2100         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
2101         if (fs_info->discard_ctl.discard_workers)
2102                 destroy_workqueue(fs_info->discard_ctl.discard_workers);
2103         /*
2104          * Now that all other work queues are destroyed, we can safely destroy
2105          * the queues used for metadata I/O, since tasks from those other work
2106          * queues can do metadata I/O operations.
2107          */
2108         if (fs_info->endio_meta_workers)
2109                 destroy_workqueue(fs_info->endio_meta_workers);
2110 }
2111
2112 static void free_root_extent_buffers(struct btrfs_root *root)
2113 {
2114         if (root) {
2115                 free_extent_buffer(root->node);
2116                 free_extent_buffer(root->commit_root);
2117                 root->node = NULL;
2118                 root->commit_root = NULL;
2119         }
2120 }
2121
2122 static void free_global_root_pointers(struct btrfs_fs_info *fs_info)
2123 {
2124         struct btrfs_root *root, *tmp;
2125
2126         rbtree_postorder_for_each_entry_safe(root, tmp,
2127                                              &fs_info->global_root_tree,
2128                                              rb_node)
2129                 free_root_extent_buffers(root);
2130 }
2131
2132 /* helper to cleanup tree roots */
2133 static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root)
2134 {
2135         free_root_extent_buffers(info->tree_root);
2136
2137         free_global_root_pointers(info);
2138         free_root_extent_buffers(info->dev_root);
2139         free_root_extent_buffers(info->quota_root);
2140         free_root_extent_buffers(info->uuid_root);
2141         free_root_extent_buffers(info->fs_root);
2142         free_root_extent_buffers(info->data_reloc_root);
2143         free_root_extent_buffers(info->block_group_root);
2144         if (free_chunk_root)
2145                 free_root_extent_buffers(info->chunk_root);
2146 }
2147
2148 void btrfs_put_root(struct btrfs_root *root)
2149 {
2150         if (!root)
2151                 return;
2152
2153         if (refcount_dec_and_test(&root->refs)) {
2154                 WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
2155                 WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state));
2156                 if (root->anon_dev)
2157                         free_anon_bdev(root->anon_dev);
2158                 btrfs_drew_lock_destroy(&root->snapshot_lock);
2159                 free_root_extent_buffers(root);
2160 #ifdef CONFIG_BTRFS_DEBUG
2161                 spin_lock(&root->fs_info->fs_roots_radix_lock);
2162                 list_del_init(&root->leak_list);
2163                 spin_unlock(&root->fs_info->fs_roots_radix_lock);
2164 #endif
2165                 kfree(root);
2166         }
2167 }
2168
2169 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
2170 {
2171         int ret;
2172         struct btrfs_root *gang[8];
2173         int i;
2174
2175         while (!list_empty(&fs_info->dead_roots)) {
2176                 gang[0] = list_entry(fs_info->dead_roots.next,
2177                                      struct btrfs_root, root_list);
2178                 list_del(&gang[0]->root_list);
2179
2180                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state))
2181                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
2182                 btrfs_put_root(gang[0]);
2183         }
2184
2185         while (1) {
2186                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
2187                                              (void **)gang, 0,
2188                                              ARRAY_SIZE(gang));
2189                 if (!ret)
2190                         break;
2191                 for (i = 0; i < ret; i++)
2192                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2193         }
2194 }
2195
2196 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2197 {
2198         mutex_init(&fs_info->scrub_lock);
2199         atomic_set(&fs_info->scrubs_running, 0);
2200         atomic_set(&fs_info->scrub_pause_req, 0);
2201         atomic_set(&fs_info->scrubs_paused, 0);
2202         atomic_set(&fs_info->scrub_cancel_req, 0);
2203         init_waitqueue_head(&fs_info->scrub_pause_wait);
2204         refcount_set(&fs_info->scrub_workers_refcnt, 0);
2205 }
2206
2207 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2208 {
2209         spin_lock_init(&fs_info->balance_lock);
2210         mutex_init(&fs_info->balance_mutex);
2211         atomic_set(&fs_info->balance_pause_req, 0);
2212         atomic_set(&fs_info->balance_cancel_req, 0);
2213         fs_info->balance_ctl = NULL;
2214         init_waitqueue_head(&fs_info->balance_wait_q);
2215         atomic_set(&fs_info->reloc_cancel_req, 0);
2216 }
2217
2218 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2219 {
2220         struct inode *inode = fs_info->btree_inode;
2221         unsigned long hash = btrfs_inode_hash(BTRFS_BTREE_INODE_OBJECTID,
2222                                               fs_info->tree_root);
2223
2224         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2225         set_nlink(inode, 1);
2226         /*
2227          * we set the i_size on the btree inode to the max possible int.
2228          * the real end of the address space is determined by all of
2229          * the devices in the system
2230          */
2231         inode->i_size = OFFSET_MAX;
2232         inode->i_mapping->a_ops = &btree_aops;
2233
2234         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2235         extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
2236                             IO_TREE_BTREE_INODE_IO, NULL);
2237         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2238
2239         BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root);
2240         BTRFS_I(inode)->location.objectid = BTRFS_BTREE_INODE_OBJECTID;
2241         BTRFS_I(inode)->location.type = 0;
2242         BTRFS_I(inode)->location.offset = 0;
2243         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2244         __insert_inode_hash(inode, hash);
2245 }
2246
2247 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2248 {
2249         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2250         init_rwsem(&fs_info->dev_replace.rwsem);
2251         init_waitqueue_head(&fs_info->dev_replace.replace_wait);
2252 }
2253
2254 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2255 {
2256         spin_lock_init(&fs_info->qgroup_lock);
2257         mutex_init(&fs_info->qgroup_ioctl_lock);
2258         fs_info->qgroup_tree = RB_ROOT;
2259         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2260         fs_info->qgroup_seq = 1;
2261         fs_info->qgroup_ulist = NULL;
2262         fs_info->qgroup_rescan_running = false;
2263         fs_info->qgroup_drop_subtree_thres = BTRFS_MAX_LEVEL;
2264         mutex_init(&fs_info->qgroup_rescan_lock);
2265 }
2266
2267 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
2268 {
2269         u32 max_active = fs_info->thread_pool_size;
2270         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2271
2272         fs_info->workers =
2273                 btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16);
2274         fs_info->hipri_workers =
2275                 btrfs_alloc_workqueue(fs_info, "worker-high",
2276                                       flags | WQ_HIGHPRI, max_active, 16);
2277
2278         fs_info->delalloc_workers =
2279                 btrfs_alloc_workqueue(fs_info, "delalloc",
2280                                       flags, max_active, 2);
2281
2282         fs_info->flush_workers =
2283                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2284                                       flags, max_active, 0);
2285
2286         fs_info->caching_workers =
2287                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2288
2289         fs_info->fixup_workers =
2290                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2291
2292         fs_info->endio_workers =
2293                 alloc_workqueue("btrfs-endio", flags, max_active);
2294         fs_info->endio_meta_workers =
2295                 alloc_workqueue("btrfs-endio-meta", flags, max_active);
2296         fs_info->endio_raid56_workers =
2297                 alloc_workqueue("btrfs-endio-raid56", flags, max_active);
2298         fs_info->rmw_workers = alloc_workqueue("btrfs-rmw", flags, max_active);
2299         fs_info->endio_write_workers =
2300                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2301                                       max_active, 2);
2302         fs_info->compressed_write_workers =
2303                 alloc_workqueue("btrfs-compressed-write", flags, max_active);
2304         fs_info->endio_freespace_worker =
2305                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2306                                       max_active, 0);
2307         fs_info->delayed_workers =
2308                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2309                                       max_active, 0);
2310         fs_info->qgroup_rescan_workers =
2311                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2312         fs_info->discard_ctl.discard_workers =
2313                 alloc_workqueue("btrfs_discard", WQ_UNBOUND | WQ_FREEZABLE, 1);
2314
2315         if (!(fs_info->workers && fs_info->hipri_workers &&
2316               fs_info->delalloc_workers && fs_info->flush_workers &&
2317               fs_info->endio_workers && fs_info->endio_meta_workers &&
2318               fs_info->compressed_write_workers &&
2319               fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
2320               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2321               fs_info->caching_workers && fs_info->fixup_workers &&
2322               fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
2323               fs_info->discard_ctl.discard_workers)) {
2324                 return -ENOMEM;
2325         }
2326
2327         return 0;
2328 }
2329
2330 static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
2331 {
2332         struct crypto_shash *csum_shash;
2333         const char *csum_driver = btrfs_super_csum_driver(csum_type);
2334
2335         csum_shash = crypto_alloc_shash(csum_driver, 0, 0);
2336
2337         if (IS_ERR(csum_shash)) {
2338                 btrfs_err(fs_info, "error allocating %s hash for checksum",
2339                           csum_driver);
2340                 return PTR_ERR(csum_shash);
2341         }
2342
2343         fs_info->csum_shash = csum_shash;
2344
2345         btrfs_info(fs_info, "using %s (%s) checksum algorithm",
2346                         btrfs_super_csum_name(csum_type),
2347                         crypto_shash_driver_name(csum_shash));
2348         return 0;
2349 }
2350
2351 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2352                             struct btrfs_fs_devices *fs_devices)
2353 {
2354         int ret;
2355         struct btrfs_root *log_tree_root;
2356         struct btrfs_super_block *disk_super = fs_info->super_copy;
2357         u64 bytenr = btrfs_super_log_root(disk_super);
2358         int level = btrfs_super_log_root_level(disk_super);
2359
2360         if (fs_devices->rw_devices == 0) {
2361                 btrfs_warn(fs_info, "log replay required on RO media");
2362                 return -EIO;
2363         }
2364
2365         log_tree_root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID,
2366                                          GFP_KERNEL);
2367         if (!log_tree_root)
2368                 return -ENOMEM;
2369
2370         log_tree_root->node = read_tree_block(fs_info, bytenr,
2371                                               BTRFS_TREE_LOG_OBJECTID,
2372                                               fs_info->generation + 1, level,
2373                                               NULL);
2374         if (IS_ERR(log_tree_root->node)) {
2375                 btrfs_warn(fs_info, "failed to read log tree");
2376                 ret = PTR_ERR(log_tree_root->node);
2377                 log_tree_root->node = NULL;
2378                 btrfs_put_root(log_tree_root);
2379                 return ret;
2380         }
2381         if (!extent_buffer_uptodate(log_tree_root->node)) {
2382                 btrfs_err(fs_info, "failed to read log tree");
2383                 btrfs_put_root(log_tree_root);
2384                 return -EIO;
2385         }
2386
2387         /* returns with log_tree_root freed on success */
2388         ret = btrfs_recover_log_trees(log_tree_root);
2389         if (ret) {
2390                 btrfs_handle_fs_error(fs_info, ret,
2391                                       "Failed to recover log tree");
2392                 btrfs_put_root(log_tree_root);
2393                 return ret;
2394         }
2395
2396         if (sb_rdonly(fs_info->sb)) {
2397                 ret = btrfs_commit_super(fs_info);
2398                 if (ret)
2399                         return ret;
2400         }
2401
2402         return 0;
2403 }
2404
2405 static int load_global_roots_objectid(struct btrfs_root *tree_root,
2406                                       struct btrfs_path *path, u64 objectid,
2407                                       const char *name)
2408 {
2409         struct btrfs_fs_info *fs_info = tree_root->fs_info;
2410         struct btrfs_root *root;
2411         u64 max_global_id = 0;
2412         int ret;
2413         struct btrfs_key key = {
2414                 .objectid = objectid,
2415                 .type = BTRFS_ROOT_ITEM_KEY,
2416                 .offset = 0,
2417         };
2418         bool found = false;
2419
2420         /* If we have IGNOREDATACSUMS skip loading these roots. */
2421         if (objectid == BTRFS_CSUM_TREE_OBJECTID &&
2422             btrfs_test_opt(fs_info, IGNOREDATACSUMS)) {
2423                 set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2424                 return 0;
2425         }
2426
2427         while (1) {
2428                 ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0);
2429                 if (ret < 0)
2430                         break;
2431
2432                 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2433                         ret = btrfs_next_leaf(tree_root, path);
2434                         if (ret) {
2435                                 if (ret > 0)
2436                                         ret = 0;
2437                                 break;
2438                         }
2439                 }
2440                 ret = 0;
2441
2442                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2443                 if (key.objectid != objectid)
2444                         break;
2445                 btrfs_release_path(path);
2446
2447                 /*
2448                  * Just worry about this for extent tree, it'll be the same for
2449                  * everybody.
2450                  */
2451                 if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2452                         max_global_id = max(max_global_id, key.offset);
2453
2454                 found = true;
2455                 root = read_tree_root_path(tree_root, path, &key);
2456                 if (IS_ERR(root)) {
2457                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2458                                 ret = PTR_ERR(root);
2459                         break;
2460                 }
2461                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2462                 ret = btrfs_global_root_insert(root);
2463                 if (ret) {
2464                         btrfs_put_root(root);
2465                         break;
2466                 }
2467                 key.offset++;
2468         }
2469         btrfs_release_path(path);
2470
2471         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2472                 fs_info->nr_global_roots = max_global_id + 1;
2473
2474         if (!found || ret) {
2475                 if (objectid == BTRFS_CSUM_TREE_OBJECTID)
2476                         set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2477
2478                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2479                         ret = ret ? ret : -ENOENT;
2480                 else
2481                         ret = 0;
2482                 btrfs_err(fs_info, "failed to load root %s", name);
2483         }
2484         return ret;
2485 }
2486
2487 static int load_global_roots(struct btrfs_root *tree_root)
2488 {
2489         struct btrfs_path *path;
2490         int ret = 0;
2491
2492         path = btrfs_alloc_path();
2493         if (!path)
2494                 return -ENOMEM;
2495
2496         ret = load_global_roots_objectid(tree_root, path,
2497                                          BTRFS_EXTENT_TREE_OBJECTID, "extent");
2498         if (ret)
2499                 goto out;
2500         ret = load_global_roots_objectid(tree_root, path,
2501                                          BTRFS_CSUM_TREE_OBJECTID, "csum");
2502         if (ret)
2503                 goto out;
2504         if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE))
2505                 goto out;
2506         ret = load_global_roots_objectid(tree_root, path,
2507                                          BTRFS_FREE_SPACE_TREE_OBJECTID,
2508                                          "free space");
2509 out:
2510         btrfs_free_path(path);
2511         return ret;
2512 }
2513
2514 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2515 {
2516         struct btrfs_root *tree_root = fs_info->tree_root;
2517         struct btrfs_root *root;
2518         struct btrfs_key location;
2519         int ret;
2520
2521         BUG_ON(!fs_info->tree_root);
2522
2523         ret = load_global_roots(tree_root);
2524         if (ret)
2525                 return ret;
2526
2527         location.type = BTRFS_ROOT_ITEM_KEY;
2528         location.offset = 0;
2529
2530         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) {
2531                 location.objectid = BTRFS_BLOCK_GROUP_TREE_OBJECTID;
2532                 root = btrfs_read_tree_root(tree_root, &location);
2533                 if (IS_ERR(root)) {
2534                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2535                                 ret = PTR_ERR(root);
2536                                 goto out;
2537                         }
2538                 } else {
2539                         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2540                         fs_info->block_group_root = root;
2541                 }
2542         }
2543
2544         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2545         root = btrfs_read_tree_root(tree_root, &location);
2546         if (IS_ERR(root)) {
2547                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2548                         ret = PTR_ERR(root);
2549                         goto out;
2550                 }
2551         } else {
2552                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2553                 fs_info->dev_root = root;
2554         }
2555         /* Initialize fs_info for all devices in any case */
2556         btrfs_init_devices_late(fs_info);
2557
2558         /*
2559          * This tree can share blocks with some other fs tree during relocation
2560          * and we need a proper setup by btrfs_get_fs_root
2561          */
2562         root = btrfs_get_fs_root(tree_root->fs_info,
2563                                  BTRFS_DATA_RELOC_TREE_OBJECTID, true);
2564         if (IS_ERR(root)) {
2565                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2566                         ret = PTR_ERR(root);
2567                         goto out;
2568                 }
2569         } else {
2570                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2571                 fs_info->data_reloc_root = root;
2572         }
2573
2574         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2575         root = btrfs_read_tree_root(tree_root, &location);
2576         if (!IS_ERR(root)) {
2577                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2578                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2579                 fs_info->quota_root = root;
2580         }
2581
2582         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2583         root = btrfs_read_tree_root(tree_root, &location);
2584         if (IS_ERR(root)) {
2585                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2586                         ret = PTR_ERR(root);
2587                         if (ret != -ENOENT)
2588                                 goto out;
2589                 }
2590         } else {
2591                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2592                 fs_info->uuid_root = root;
2593         }
2594
2595         return 0;
2596 out:
2597         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2598                    location.objectid, ret);
2599         return ret;
2600 }
2601
2602 /*
2603  * Real super block validation
2604  * NOTE: super csum type and incompat features will not be checked here.
2605  *
2606  * @sb:         super block to check
2607  * @mirror_num: the super block number to check its bytenr:
2608  *              0       the primary (1st) sb
2609  *              1, 2    2nd and 3rd backup copy
2610  *             -1       skip bytenr check
2611  */
2612 int btrfs_validate_super(struct btrfs_fs_info *fs_info,
2613                          struct btrfs_super_block *sb, int mirror_num)
2614 {
2615         u64 nodesize = btrfs_super_nodesize(sb);
2616         u64 sectorsize = btrfs_super_sectorsize(sb);
2617         int ret = 0;
2618
2619         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2620                 btrfs_err(fs_info, "no valid FS found");
2621                 ret = -EINVAL;
2622         }
2623         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2624                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2625                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2626                 ret = -EINVAL;
2627         }
2628         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2629                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2630                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2631                 ret = -EINVAL;
2632         }
2633         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2634                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2635                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2636                 ret = -EINVAL;
2637         }
2638         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2639                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2640                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2641                 ret = -EINVAL;
2642         }
2643
2644         /*
2645          * Check sectorsize and nodesize first, other check will need it.
2646          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2647          */
2648         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2649             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2650                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2651                 ret = -EINVAL;
2652         }
2653
2654         /*
2655          * We only support at most two sectorsizes: 4K and PAGE_SIZE.
2656          *
2657          * We can support 16K sectorsize with 64K page size without problem,
2658          * but such sectorsize/pagesize combination doesn't make much sense.
2659          * 4K will be our future standard, PAGE_SIZE is supported from the very
2660          * beginning.
2661          */
2662         if (sectorsize > PAGE_SIZE || (sectorsize != SZ_4K && sectorsize != PAGE_SIZE)) {
2663                 btrfs_err(fs_info,
2664                         "sectorsize %llu not yet supported for page size %lu",
2665                         sectorsize, PAGE_SIZE);
2666                 ret = -EINVAL;
2667         }
2668
2669         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2670             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2671                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2672                 ret = -EINVAL;
2673         }
2674         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2675                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2676                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2677                 ret = -EINVAL;
2678         }
2679
2680         /* Root alignment check */
2681         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2682                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2683                            btrfs_super_root(sb));
2684                 ret = -EINVAL;
2685         }
2686         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2687                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2688                            btrfs_super_chunk_root(sb));
2689                 ret = -EINVAL;
2690         }
2691         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2692                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2693                            btrfs_super_log_root(sb));
2694                 ret = -EINVAL;
2695         }
2696
2697         if (memcmp(fs_info->fs_devices->fsid, fs_info->super_copy->fsid,
2698                    BTRFS_FSID_SIZE)) {
2699                 btrfs_err(fs_info,
2700                 "superblock fsid doesn't match fsid of fs_devices: %pU != %pU",
2701                         fs_info->super_copy->fsid, fs_info->fs_devices->fsid);
2702                 ret = -EINVAL;
2703         }
2704
2705         if (btrfs_fs_incompat(fs_info, METADATA_UUID) &&
2706             memcmp(fs_info->fs_devices->metadata_uuid,
2707                    fs_info->super_copy->metadata_uuid, BTRFS_FSID_SIZE)) {
2708                 btrfs_err(fs_info,
2709 "superblock metadata_uuid doesn't match metadata uuid of fs_devices: %pU != %pU",
2710                         fs_info->super_copy->metadata_uuid,
2711                         fs_info->fs_devices->metadata_uuid);
2712                 ret = -EINVAL;
2713         }
2714
2715         /*
2716          * Artificial requirement for block-group-tree to force newer features
2717          * (free-space-tree, no-holes) so the test matrix is smaller.
2718          */
2719         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
2720             (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
2721              !btrfs_fs_incompat(fs_info, NO_HOLES))) {
2722                 btrfs_err(fs_info,
2723                 "block-group-tree feature requires fres-space-tree and no-holes");
2724                 ret = -EINVAL;
2725         }
2726
2727         if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
2728                    BTRFS_FSID_SIZE) != 0) {
2729                 btrfs_err(fs_info,
2730                         "dev_item UUID does not match metadata fsid: %pU != %pU",
2731                         fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid);
2732                 ret = -EINVAL;
2733         }
2734
2735         /*
2736          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2737          * done later
2738          */
2739         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2740                 btrfs_err(fs_info, "bytes_used is too small %llu",
2741                           btrfs_super_bytes_used(sb));
2742                 ret = -EINVAL;
2743         }
2744         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2745                 btrfs_err(fs_info, "invalid stripesize %u",
2746                           btrfs_super_stripesize(sb));
2747                 ret = -EINVAL;
2748         }
2749         if (btrfs_super_num_devices(sb) > (1UL << 31))
2750                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2751                            btrfs_super_num_devices(sb));
2752         if (btrfs_super_num_devices(sb) == 0) {
2753                 btrfs_err(fs_info, "number of devices is 0");
2754                 ret = -EINVAL;
2755         }
2756
2757         if (mirror_num >= 0 &&
2758             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2759                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2760                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2761                 ret = -EINVAL;
2762         }
2763
2764         /*
2765          * Obvious sys_chunk_array corruptions, it must hold at least one key
2766          * and one chunk
2767          */
2768         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2769                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2770                           btrfs_super_sys_array_size(sb),
2771                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2772                 ret = -EINVAL;
2773         }
2774         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2775                         + sizeof(struct btrfs_chunk)) {
2776                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2777                           btrfs_super_sys_array_size(sb),
2778                           sizeof(struct btrfs_disk_key)
2779                           + sizeof(struct btrfs_chunk));
2780                 ret = -EINVAL;
2781         }
2782
2783         /*
2784          * The generation is a global counter, we'll trust it more than the others
2785          * but it's still possible that it's the one that's wrong.
2786          */
2787         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2788                 btrfs_warn(fs_info,
2789                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2790                         btrfs_super_generation(sb),
2791                         btrfs_super_chunk_root_generation(sb));
2792         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2793             && btrfs_super_cache_generation(sb) != (u64)-1)
2794                 btrfs_warn(fs_info,
2795                         "suspicious: generation < cache_generation: %llu < %llu",
2796                         btrfs_super_generation(sb),
2797                         btrfs_super_cache_generation(sb));
2798
2799         return ret;
2800 }
2801
2802 /*
2803  * Validation of super block at mount time.
2804  * Some checks already done early at mount time, like csum type and incompat
2805  * flags will be skipped.
2806  */
2807 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2808 {
2809         return btrfs_validate_super(fs_info, fs_info->super_copy, 0);
2810 }
2811
2812 /*
2813  * Validation of super block at write time.
2814  * Some checks like bytenr check will be skipped as their values will be
2815  * overwritten soon.
2816  * Extra checks like csum type and incompat flags will be done here.
2817  */
2818 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2819                                       struct btrfs_super_block *sb)
2820 {
2821         int ret;
2822
2823         ret = btrfs_validate_super(fs_info, sb, -1);
2824         if (ret < 0)
2825                 goto out;
2826         if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) {
2827                 ret = -EUCLEAN;
2828                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2829                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2830                 goto out;
2831         }
2832         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2833                 ret = -EUCLEAN;
2834                 btrfs_err(fs_info,
2835                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2836                           btrfs_super_incompat_flags(sb),
2837                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2838                 goto out;
2839         }
2840 out:
2841         if (ret < 0)
2842                 btrfs_err(fs_info,
2843                 "super block corruption detected before writing it to disk");
2844         return ret;
2845 }
2846
2847 static int load_super_root(struct btrfs_root *root, u64 bytenr, u64 gen, int level)
2848 {
2849         int ret = 0;
2850
2851         root->node = read_tree_block(root->fs_info, bytenr,
2852                                      root->root_key.objectid, gen, level, NULL);
2853         if (IS_ERR(root->node)) {
2854                 ret = PTR_ERR(root->node);
2855                 root->node = NULL;
2856                 return ret;
2857         }
2858         if (!extent_buffer_uptodate(root->node)) {
2859                 free_extent_buffer(root->node);
2860                 root->node = NULL;
2861                 return -EIO;
2862         }
2863
2864         btrfs_set_root_node(&root->root_item, root->node);
2865         root->commit_root = btrfs_root_node(root);
2866         btrfs_set_root_refs(&root->root_item, 1);
2867         return ret;
2868 }
2869
2870 static int load_important_roots(struct btrfs_fs_info *fs_info)
2871 {
2872         struct btrfs_super_block *sb = fs_info->super_copy;
2873         u64 gen, bytenr;
2874         int level, ret;
2875
2876         bytenr = btrfs_super_root(sb);
2877         gen = btrfs_super_generation(sb);
2878         level = btrfs_super_root_level(sb);
2879         ret = load_super_root(fs_info->tree_root, bytenr, gen, level);
2880         if (ret) {
2881                 btrfs_warn(fs_info, "couldn't read tree root");
2882                 return ret;
2883         }
2884         return 0;
2885 }
2886
2887 static int __cold init_tree_roots(struct btrfs_fs_info *fs_info)
2888 {
2889         int backup_index = find_newest_super_backup(fs_info);
2890         struct btrfs_super_block *sb = fs_info->super_copy;
2891         struct btrfs_root *tree_root = fs_info->tree_root;
2892         bool handle_error = false;
2893         int ret = 0;
2894         int i;
2895
2896         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
2897                 if (handle_error) {
2898                         if (!IS_ERR(tree_root->node))
2899                                 free_extent_buffer(tree_root->node);
2900                         tree_root->node = NULL;
2901
2902                         if (!btrfs_test_opt(fs_info, USEBACKUPROOT))
2903                                 break;
2904
2905                         free_root_pointers(fs_info, 0);
2906
2907                         /*
2908                          * Don't use the log in recovery mode, it won't be
2909                          * valid
2910                          */
2911                         btrfs_set_super_log_root(sb, 0);
2912
2913                         /* We can't trust the free space cache either */
2914                         btrfs_set_opt(fs_info->mount_opt, CLEAR_CACHE);
2915
2916                         ret = read_backup_root(fs_info, i);
2917                         backup_index = ret;
2918                         if (ret < 0)
2919                                 return ret;
2920                 }
2921
2922                 ret = load_important_roots(fs_info);
2923                 if (ret) {
2924                         handle_error = true;
2925                         continue;
2926                 }
2927
2928                 /*
2929                  * No need to hold btrfs_root::objectid_mutex since the fs
2930                  * hasn't been fully initialised and we are the only user
2931                  */
2932                 ret = btrfs_init_root_free_objectid(tree_root);
2933                 if (ret < 0) {
2934                         handle_error = true;
2935                         continue;
2936                 }
2937
2938                 ASSERT(tree_root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
2939
2940                 ret = btrfs_read_roots(fs_info);
2941                 if (ret < 0) {
2942                         handle_error = true;
2943                         continue;
2944                 }
2945
2946                 /* All successful */
2947                 fs_info->generation = btrfs_header_generation(tree_root->node);
2948                 fs_info->last_trans_committed = fs_info->generation;
2949                 fs_info->last_reloc_trans = 0;
2950
2951                 /* Always begin writing backup roots after the one being used */
2952                 if (backup_index < 0) {
2953                         fs_info->backup_root_index = 0;
2954                 } else {
2955                         fs_info->backup_root_index = backup_index + 1;
2956                         fs_info->backup_root_index %= BTRFS_NUM_BACKUP_ROOTS;
2957                 }
2958                 break;
2959         }
2960
2961         return ret;
2962 }
2963
2964 void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
2965 {
2966         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2967         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2968         INIT_LIST_HEAD(&fs_info->trans_list);
2969         INIT_LIST_HEAD(&fs_info->dead_roots);
2970         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2971         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2972         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2973         spin_lock_init(&fs_info->delalloc_root_lock);
2974         spin_lock_init(&fs_info->trans_lock);
2975         spin_lock_init(&fs_info->fs_roots_radix_lock);
2976         spin_lock_init(&fs_info->delayed_iput_lock);
2977         spin_lock_init(&fs_info->defrag_inodes_lock);
2978         spin_lock_init(&fs_info->super_lock);
2979         spin_lock_init(&fs_info->buffer_lock);
2980         spin_lock_init(&fs_info->unused_bgs_lock);
2981         spin_lock_init(&fs_info->treelog_bg_lock);
2982         spin_lock_init(&fs_info->zone_active_bgs_lock);
2983         spin_lock_init(&fs_info->relocation_bg_lock);
2984         rwlock_init(&fs_info->tree_mod_log_lock);
2985         rwlock_init(&fs_info->global_root_lock);
2986         mutex_init(&fs_info->unused_bg_unpin_mutex);
2987         mutex_init(&fs_info->reclaim_bgs_lock);
2988         mutex_init(&fs_info->reloc_mutex);
2989         mutex_init(&fs_info->delalloc_root_mutex);
2990         mutex_init(&fs_info->zoned_meta_io_lock);
2991         mutex_init(&fs_info->zoned_data_reloc_io_lock);
2992         seqlock_init(&fs_info->profiles_lock);
2993
2994         btrfs_lockdep_init_map(fs_info, btrfs_trans_num_writers);
2995         btrfs_lockdep_init_map(fs_info, btrfs_trans_num_extwriters);
2996         btrfs_lockdep_init_map(fs_info, btrfs_trans_pending_ordered);
2997         btrfs_lockdep_init_map(fs_info, btrfs_ordered_extent);
2998         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_commit_start,
2999                                      BTRFS_LOCKDEP_TRANS_COMMIT_START);
3000         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_unblocked,
3001                                      BTRFS_LOCKDEP_TRANS_UNBLOCKED);
3002         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_super_committed,
3003                                      BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED);
3004         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_completed,
3005                                      BTRFS_LOCKDEP_TRANS_COMPLETED);
3006
3007         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
3008         INIT_LIST_HEAD(&fs_info->space_info);
3009         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
3010         INIT_LIST_HEAD(&fs_info->unused_bgs);
3011         INIT_LIST_HEAD(&fs_info->reclaim_bgs);
3012         INIT_LIST_HEAD(&fs_info->zone_active_bgs);
3013 #ifdef CONFIG_BTRFS_DEBUG
3014         INIT_LIST_HEAD(&fs_info->allocated_roots);
3015         INIT_LIST_HEAD(&fs_info->allocated_ebs);
3016         spin_lock_init(&fs_info->eb_leak_lock);
3017 #endif
3018         extent_map_tree_init(&fs_info->mapping_tree);
3019         btrfs_init_block_rsv(&fs_info->global_block_rsv,
3020                              BTRFS_BLOCK_RSV_GLOBAL);
3021         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
3022         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
3023         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
3024         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
3025                              BTRFS_BLOCK_RSV_DELOPS);
3026         btrfs_init_block_rsv(&fs_info->delayed_refs_rsv,
3027                              BTRFS_BLOCK_RSV_DELREFS);
3028
3029         atomic_set(&fs_info->async_delalloc_pages, 0);
3030         atomic_set(&fs_info->defrag_running, 0);
3031         atomic_set(&fs_info->nr_delayed_iputs, 0);
3032         atomic64_set(&fs_info->tree_mod_seq, 0);
3033         fs_info->global_root_tree = RB_ROOT;
3034         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
3035         fs_info->metadata_ratio = 0;
3036         fs_info->defrag_inodes = RB_ROOT;
3037         atomic64_set(&fs_info->free_chunk_space, 0);
3038         fs_info->tree_mod_log = RB_ROOT;
3039         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
3040         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
3041         btrfs_init_ref_verify(fs_info);
3042
3043         fs_info->thread_pool_size = min_t(unsigned long,
3044                                           num_online_cpus() + 2, 8);
3045
3046         INIT_LIST_HEAD(&fs_info->ordered_roots);
3047         spin_lock_init(&fs_info->ordered_root_lock);
3048
3049         btrfs_init_scrub(fs_info);
3050 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3051         fs_info->check_integrity_print_mask = 0;
3052 #endif
3053         btrfs_init_balance(fs_info);
3054         btrfs_init_async_reclaim_work(fs_info);
3055
3056         rwlock_init(&fs_info->block_group_cache_lock);
3057         fs_info->block_group_cache_tree = RB_ROOT_CACHED;
3058
3059         extent_io_tree_init(fs_info, &fs_info->excluded_extents,
3060                             IO_TREE_FS_EXCLUDED_EXTENTS, NULL);
3061
3062         mutex_init(&fs_info->ordered_operations_mutex);
3063         mutex_init(&fs_info->tree_log_mutex);
3064         mutex_init(&fs_info->chunk_mutex);
3065         mutex_init(&fs_info->transaction_kthread_mutex);
3066         mutex_init(&fs_info->cleaner_mutex);
3067         mutex_init(&fs_info->ro_block_group_mutex);
3068         init_rwsem(&fs_info->commit_root_sem);
3069         init_rwsem(&fs_info->cleanup_work_sem);
3070         init_rwsem(&fs_info->subvol_sem);
3071         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
3072
3073         btrfs_init_dev_replace_locks(fs_info);
3074         btrfs_init_qgroup(fs_info);
3075         btrfs_discard_init(fs_info);
3076
3077         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
3078         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
3079
3080         init_waitqueue_head(&fs_info->transaction_throttle);
3081         init_waitqueue_head(&fs_info->transaction_wait);
3082         init_waitqueue_head(&fs_info->transaction_blocked_wait);
3083         init_waitqueue_head(&fs_info->async_submit_wait);
3084         init_waitqueue_head(&fs_info->delayed_iputs_wait);
3085
3086         /* Usable values until the real ones are cached from the superblock */
3087         fs_info->nodesize = 4096;
3088         fs_info->sectorsize = 4096;
3089         fs_info->sectorsize_bits = ilog2(4096);
3090         fs_info->stripesize = 4096;
3091
3092         fs_info->max_extent_size = BTRFS_MAX_EXTENT_SIZE;
3093
3094         spin_lock_init(&fs_info->swapfile_pins_lock);
3095         fs_info->swapfile_pins = RB_ROOT;
3096
3097         fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH;
3098         INIT_WORK(&fs_info->reclaim_bgs_work, btrfs_reclaim_bgs_work);
3099 }
3100
3101 static int init_mount_fs_info(struct btrfs_fs_info *fs_info, struct super_block *sb)
3102 {
3103         int ret;
3104
3105         fs_info->sb = sb;
3106         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
3107         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
3108
3109         ret = percpu_counter_init(&fs_info->ordered_bytes, 0, GFP_KERNEL);
3110         if (ret)
3111                 return ret;
3112
3113         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
3114         if (ret)
3115                 return ret;
3116
3117         fs_info->dirty_metadata_batch = PAGE_SIZE *
3118                                         (1 + ilog2(nr_cpu_ids));
3119
3120         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
3121         if (ret)
3122                 return ret;
3123
3124         ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0,
3125                         GFP_KERNEL);
3126         if (ret)
3127                 return ret;
3128
3129         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
3130                                         GFP_KERNEL);
3131         if (!fs_info->delayed_root)
3132                 return -ENOMEM;
3133         btrfs_init_delayed_root(fs_info->delayed_root);
3134
3135         if (sb_rdonly(sb))
3136                 set_bit(BTRFS_FS_STATE_RO, &fs_info->fs_state);
3137
3138         return btrfs_alloc_stripe_hash_table(fs_info);
3139 }
3140
3141 static int btrfs_uuid_rescan_kthread(void *data)
3142 {
3143         struct btrfs_fs_info *fs_info = data;
3144         int ret;
3145
3146         /*
3147          * 1st step is to iterate through the existing UUID tree and
3148          * to delete all entries that contain outdated data.
3149          * 2nd step is to add all missing entries to the UUID tree.
3150          */
3151         ret = btrfs_uuid_tree_iterate(fs_info);
3152         if (ret < 0) {
3153                 if (ret != -EINTR)
3154                         btrfs_warn(fs_info, "iterating uuid_tree failed %d",
3155                                    ret);
3156                 up(&fs_info->uuid_tree_rescan_sem);
3157                 return ret;
3158         }
3159         return btrfs_uuid_scan_kthread(data);
3160 }
3161
3162 static int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info)
3163 {
3164         struct task_struct *task;
3165
3166         down(&fs_info->uuid_tree_rescan_sem);
3167         task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid");
3168         if (IS_ERR(task)) {
3169                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
3170                 btrfs_warn(fs_info, "failed to start uuid_rescan task");
3171                 up(&fs_info->uuid_tree_rescan_sem);
3172                 return PTR_ERR(task);
3173         }
3174
3175         return 0;
3176 }
3177
3178 /*
3179  * Some options only have meaning at mount time and shouldn't persist across
3180  * remounts, or be displayed. Clear these at the end of mount and remount
3181  * code paths.
3182  */
3183 void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info)
3184 {
3185         btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT);
3186         btrfs_clear_opt(fs_info->mount_opt, CLEAR_CACHE);
3187 }
3188
3189 /*
3190  * Mounting logic specific to read-write file systems. Shared by open_ctree
3191  * and btrfs_remount when remounting from read-only to read-write.
3192  */
3193 int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info)
3194 {
3195         int ret;
3196         const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE);
3197         bool clear_free_space_tree = false;
3198
3199         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3200             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3201                 clear_free_space_tree = true;
3202         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3203                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3204                 btrfs_warn(fs_info, "free space tree is invalid");
3205                 clear_free_space_tree = true;
3206         }
3207
3208         if (clear_free_space_tree) {
3209                 btrfs_info(fs_info, "clearing free space tree");
3210                 ret = btrfs_clear_free_space_tree(fs_info);
3211                 if (ret) {
3212                         btrfs_warn(fs_info,
3213                                    "failed to clear free space tree: %d", ret);
3214                         goto out;
3215                 }
3216         }
3217
3218         /*
3219          * btrfs_find_orphan_roots() is responsible for finding all the dead
3220          * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load
3221          * them into the fs_info->fs_roots_radix tree. This must be done before
3222          * calling btrfs_orphan_cleanup() on the tree root. If we don't do it
3223          * first, then btrfs_orphan_cleanup() will delete a dead root's orphan
3224          * item before the root's tree is deleted - this means that if we unmount
3225          * or crash before the deletion completes, on the next mount we will not
3226          * delete what remains of the tree because the orphan item does not
3227          * exists anymore, which is what tells us we have a pending deletion.
3228          */
3229         ret = btrfs_find_orphan_roots(fs_info);
3230         if (ret)
3231                 goto out;
3232
3233         ret = btrfs_cleanup_fs_roots(fs_info);
3234         if (ret)
3235                 goto out;
3236
3237         down_read(&fs_info->cleanup_work_sem);
3238         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3239             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3240                 up_read(&fs_info->cleanup_work_sem);
3241                 goto out;
3242         }
3243         up_read(&fs_info->cleanup_work_sem);
3244
3245         mutex_lock(&fs_info->cleaner_mutex);
3246         ret = btrfs_recover_relocation(fs_info);
3247         mutex_unlock(&fs_info->cleaner_mutex);
3248         if (ret < 0) {
3249                 btrfs_warn(fs_info, "failed to recover relocation: %d", ret);
3250                 goto out;
3251         }
3252
3253         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3254             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3255                 btrfs_info(fs_info, "creating free space tree");
3256                 ret = btrfs_create_free_space_tree(fs_info);
3257                 if (ret) {
3258                         btrfs_warn(fs_info,
3259                                 "failed to create free space tree: %d", ret);
3260                         goto out;
3261                 }
3262         }
3263
3264         if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) {
3265                 ret = btrfs_set_free_space_cache_v1_active(fs_info, cache_opt);
3266                 if (ret)
3267                         goto out;
3268         }
3269
3270         ret = btrfs_resume_balance_async(fs_info);
3271         if (ret)
3272                 goto out;
3273
3274         ret = btrfs_resume_dev_replace_async(fs_info);
3275         if (ret) {
3276                 btrfs_warn(fs_info, "failed to resume dev_replace");
3277                 goto out;
3278         }
3279
3280         btrfs_qgroup_rescan_resume(fs_info);
3281
3282         if (!fs_info->uuid_root) {
3283                 btrfs_info(fs_info, "creating UUID tree");
3284                 ret = btrfs_create_uuid_tree(fs_info);
3285                 if (ret) {
3286                         btrfs_warn(fs_info,
3287                                    "failed to create the UUID tree %d", ret);
3288                         goto out;
3289                 }
3290         }
3291
3292 out:
3293         return ret;
3294 }
3295
3296 /*
3297  * Do various sanity and dependency checks of different features.
3298  *
3299  * This is the place for less strict checks (like for subpage or artificial
3300  * feature dependencies).
3301  *
3302  * For strict checks or possible corruption detection, see
3303  * btrfs_validate_super().
3304  *
3305  * This should be called after btrfs_parse_options(), as some mount options
3306  * (space cache related) can modify on-disk format like free space tree and
3307  * screw up certain feature dependencies.
3308  */
3309 int btrfs_check_features(struct btrfs_fs_info *fs_info, struct super_block *sb)
3310 {
3311         struct btrfs_super_block *disk_super = fs_info->super_copy;
3312         u64 incompat = btrfs_super_incompat_flags(disk_super);
3313         const u64 compat_ro = btrfs_super_compat_ro_flags(disk_super);
3314         const u64 compat_ro_unsupp = (compat_ro & ~BTRFS_FEATURE_COMPAT_RO_SUPP);
3315
3316         if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
3317                 btrfs_err(fs_info,
3318                 "cannot mount because of unknown incompat features (0x%llx)",
3319                     incompat);
3320                 return -EINVAL;
3321         }
3322
3323         /* Runtime limitation for mixed block groups. */
3324         if ((incompat & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
3325             (fs_info->sectorsize != fs_info->nodesize)) {
3326                 btrfs_err(fs_info,
3327 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
3328                         fs_info->nodesize, fs_info->sectorsize);
3329                 return -EINVAL;
3330         }
3331
3332         /* Mixed backref is an always-enabled feature. */
3333         incompat |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
3334
3335         /* Set compression related flags just in case. */
3336         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
3337                 incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
3338         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
3339                 incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
3340
3341         /*
3342          * An ancient flag, which should really be marked deprecated.
3343          * Such runtime limitation doesn't really need a incompat flag.
3344          */
3345         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE)
3346                 incompat |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
3347
3348         if (compat_ro_unsupp && !sb_rdonly(sb)) {
3349                 btrfs_err(fs_info,
3350         "cannot mount read-write because of unknown compat_ro features (0x%llx)",
3351                        compat_ro);
3352                 return -EINVAL;
3353         }
3354
3355         /*
3356          * We have unsupported RO compat features, although RO mounted, we
3357          * should not cause any metadata writes, including log replay.
3358          * Or we could screw up whatever the new feature requires.
3359          */
3360         if (compat_ro_unsupp && btrfs_super_log_root(disk_super) &&
3361             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3362                 btrfs_err(fs_info,
3363 "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay",
3364                           compat_ro);
3365                 return -EINVAL;
3366         }
3367
3368         /*
3369          * Artificial limitations for block group tree, to force
3370          * block-group-tree to rely on no-holes and free-space-tree.
3371          */
3372         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
3373             (!btrfs_fs_incompat(fs_info, NO_HOLES) ||
3374              !btrfs_test_opt(fs_info, FREE_SPACE_TREE))) {
3375                 btrfs_err(fs_info,
3376 "block-group-tree feature requires no-holes and free-space-tree features");
3377                 return -EINVAL;
3378         }
3379
3380         /*
3381          * Subpage runtime limitation on v1 cache.
3382          *
3383          * V1 space cache still has some hard codeed PAGE_SIZE usage, while
3384          * we're already defaulting to v2 cache, no need to bother v1 as it's
3385          * going to be deprecated anyway.
3386          */
3387         if (fs_info->sectorsize < PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) {
3388                 btrfs_warn(fs_info,
3389         "v1 space cache is not supported for page size %lu with sectorsize %u",
3390                            PAGE_SIZE, fs_info->sectorsize);
3391                 return -EINVAL;
3392         }
3393
3394         /* This can be called by remount, we need to protect the super block. */
3395         spin_lock(&fs_info->super_lock);
3396         btrfs_set_super_incompat_flags(disk_super, incompat);
3397         spin_unlock(&fs_info->super_lock);
3398
3399         return 0;
3400 }
3401
3402 int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices,
3403                       char *options)
3404 {
3405         u32 sectorsize;
3406         u32 nodesize;
3407         u32 stripesize;
3408         u64 generation;
3409         u64 features;
3410         u16 csum_type;
3411         struct btrfs_super_block *disk_super;
3412         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
3413         struct btrfs_root *tree_root;
3414         struct btrfs_root *chunk_root;
3415         int ret;
3416         int err = -EINVAL;
3417         int level;
3418
3419         ret = init_mount_fs_info(fs_info, sb);
3420         if (ret) {
3421                 err = ret;
3422                 goto fail;
3423         }
3424
3425         /* These need to be init'ed before we start creating inodes and such. */
3426         tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID,
3427                                      GFP_KERNEL);
3428         fs_info->tree_root = tree_root;
3429         chunk_root = btrfs_alloc_root(fs_info, BTRFS_CHUNK_TREE_OBJECTID,
3430                                       GFP_KERNEL);
3431         fs_info->chunk_root = chunk_root;
3432         if (!tree_root || !chunk_root) {
3433                 err = -ENOMEM;
3434                 goto fail;
3435         }
3436
3437         fs_info->btree_inode = new_inode(sb);
3438         if (!fs_info->btree_inode) {
3439                 err = -ENOMEM;
3440                 goto fail;
3441         }
3442         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
3443         btrfs_init_btree_inode(fs_info);
3444
3445         invalidate_bdev(fs_devices->latest_dev->bdev);
3446
3447         /*
3448          * Read super block and check the signature bytes only
3449          */
3450         disk_super = btrfs_read_dev_super(fs_devices->latest_dev->bdev);
3451         if (IS_ERR(disk_super)) {
3452                 err = PTR_ERR(disk_super);
3453                 goto fail_alloc;
3454         }
3455
3456         /*
3457          * Verify the type first, if that or the checksum value are
3458          * corrupted, we'll find out
3459          */
3460         csum_type = btrfs_super_csum_type(disk_super);
3461         if (!btrfs_supported_super_csum(csum_type)) {
3462                 btrfs_err(fs_info, "unsupported checksum algorithm: %u",
3463                           csum_type);
3464                 err = -EINVAL;
3465                 btrfs_release_disk_super(disk_super);
3466                 goto fail_alloc;
3467         }
3468
3469         fs_info->csum_size = btrfs_super_csum_size(disk_super);
3470
3471         ret = btrfs_init_csum_hash(fs_info, csum_type);
3472         if (ret) {
3473                 err = ret;
3474                 btrfs_release_disk_super(disk_super);
3475                 goto fail_alloc;
3476         }
3477
3478         /*
3479          * We want to check superblock checksum, the type is stored inside.
3480          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
3481          */
3482         if (btrfs_check_super_csum(fs_info, (u8 *)disk_super)) {
3483                 btrfs_err(fs_info, "superblock checksum mismatch");
3484                 err = -EINVAL;
3485                 btrfs_release_disk_super(disk_super);
3486                 goto fail_alloc;
3487         }
3488
3489         /*
3490          * super_copy is zeroed at allocation time and we never touch the
3491          * following bytes up to INFO_SIZE, the checksum is calculated from
3492          * the whole block of INFO_SIZE
3493          */
3494         memcpy(fs_info->super_copy, disk_super, sizeof(*fs_info->super_copy));
3495         btrfs_release_disk_super(disk_super);
3496
3497         disk_super = fs_info->super_copy;
3498
3499
3500         features = btrfs_super_flags(disk_super);
3501         if (features & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
3502                 features &= ~BTRFS_SUPER_FLAG_CHANGING_FSID_V2;
3503                 btrfs_set_super_flags(disk_super, features);
3504                 btrfs_info(fs_info,
3505                         "found metadata UUID change in progress flag, clearing");
3506         }
3507
3508         memcpy(fs_info->super_for_commit, fs_info->super_copy,
3509                sizeof(*fs_info->super_for_commit));
3510
3511         ret = btrfs_validate_mount_super(fs_info);
3512         if (ret) {
3513                 btrfs_err(fs_info, "superblock contains fatal errors");
3514                 err = -EINVAL;
3515                 goto fail_alloc;
3516         }
3517
3518         if (!btrfs_super_root(disk_super))
3519                 goto fail_alloc;
3520
3521         /* check FS state, whether FS is broken. */
3522         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
3523                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
3524
3525         /*
3526          * In the long term, we'll store the compression type in the super
3527          * block, and it'll be used for per file compression control.
3528          */
3529         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
3530
3531
3532         /* Set up fs_info before parsing mount options */
3533         nodesize = btrfs_super_nodesize(disk_super);
3534         sectorsize = btrfs_super_sectorsize(disk_super);
3535         stripesize = sectorsize;
3536         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
3537         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
3538
3539         fs_info->nodesize = nodesize;
3540         fs_info->sectorsize = sectorsize;
3541         fs_info->sectorsize_bits = ilog2(sectorsize);
3542         fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size;
3543         fs_info->stripesize = stripesize;
3544
3545         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
3546         if (ret) {
3547                 err = ret;
3548                 goto fail_alloc;
3549         }
3550
3551         ret = btrfs_check_features(fs_info, sb);
3552         if (ret < 0) {
3553                 err = ret;
3554                 goto fail_alloc;
3555         }
3556
3557         if (sectorsize < PAGE_SIZE) {
3558                 struct btrfs_subpage_info *subpage_info;
3559
3560                 /*
3561                  * V1 space cache has some hardcoded PAGE_SIZE usage, and is
3562                  * going to be deprecated.
3563                  *
3564                  * Force to use v2 cache for subpage case.
3565                  */
3566                 btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
3567                 btrfs_set_and_info(fs_info, FREE_SPACE_TREE,
3568                         "forcing free space tree for sector size %u with page size %lu",
3569                         sectorsize, PAGE_SIZE);
3570
3571                 btrfs_warn(fs_info,
3572                 "read-write for sector size %u with page size %lu is experimental",
3573                            sectorsize, PAGE_SIZE);
3574                 subpage_info = kzalloc(sizeof(*subpage_info), GFP_KERNEL);
3575                 if (!subpage_info)
3576                         goto fail_alloc;
3577                 btrfs_init_subpage_info(subpage_info, sectorsize);
3578                 fs_info->subpage_info = subpage_info;
3579         }
3580
3581         ret = btrfs_init_workqueues(fs_info);
3582         if (ret) {
3583                 err = ret;
3584                 goto fail_sb_buffer;
3585         }
3586
3587         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
3588         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
3589
3590         sb->s_blocksize = sectorsize;
3591         sb->s_blocksize_bits = blksize_bits(sectorsize);
3592         memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE);
3593
3594         mutex_lock(&fs_info->chunk_mutex);
3595         ret = btrfs_read_sys_array(fs_info);
3596         mutex_unlock(&fs_info->chunk_mutex);
3597         if (ret) {
3598                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
3599                 goto fail_sb_buffer;
3600         }
3601
3602         generation = btrfs_super_chunk_root_generation(disk_super);
3603         level = btrfs_super_chunk_root_level(disk_super);
3604         ret = load_super_root(chunk_root, btrfs_super_chunk_root(disk_super),
3605                               generation, level);
3606         if (ret) {
3607                 btrfs_err(fs_info, "failed to read chunk root");
3608                 goto fail_tree_roots;
3609         }
3610
3611         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
3612                            offsetof(struct btrfs_header, chunk_tree_uuid),
3613                            BTRFS_UUID_SIZE);
3614
3615         ret = btrfs_read_chunk_tree(fs_info);
3616         if (ret) {
3617                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
3618                 goto fail_tree_roots;
3619         }
3620
3621         /*
3622          * At this point we know all the devices that make this filesystem,
3623          * including the seed devices but we don't know yet if the replace
3624          * target is required. So free devices that are not part of this
3625          * filesystem but skip the replace target device which is checked
3626          * below in btrfs_init_dev_replace().
3627          */
3628         btrfs_free_extra_devids(fs_devices);
3629         if (!fs_devices->latest_dev->bdev) {
3630                 btrfs_err(fs_info, "failed to read devices");
3631                 goto fail_tree_roots;
3632         }
3633
3634         ret = init_tree_roots(fs_info);
3635         if (ret)
3636                 goto fail_tree_roots;
3637
3638         /*
3639          * Get zone type information of zoned block devices. This will also
3640          * handle emulation of a zoned filesystem if a regular device has the
3641          * zoned incompat feature flag set.
3642          */
3643         ret = btrfs_get_dev_zone_info_all_devices(fs_info);
3644         if (ret) {
3645                 btrfs_err(fs_info,
3646                           "zoned: failed to read device zone info: %d",
3647                           ret);
3648                 goto fail_block_groups;
3649         }
3650
3651         /*
3652          * If we have a uuid root and we're not being told to rescan we need to
3653          * check the generation here so we can set the
3654          * BTRFS_FS_UPDATE_UUID_TREE_GEN bit.  Otherwise we could commit the
3655          * transaction during a balance or the log replay without updating the
3656          * uuid generation, and then if we crash we would rescan the uuid tree,
3657          * even though it was perfectly fine.
3658          */
3659         if (fs_info->uuid_root && !btrfs_test_opt(fs_info, RESCAN_UUID_TREE) &&
3660             fs_info->generation == btrfs_super_uuid_tree_generation(disk_super))
3661                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
3662
3663         ret = btrfs_verify_dev_extents(fs_info);
3664         if (ret) {
3665                 btrfs_err(fs_info,
3666                           "failed to verify dev extents against chunks: %d",
3667                           ret);
3668                 goto fail_block_groups;
3669         }
3670         ret = btrfs_recover_balance(fs_info);
3671         if (ret) {
3672                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3673                 goto fail_block_groups;
3674         }
3675
3676         ret = btrfs_init_dev_stats(fs_info);
3677         if (ret) {
3678                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3679                 goto fail_block_groups;
3680         }
3681
3682         ret = btrfs_init_dev_replace(fs_info);
3683         if (ret) {
3684                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3685                 goto fail_block_groups;
3686         }
3687
3688         ret = btrfs_check_zoned_mode(fs_info);
3689         if (ret) {
3690                 btrfs_err(fs_info, "failed to initialize zoned mode: %d",
3691                           ret);
3692                 goto fail_block_groups;
3693         }
3694
3695         ret = btrfs_sysfs_add_fsid(fs_devices);
3696         if (ret) {
3697                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3698                                 ret);
3699                 goto fail_block_groups;
3700         }
3701
3702         ret = btrfs_sysfs_add_mounted(fs_info);
3703         if (ret) {
3704                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3705                 goto fail_fsdev_sysfs;
3706         }
3707
3708         ret = btrfs_init_space_info(fs_info);
3709         if (ret) {
3710                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3711                 goto fail_sysfs;
3712         }
3713
3714         ret = btrfs_read_block_groups(fs_info);
3715         if (ret) {
3716                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3717                 goto fail_sysfs;
3718         }
3719
3720         btrfs_free_zone_cache(fs_info);
3721
3722         if (!sb_rdonly(sb) && fs_info->fs_devices->missing_devices &&
3723             !btrfs_check_rw_degradable(fs_info, NULL)) {
3724                 btrfs_warn(fs_info,
3725                 "writable mount is not allowed due to too many missing devices");
3726                 goto fail_sysfs;
3727         }
3728
3729         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, fs_info,
3730                                                "btrfs-cleaner");
3731         if (IS_ERR(fs_info->cleaner_kthread))
3732                 goto fail_sysfs;
3733
3734         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3735                                                    tree_root,
3736                                                    "btrfs-transaction");
3737         if (IS_ERR(fs_info->transaction_kthread))
3738                 goto fail_cleaner;
3739
3740         if (!btrfs_test_opt(fs_info, NOSSD) &&
3741             !fs_info->fs_devices->rotating) {
3742                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3743         }
3744
3745         /*
3746          * Mount does not set all options immediately, we can do it now and do
3747          * not have to wait for transaction commit
3748          */
3749         btrfs_apply_pending_changes(fs_info);
3750
3751 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3752         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3753                 ret = btrfsic_mount(fs_info, fs_devices,
3754                                     btrfs_test_opt(fs_info,
3755                                         CHECK_INTEGRITY_DATA) ? 1 : 0,
3756                                     fs_info->check_integrity_print_mask);
3757                 if (ret)
3758                         btrfs_warn(fs_info,
3759                                 "failed to initialize integrity check module: %d",
3760                                 ret);
3761         }
3762 #endif
3763         ret = btrfs_read_qgroup_config(fs_info);
3764         if (ret)
3765                 goto fail_trans_kthread;
3766
3767         if (btrfs_build_ref_tree(fs_info))
3768                 btrfs_err(fs_info, "couldn't build ref tree");
3769
3770         /* do not make disk changes in broken FS or nologreplay is given */
3771         if (btrfs_super_log_root(disk_super) != 0 &&
3772             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3773                 btrfs_info(fs_info, "start tree-log replay");
3774                 ret = btrfs_replay_log(fs_info, fs_devices);
3775                 if (ret) {
3776                         err = ret;
3777                         goto fail_qgroup;
3778                 }
3779         }
3780
3781         fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true);
3782         if (IS_ERR(fs_info->fs_root)) {
3783                 err = PTR_ERR(fs_info->fs_root);
3784                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3785                 fs_info->fs_root = NULL;
3786                 goto fail_qgroup;
3787         }
3788
3789         if (sb_rdonly(sb))
3790                 goto clear_oneshot;
3791
3792         ret = btrfs_start_pre_rw_mount(fs_info);
3793         if (ret) {
3794                 close_ctree(fs_info);
3795                 return ret;
3796         }
3797         btrfs_discard_resume(fs_info);
3798
3799         if (fs_info->uuid_root &&
3800             (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) ||
3801              fs_info->generation != btrfs_super_uuid_tree_generation(disk_super))) {
3802                 btrfs_info(fs_info, "checking UUID tree");
3803                 ret = btrfs_check_uuid_tree(fs_info);
3804                 if (ret) {
3805                         btrfs_warn(fs_info,
3806                                 "failed to check the UUID tree: %d", ret);
3807                         close_ctree(fs_info);
3808                         return ret;
3809                 }
3810         }
3811
3812         set_bit(BTRFS_FS_OPEN, &fs_info->flags);
3813
3814         /* Kick the cleaner thread so it'll start deleting snapshots. */
3815         if (test_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags))
3816                 wake_up_process(fs_info->cleaner_kthread);
3817
3818 clear_oneshot:
3819         btrfs_clear_oneshot_options(fs_info);
3820         return 0;
3821
3822 fail_qgroup:
3823         btrfs_free_qgroup_config(fs_info);
3824 fail_trans_kthread:
3825         kthread_stop(fs_info->transaction_kthread);
3826         btrfs_cleanup_transaction(fs_info);
3827         btrfs_free_fs_roots(fs_info);
3828 fail_cleaner:
3829         kthread_stop(fs_info->cleaner_kthread);
3830
3831         /*
3832          * make sure we're done with the btree inode before we stop our
3833          * kthreads
3834          */
3835         filemap_write_and_wait(fs_info->btree_inode->i_mapping);
3836
3837 fail_sysfs:
3838         btrfs_sysfs_remove_mounted(fs_info);
3839
3840 fail_fsdev_sysfs:
3841         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3842
3843 fail_block_groups:
3844         btrfs_put_block_group_cache(fs_info);
3845
3846 fail_tree_roots:
3847         if (fs_info->data_reloc_root)
3848                 btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root);
3849         free_root_pointers(fs_info, true);
3850         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
3851
3852 fail_sb_buffer:
3853         btrfs_stop_all_workers(fs_info);
3854         btrfs_free_block_groups(fs_info);
3855 fail_alloc:
3856         btrfs_mapping_tree_free(&fs_info->mapping_tree);
3857
3858         iput(fs_info->btree_inode);
3859 fail:
3860         btrfs_close_devices(fs_info->fs_devices);
3861         return err;
3862 }
3863 ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
3864
3865 static void btrfs_end_super_write(struct bio *bio)
3866 {
3867         struct btrfs_device *device = bio->bi_private;
3868         struct bio_vec *bvec;
3869         struct bvec_iter_all iter_all;
3870         struct page *page;
3871
3872         bio_for_each_segment_all(bvec, bio, iter_all) {
3873                 page = bvec->bv_page;
3874
3875                 if (bio->bi_status) {
3876                         btrfs_warn_rl_in_rcu(device->fs_info,
3877                                 "lost page write due to IO error on %s (%d)",
3878                                 rcu_str_deref(device->name),
3879                                 blk_status_to_errno(bio->bi_status));
3880                         ClearPageUptodate(page);
3881                         SetPageError(page);
3882                         btrfs_dev_stat_inc_and_print(device,
3883                                                      BTRFS_DEV_STAT_WRITE_ERRS);
3884                 } else {
3885                         SetPageUptodate(page);
3886                 }
3887
3888                 put_page(page);
3889                 unlock_page(page);
3890         }
3891
3892         bio_put(bio);
3893 }
3894
3895 struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev,
3896                                                    int copy_num, bool drop_cache)
3897 {
3898         struct btrfs_super_block *super;
3899         struct page *page;
3900         u64 bytenr, bytenr_orig;
3901         struct address_space *mapping = bdev->bd_inode->i_mapping;
3902         int ret;
3903
3904         bytenr_orig = btrfs_sb_offset(copy_num);
3905         ret = btrfs_sb_log_location_bdev(bdev, copy_num, READ, &bytenr);
3906         if (ret == -ENOENT)
3907                 return ERR_PTR(-EINVAL);
3908         else if (ret)
3909                 return ERR_PTR(ret);
3910
3911         if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev))
3912                 return ERR_PTR(-EINVAL);
3913
3914         if (drop_cache) {
3915                 /* This should only be called with the primary sb. */
3916                 ASSERT(copy_num == 0);
3917
3918                 /*
3919                  * Drop the page of the primary superblock, so later read will
3920                  * always read from the device.
3921                  */
3922                 invalidate_inode_pages2_range(mapping,
3923                                 bytenr >> PAGE_SHIFT,
3924                                 (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT);
3925         }
3926
3927         page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS);
3928         if (IS_ERR(page))
3929                 return ERR_CAST(page);
3930
3931         super = page_address(page);
3932         if (btrfs_super_magic(super) != BTRFS_MAGIC) {
3933                 btrfs_release_disk_super(super);
3934                 return ERR_PTR(-ENODATA);
3935         }
3936
3937         if (btrfs_super_bytenr(super) != bytenr_orig) {
3938                 btrfs_release_disk_super(super);
3939                 return ERR_PTR(-EINVAL);
3940         }
3941
3942         return super;
3943 }
3944
3945
3946 struct btrfs_super_block *btrfs_read_dev_super(struct block_device *bdev)
3947 {
3948         struct btrfs_super_block *super, *latest = NULL;
3949         int i;
3950         u64 transid = 0;
3951
3952         /* we would like to check all the supers, but that would make
3953          * a btrfs mount succeed after a mkfs from a different FS.
3954          * So, we need to add a special mount option to scan for
3955          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
3956          */
3957         for (i = 0; i < 1; i++) {
3958                 super = btrfs_read_dev_one_super(bdev, i, false);
3959                 if (IS_ERR(super))
3960                         continue;
3961
3962                 if (!latest || btrfs_super_generation(super) > transid) {
3963                         if (latest)
3964                                 btrfs_release_disk_super(super);
3965
3966                         latest = super;
3967                         transid = btrfs_super_generation(super);
3968                 }
3969         }
3970
3971         return super;
3972 }
3973
3974 /*
3975  * Write superblock @sb to the @device. Do not wait for completion, all the
3976  * pages we use for writing are locked.
3977  *
3978  * Write @max_mirrors copies of the superblock, where 0 means default that fit
3979  * the expected device size at commit time. Note that max_mirrors must be
3980  * same for write and wait phases.
3981  *
3982  * Return number of errors when page is not found or submission fails.
3983  */
3984 static int write_dev_supers(struct btrfs_device *device,
3985                             struct btrfs_super_block *sb, int max_mirrors)
3986 {
3987         struct btrfs_fs_info *fs_info = device->fs_info;
3988         struct address_space *mapping = device->bdev->bd_inode->i_mapping;
3989         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3990         int i;
3991         int errors = 0;
3992         int ret;
3993         u64 bytenr, bytenr_orig;
3994
3995         if (max_mirrors == 0)
3996                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3997
3998         shash->tfm = fs_info->csum_shash;
3999
4000         for (i = 0; i < max_mirrors; i++) {
4001                 struct page *page;
4002                 struct bio *bio;
4003                 struct btrfs_super_block *disk_super;
4004
4005                 bytenr_orig = btrfs_sb_offset(i);
4006                 ret = btrfs_sb_log_location(device, i, WRITE, &bytenr);
4007                 if (ret == -ENOENT) {
4008                         continue;
4009                 } else if (ret < 0) {
4010                         btrfs_err(device->fs_info,
4011                                 "couldn't get super block location for mirror %d",
4012                                 i);
4013                         errors++;
4014                         continue;
4015                 }
4016                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
4017                     device->commit_total_bytes)
4018                         break;
4019
4020                 btrfs_set_super_bytenr(sb, bytenr_orig);
4021
4022                 crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE,
4023                                     BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE,
4024                                     sb->csum);
4025
4026                 page = find_or_create_page(mapping, bytenr >> PAGE_SHIFT,
4027                                            GFP_NOFS);
4028                 if (!page) {
4029                         btrfs_err(device->fs_info,
4030                             "couldn't get super block page for bytenr %llu",
4031                             bytenr);
4032                         errors++;
4033                         continue;
4034                 }
4035
4036                 /* Bump the refcount for wait_dev_supers() */
4037                 get_page(page);
4038
4039                 disk_super = page_address(page);
4040                 memcpy(disk_super, sb, BTRFS_SUPER_INFO_SIZE);
4041
4042                 /*
4043                  * Directly use bios here instead of relying on the page cache
4044                  * to do I/O, so we don't lose the ability to do integrity
4045                  * checking.
4046                  */
4047                 bio = bio_alloc(device->bdev, 1,
4048                                 REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO,
4049                                 GFP_NOFS);
4050                 bio->bi_iter.bi_sector = bytenr >> SECTOR_SHIFT;
4051                 bio->bi_private = device;
4052                 bio->bi_end_io = btrfs_end_super_write;
4053                 __bio_add_page(bio, page, BTRFS_SUPER_INFO_SIZE,
4054                                offset_in_page(bytenr));
4055
4056                 /*
4057                  * We FUA only the first super block.  The others we allow to
4058                  * go down lazy and there's a short window where the on-disk
4059                  * copies might still contain the older version.
4060                  */
4061                 if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER))
4062                         bio->bi_opf |= REQ_FUA;
4063
4064                 btrfsic_check_bio(bio);
4065                 submit_bio(bio);
4066
4067                 if (btrfs_advance_sb_log(device, i))
4068                         errors++;
4069         }
4070         return errors < i ? 0 : -1;
4071 }
4072
4073 /*
4074  * Wait for write completion of superblocks done by write_dev_supers,
4075  * @max_mirrors same for write and wait phases.
4076  *
4077  * Return number of errors when page is not found or not marked up to
4078  * date.
4079  */
4080 static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
4081 {
4082         int i;
4083         int errors = 0;
4084         bool primary_failed = false;
4085         int ret;
4086         u64 bytenr;
4087
4088         if (max_mirrors == 0)
4089                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
4090
4091         for (i = 0; i < max_mirrors; i++) {
4092                 struct page *page;
4093
4094                 ret = btrfs_sb_log_location(device, i, READ, &bytenr);
4095                 if (ret == -ENOENT) {
4096                         break;
4097                 } else if (ret < 0) {
4098                         errors++;
4099                         if (i == 0)
4100                                 primary_failed = true;
4101                         continue;
4102                 }
4103                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
4104                     device->commit_total_bytes)
4105                         break;
4106
4107                 page = find_get_page(device->bdev->bd_inode->i_mapping,
4108                                      bytenr >> PAGE_SHIFT);
4109                 if (!page) {
4110                         errors++;
4111                         if (i == 0)
4112                                 primary_failed = true;
4113                         continue;
4114                 }
4115                 /* Page is submitted locked and unlocked once the IO completes */
4116                 wait_on_page_locked(page);
4117                 if (PageError(page)) {
4118                         errors++;
4119                         if (i == 0)
4120                                 primary_failed = true;
4121                 }
4122
4123                 /* Drop our reference */
4124                 put_page(page);
4125
4126                 /* Drop the reference from the writing run */
4127                 put_page(page);
4128         }
4129
4130         /* log error, force error return */
4131         if (primary_failed) {
4132                 btrfs_err(device->fs_info, "error writing primary super block to device %llu",
4133                           device->devid);
4134                 return -1;
4135         }
4136
4137         return errors < i ? 0 : -1;
4138 }
4139
4140 /*
4141  * endio for the write_dev_flush, this will wake anyone waiting
4142  * for the barrier when it is done
4143  */
4144 static void btrfs_end_empty_barrier(struct bio *bio)
4145 {
4146         bio_uninit(bio);
4147         complete(bio->bi_private);
4148 }
4149
4150 /*
4151  * Submit a flush request to the device if it supports it. Error handling is
4152  * done in the waiting counterpart.
4153  */
4154 static void write_dev_flush(struct btrfs_device *device)
4155 {
4156         struct bio *bio = &device->flush_bio;
4157
4158 #ifndef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4159         /*
4160          * When a disk has write caching disabled, we skip submission of a bio
4161          * with flush and sync requests before writing the superblock, since
4162          * it's not needed. However when the integrity checker is enabled, this
4163          * results in reports that there are metadata blocks referred by a
4164          * superblock that were not properly flushed. So don't skip the bio
4165          * submission only when the integrity checker is enabled for the sake
4166          * of simplicity, since this is a debug tool and not meant for use in
4167          * non-debug builds.
4168          */
4169         if (!bdev_write_cache(device->bdev))
4170                 return;
4171 #endif
4172
4173         bio_init(bio, device->bdev, NULL, 0,
4174                  REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH);
4175         bio->bi_end_io = btrfs_end_empty_barrier;
4176         init_completion(&device->flush_wait);
4177         bio->bi_private = &device->flush_wait;
4178
4179         btrfsic_check_bio(bio);
4180         submit_bio(bio);
4181         set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4182 }
4183
4184 /*
4185  * If the flush bio has been submitted by write_dev_flush, wait for it.
4186  */
4187 static blk_status_t wait_dev_flush(struct btrfs_device *device)
4188 {
4189         struct bio *bio = &device->flush_bio;
4190
4191         if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
4192                 return BLK_STS_OK;
4193
4194         clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4195         wait_for_completion_io(&device->flush_wait);
4196
4197         return bio->bi_status;
4198 }
4199
4200 static int check_barrier_error(struct btrfs_fs_info *fs_info)
4201 {
4202         if (!btrfs_check_rw_degradable(fs_info, NULL))
4203                 return -EIO;
4204         return 0;
4205 }
4206
4207 /*
4208  * send an empty flush down to each device in parallel,
4209  * then wait for them
4210  */
4211 static int barrier_all_devices(struct btrfs_fs_info *info)
4212 {
4213         struct list_head *head;
4214         struct btrfs_device *dev;
4215         int errors_wait = 0;
4216         blk_status_t ret;
4217
4218         lockdep_assert_held(&info->fs_devices->device_list_mutex);
4219         /* send down all the barriers */
4220         head = &info->fs_devices->devices;
4221         list_for_each_entry(dev, head, dev_list) {
4222                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4223                         continue;
4224                 if (!dev->bdev)
4225                         continue;
4226                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4227                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4228                         continue;
4229
4230                 write_dev_flush(dev);
4231                 dev->last_flush_error = BLK_STS_OK;
4232         }
4233
4234         /* wait for all the barriers */
4235         list_for_each_entry(dev, head, dev_list) {
4236                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4237                         continue;
4238                 if (!dev->bdev) {
4239                         errors_wait++;
4240                         continue;
4241                 }
4242                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4243                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4244                         continue;
4245
4246                 ret = wait_dev_flush(dev);
4247                 if (ret) {
4248                         dev->last_flush_error = ret;
4249                         btrfs_dev_stat_inc_and_print(dev,
4250                                         BTRFS_DEV_STAT_FLUSH_ERRS);
4251                         errors_wait++;
4252                 }
4253         }
4254
4255         if (errors_wait) {
4256                 /*
4257                  * At some point we need the status of all disks
4258                  * to arrive at the volume status. So error checking
4259                  * is being pushed to a separate loop.
4260                  */
4261                 return check_barrier_error(info);
4262         }
4263         return 0;
4264 }
4265
4266 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
4267 {
4268         int raid_type;
4269         int min_tolerated = INT_MAX;
4270
4271         if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 ||
4272             (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE))
4273                 min_tolerated = min_t(int, min_tolerated,
4274                                     btrfs_raid_array[BTRFS_RAID_SINGLE].
4275                                     tolerated_failures);
4276
4277         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
4278                 if (raid_type == BTRFS_RAID_SINGLE)
4279                         continue;
4280                 if (!(flags & btrfs_raid_array[raid_type].bg_flag))
4281                         continue;
4282                 min_tolerated = min_t(int, min_tolerated,
4283                                     btrfs_raid_array[raid_type].
4284                                     tolerated_failures);
4285         }
4286
4287         if (min_tolerated == INT_MAX) {
4288                 pr_warn("BTRFS: unknown raid flag: %llu", flags);
4289                 min_tolerated = 0;
4290         }
4291
4292         return min_tolerated;
4293 }
4294
4295 int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
4296 {
4297         struct list_head *head;
4298         struct btrfs_device *dev;
4299         struct btrfs_super_block *sb;
4300         struct btrfs_dev_item *dev_item;
4301         int ret;
4302         int do_barriers;
4303         int max_errors;
4304         int total_errors = 0;
4305         u64 flags;
4306
4307         do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
4308
4309         /*
4310          * max_mirrors == 0 indicates we're from commit_transaction,
4311          * not from fsync where the tree roots in fs_info have not
4312          * been consistent on disk.
4313          */
4314         if (max_mirrors == 0)
4315                 backup_super_roots(fs_info);
4316
4317         sb = fs_info->super_for_commit;
4318         dev_item = &sb->dev_item;
4319
4320         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4321         head = &fs_info->fs_devices->devices;
4322         max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1;
4323
4324         if (do_barriers) {
4325                 ret = barrier_all_devices(fs_info);
4326                 if (ret) {
4327                         mutex_unlock(
4328                                 &fs_info->fs_devices->device_list_mutex);
4329                         btrfs_handle_fs_error(fs_info, ret,
4330                                               "errors while submitting device barriers.");
4331                         return ret;
4332                 }
4333         }
4334
4335         list_for_each_entry(dev, head, dev_list) {
4336                 if (!dev->bdev) {
4337                         total_errors++;
4338                         continue;
4339                 }
4340                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4341                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4342                         continue;
4343
4344                 btrfs_set_stack_device_generation(dev_item, 0);
4345                 btrfs_set_stack_device_type(dev_item, dev->type);
4346                 btrfs_set_stack_device_id(dev_item, dev->devid);
4347                 btrfs_set_stack_device_total_bytes(dev_item,
4348                                                    dev->commit_total_bytes);
4349                 btrfs_set_stack_device_bytes_used(dev_item,
4350                                                   dev->commit_bytes_used);
4351                 btrfs_set_stack_device_io_align(dev_item, dev->io_align);
4352                 btrfs_set_stack_device_io_width(dev_item, dev->io_width);
4353                 btrfs_set_stack_device_sector_size(dev_item, dev->sector_size);
4354                 memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE);
4355                 memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid,
4356                        BTRFS_FSID_SIZE);
4357
4358                 flags = btrfs_super_flags(sb);
4359                 btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
4360
4361                 ret = btrfs_validate_write_super(fs_info, sb);
4362                 if (ret < 0) {
4363                         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4364                         btrfs_handle_fs_error(fs_info, -EUCLEAN,
4365                                 "unexpected superblock corruption detected");
4366                         return -EUCLEAN;
4367                 }
4368
4369                 ret = write_dev_supers(dev, sb, max_mirrors);
4370                 if (ret)
4371                         total_errors++;
4372         }
4373         if (total_errors > max_errors) {
4374                 btrfs_err(fs_info, "%d errors while writing supers",
4375                           total_errors);
4376                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4377
4378                 /* FUA is masked off if unsupported and can't be the reason */
4379                 btrfs_handle_fs_error(fs_info, -EIO,
4380                                       "%d errors while writing supers",
4381                                       total_errors);
4382                 return -EIO;
4383         }
4384
4385         total_errors = 0;
4386         list_for_each_entry(dev, head, dev_list) {
4387                 if (!dev->bdev)
4388                         continue;
4389                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4390                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4391                         continue;
4392
4393                 ret = wait_dev_supers(dev, max_mirrors);
4394                 if (ret)
4395                         total_errors++;
4396         }
4397         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4398         if (total_errors > max_errors) {
4399                 btrfs_handle_fs_error(fs_info, -EIO,
4400                                       "%d errors while writing supers",
4401                                       total_errors);
4402                 return -EIO;
4403         }
4404         return 0;
4405 }
4406
4407 /* Drop a fs root from the radix tree and free it. */
4408 void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
4409                                   struct btrfs_root *root)
4410 {
4411         bool drop_ref = false;
4412
4413         spin_lock(&fs_info->fs_roots_radix_lock);
4414         radix_tree_delete(&fs_info->fs_roots_radix,
4415                           (unsigned long)root->root_key.objectid);
4416         if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state))
4417                 drop_ref = true;
4418         spin_unlock(&fs_info->fs_roots_radix_lock);
4419
4420         if (BTRFS_FS_ERROR(fs_info)) {
4421                 ASSERT(root->log_root == NULL);
4422                 if (root->reloc_root) {
4423                         btrfs_put_root(root->reloc_root);
4424                         root->reloc_root = NULL;
4425                 }
4426         }
4427
4428         if (drop_ref)
4429                 btrfs_put_root(root);
4430 }
4431
4432 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
4433 {
4434         u64 root_objectid = 0;
4435         struct btrfs_root *gang[8];
4436         int i = 0;
4437         int err = 0;
4438         unsigned int ret = 0;
4439
4440         while (1) {
4441                 spin_lock(&fs_info->fs_roots_radix_lock);
4442                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4443                                              (void **)gang, root_objectid,
4444                                              ARRAY_SIZE(gang));
4445                 if (!ret) {
4446                         spin_unlock(&fs_info->fs_roots_radix_lock);
4447                         break;
4448                 }
4449                 root_objectid = gang[ret - 1]->root_key.objectid + 1;
4450
4451                 for (i = 0; i < ret; i++) {
4452                         /* Avoid to grab roots in dead_roots */
4453                         if (btrfs_root_refs(&gang[i]->root_item) == 0) {
4454                                 gang[i] = NULL;
4455                                 continue;
4456                         }
4457                         /* grab all the search result for later use */
4458                         gang[i] = btrfs_grab_root(gang[i]);
4459                 }
4460                 spin_unlock(&fs_info->fs_roots_radix_lock);
4461
4462                 for (i = 0; i < ret; i++) {
4463                         if (!gang[i])
4464                                 continue;
4465                         root_objectid = gang[i]->root_key.objectid;
4466                         err = btrfs_orphan_cleanup(gang[i]);
4467                         if (err)
4468                                 break;
4469                         btrfs_put_root(gang[i]);
4470                 }
4471                 root_objectid++;
4472         }
4473
4474         /* release the uncleaned roots due to error */
4475         for (; i < ret; i++) {
4476                 if (gang[i])
4477                         btrfs_put_root(gang[i]);
4478         }
4479         return err;
4480 }
4481
4482 int btrfs_commit_super(struct btrfs_fs_info *fs_info)
4483 {
4484         struct btrfs_root *root = fs_info->tree_root;
4485         struct btrfs_trans_handle *trans;
4486
4487         mutex_lock(&fs_info->cleaner_mutex);
4488         btrfs_run_delayed_iputs(fs_info);
4489         mutex_unlock(&fs_info->cleaner_mutex);
4490         wake_up_process(fs_info->cleaner_kthread);
4491
4492         /* wait until ongoing cleanup work done */
4493         down_write(&fs_info->cleanup_work_sem);
4494         up_write(&fs_info->cleanup_work_sem);
4495
4496         trans = btrfs_join_transaction(root);
4497         if (IS_ERR(trans))
4498                 return PTR_ERR(trans);
4499         return btrfs_commit_transaction(trans);
4500 }
4501
4502 static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info)
4503 {
4504         struct btrfs_transaction *trans;
4505         struct btrfs_transaction *tmp;
4506         bool found = false;
4507
4508         if (list_empty(&fs_info->trans_list))
4509                 return;
4510
4511         /*
4512          * This function is only called at the very end of close_ctree(),
4513          * thus no other running transaction, no need to take trans_lock.
4514          */
4515         ASSERT(test_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags));
4516         list_for_each_entry_safe(trans, tmp, &fs_info->trans_list, list) {
4517                 struct extent_state *cached = NULL;
4518                 u64 dirty_bytes = 0;
4519                 u64 cur = 0;
4520                 u64 found_start;
4521                 u64 found_end;
4522
4523                 found = true;
4524                 while (!find_first_extent_bit(&trans->dirty_pages, cur,
4525                         &found_start, &found_end, EXTENT_DIRTY, &cached)) {
4526                         dirty_bytes += found_end + 1 - found_start;
4527                         cur = found_end + 1;
4528                 }
4529                 btrfs_warn(fs_info,
4530         "transaction %llu (with %llu dirty metadata bytes) is not committed",
4531                            trans->transid, dirty_bytes);
4532                 btrfs_cleanup_one_transaction(trans, fs_info);
4533
4534                 if (trans == fs_info->running_transaction)
4535                         fs_info->running_transaction = NULL;
4536                 list_del_init(&trans->list);
4537
4538                 btrfs_put_transaction(trans);
4539                 trace_btrfs_transaction_commit(fs_info);
4540         }
4541         ASSERT(!found);
4542 }
4543
4544 void __cold close_ctree(struct btrfs_fs_info *fs_info)
4545 {
4546         int ret;
4547
4548         set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags);
4549
4550         /*
4551          * If we had UNFINISHED_DROPS we could still be processing them, so
4552          * clear that bit and wake up relocation so it can stop.
4553          * We must do this before stopping the block group reclaim task, because
4554          * at btrfs_relocate_block_group() we wait for this bit, and after the
4555          * wait we stop with -EINTR if btrfs_fs_closing() returns non-zero - we
4556          * have just set BTRFS_FS_CLOSING_START, so btrfs_fs_closing() will
4557          * return 1.
4558          */
4559         btrfs_wake_unfinished_drop(fs_info);
4560
4561         /*
4562          * We may have the reclaim task running and relocating a data block group,
4563          * in which case it may create delayed iputs. So stop it before we park
4564          * the cleaner kthread otherwise we can get new delayed iputs after
4565          * parking the cleaner, and that can make the async reclaim task to hang
4566          * if it's waiting for delayed iputs to complete, since the cleaner is
4567          * parked and can not run delayed iputs - this will make us hang when
4568          * trying to stop the async reclaim task.
4569          */
4570         cancel_work_sync(&fs_info->reclaim_bgs_work);
4571         /*
4572          * We don't want the cleaner to start new transactions, add more delayed
4573          * iputs, etc. while we're closing. We can't use kthread_stop() yet
4574          * because that frees the task_struct, and the transaction kthread might
4575          * still try to wake up the cleaner.
4576          */
4577         kthread_park(fs_info->cleaner_kthread);
4578
4579         /* wait for the qgroup rescan worker to stop */
4580         btrfs_qgroup_wait_for_completion(fs_info, false);
4581
4582         /* wait for the uuid_scan task to finish */
4583         down(&fs_info->uuid_tree_rescan_sem);
4584         /* avoid complains from lockdep et al., set sem back to initial state */
4585         up(&fs_info->uuid_tree_rescan_sem);
4586
4587         /* pause restriper - we want to resume on mount */
4588         btrfs_pause_balance(fs_info);
4589
4590         btrfs_dev_replace_suspend_for_unmount(fs_info);
4591
4592         btrfs_scrub_cancel(fs_info);
4593
4594         /* wait for any defraggers to finish */
4595         wait_event(fs_info->transaction_wait,
4596                    (atomic_read(&fs_info->defrag_running) == 0));
4597
4598         /* clear out the rbtree of defraggable inodes */
4599         btrfs_cleanup_defrag_inodes(fs_info);
4600
4601         /*
4602          * After we parked the cleaner kthread, ordered extents may have
4603          * completed and created new delayed iputs. If one of the async reclaim
4604          * tasks is running and in the RUN_DELAYED_IPUTS flush state, then we
4605          * can hang forever trying to stop it, because if a delayed iput is
4606          * added after it ran btrfs_run_delayed_iputs() and before it called
4607          * btrfs_wait_on_delayed_iputs(), it will hang forever since there is
4608          * no one else to run iputs.
4609          *
4610          * So wait for all ongoing ordered extents to complete and then run
4611          * delayed iputs. This works because once we reach this point no one
4612          * can either create new ordered extents nor create delayed iputs
4613          * through some other means.
4614          *
4615          * Also note that btrfs_wait_ordered_roots() is not safe here, because
4616          * it waits for BTRFS_ORDERED_COMPLETE to be set on an ordered extent,
4617          * but the delayed iput for the respective inode is made only when doing
4618          * the final btrfs_put_ordered_extent() (which must happen at
4619          * btrfs_finish_ordered_io() when we are unmounting).
4620          */
4621         btrfs_flush_workqueue(fs_info->endio_write_workers);
4622         /* Ordered extents for free space inodes. */
4623         btrfs_flush_workqueue(fs_info->endio_freespace_worker);
4624         btrfs_run_delayed_iputs(fs_info);
4625
4626         cancel_work_sync(&fs_info->async_reclaim_work);
4627         cancel_work_sync(&fs_info->async_data_reclaim_work);
4628         cancel_work_sync(&fs_info->preempt_reclaim_work);
4629
4630         /* Cancel or finish ongoing discard work */
4631         btrfs_discard_cleanup(fs_info);
4632
4633         if (!sb_rdonly(fs_info->sb)) {
4634                 /*
4635                  * The cleaner kthread is stopped, so do one final pass over
4636                  * unused block groups.
4637                  */
4638                 btrfs_delete_unused_bgs(fs_info);
4639
4640                 /*
4641                  * There might be existing delayed inode workers still running
4642                  * and holding an empty delayed inode item. We must wait for
4643                  * them to complete first because they can create a transaction.
4644                  * This happens when someone calls btrfs_balance_delayed_items()
4645                  * and then a transaction commit runs the same delayed nodes
4646                  * before any delayed worker has done something with the nodes.
4647                  * We must wait for any worker here and not at transaction
4648                  * commit time since that could cause a deadlock.
4649                  * This is a very rare case.
4650                  */
4651                 btrfs_flush_workqueue(fs_info->delayed_workers);
4652
4653                 ret = btrfs_commit_super(fs_info);
4654                 if (ret)
4655                         btrfs_err(fs_info, "commit super ret %d", ret);
4656         }
4657
4658         if (BTRFS_FS_ERROR(fs_info))
4659                 btrfs_error_commit_super(fs_info);
4660
4661         kthread_stop(fs_info->transaction_kthread);
4662         kthread_stop(fs_info->cleaner_kthread);
4663
4664         ASSERT(list_empty(&fs_info->delayed_iputs));
4665         set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags);
4666
4667         if (btrfs_check_quota_leak(fs_info)) {
4668                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4669                 btrfs_err(fs_info, "qgroup reserved space leaked");
4670         }
4671
4672         btrfs_free_qgroup_config(fs_info);
4673         ASSERT(list_empty(&fs_info->delalloc_roots));
4674
4675         if (percpu_counter_sum(&fs_info->delalloc_bytes)) {
4676                 btrfs_info(fs_info, "at unmount delalloc count %lld",
4677                        percpu_counter_sum(&fs_info->delalloc_bytes));
4678         }
4679
4680         if (percpu_counter_sum(&fs_info->ordered_bytes))
4681                 btrfs_info(fs_info, "at unmount dio bytes count %lld",
4682                            percpu_counter_sum(&fs_info->ordered_bytes));
4683
4684         btrfs_sysfs_remove_mounted(fs_info);
4685         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
4686
4687         btrfs_put_block_group_cache(fs_info);
4688
4689         /*
4690          * we must make sure there is not any read request to
4691          * submit after we stopping all workers.
4692          */
4693         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
4694         btrfs_stop_all_workers(fs_info);
4695
4696         /* We shouldn't have any transaction open at this point */
4697         warn_about_uncommitted_trans(fs_info);
4698
4699         clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
4700         free_root_pointers(fs_info, true);
4701         btrfs_free_fs_roots(fs_info);
4702
4703         /*
4704          * We must free the block groups after dropping the fs_roots as we could
4705          * have had an IO error and have left over tree log blocks that aren't
4706          * cleaned up until the fs roots are freed.  This makes the block group
4707          * accounting appear to be wrong because there's pending reserved bytes,
4708          * so make sure we do the block group cleanup afterwards.
4709          */
4710         btrfs_free_block_groups(fs_info);
4711
4712         iput(fs_info->btree_inode);
4713
4714 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4715         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY))
4716                 btrfsic_unmount(fs_info->fs_devices);
4717 #endif
4718
4719         btrfs_mapping_tree_free(&fs_info->mapping_tree);
4720         btrfs_close_devices(fs_info->fs_devices);
4721 }
4722
4723 int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
4724                           int atomic)
4725 {
4726         int ret;
4727         struct inode *btree_inode = buf->pages[0]->mapping->host;
4728
4729         ret = extent_buffer_uptodate(buf);
4730         if (!ret)
4731                 return ret;
4732
4733         ret = verify_parent_transid(&BTRFS_I(btree_inode)->io_tree, buf,
4734                                     parent_transid, atomic);
4735         if (ret == -EAGAIN)
4736                 return ret;
4737         return !ret;
4738 }
4739
4740 void btrfs_mark_buffer_dirty(struct extent_buffer *buf)
4741 {
4742         struct btrfs_fs_info *fs_info = buf->fs_info;
4743         u64 transid = btrfs_header_generation(buf);
4744         int was_dirty;
4745
4746 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4747         /*
4748          * This is a fast path so only do this check if we have sanity tests
4749          * enabled.  Normal people shouldn't be using unmapped buffers as dirty
4750          * outside of the sanity tests.
4751          */
4752         if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags)))
4753                 return;
4754 #endif
4755         btrfs_assert_tree_write_locked(buf);
4756         if (transid != fs_info->generation)
4757                 WARN(1, KERN_CRIT "btrfs transid mismatch buffer %llu, found %llu running %llu\n",
4758                         buf->start, transid, fs_info->generation);
4759         was_dirty = set_extent_buffer_dirty(buf);
4760         if (!was_dirty)
4761                 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4762                                          buf->len,
4763                                          fs_info->dirty_metadata_batch);
4764 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4765         /*
4766          * Since btrfs_mark_buffer_dirty() can be called with item pointer set
4767          * but item data not updated.
4768          * So here we should only check item pointers, not item data.
4769          */
4770         if (btrfs_header_level(buf) == 0 &&
4771             btrfs_check_leaf_relaxed(buf)) {
4772                 btrfs_print_leaf(buf);
4773                 ASSERT(0);
4774         }
4775 #endif
4776 }
4777
4778 static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info,
4779                                         int flush_delayed)
4780 {
4781         /*
4782          * looks as though older kernels can get into trouble with
4783          * this code, they end up stuck in balance_dirty_pages forever
4784          */
4785         int ret;
4786
4787         if (current->flags & PF_MEMALLOC)
4788                 return;
4789
4790         if (flush_delayed)
4791                 btrfs_balance_delayed_items(fs_info);
4792
4793         ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
4794                                      BTRFS_DIRTY_METADATA_THRESH,
4795                                      fs_info->dirty_metadata_batch);
4796         if (ret > 0) {
4797                 balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping);
4798         }
4799 }
4800
4801 void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info)
4802 {
4803         __btrfs_btree_balance_dirty(fs_info, 1);
4804 }
4805
4806 void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info)
4807 {
4808         __btrfs_btree_balance_dirty(fs_info, 0);
4809 }
4810
4811 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info)
4812 {
4813         /* cleanup FS via transaction */
4814         btrfs_cleanup_transaction(fs_info);
4815
4816         mutex_lock(&fs_info->cleaner_mutex);
4817         btrfs_run_delayed_iputs(fs_info);
4818         mutex_unlock(&fs_info->cleaner_mutex);
4819
4820         down_write(&fs_info->cleanup_work_sem);
4821         up_write(&fs_info->cleanup_work_sem);
4822 }
4823
4824 static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info)
4825 {
4826         struct btrfs_root *gang[8];
4827         u64 root_objectid = 0;
4828         int ret;
4829
4830         spin_lock(&fs_info->fs_roots_radix_lock);
4831         while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4832                                              (void **)gang, root_objectid,
4833                                              ARRAY_SIZE(gang))) != 0) {
4834                 int i;
4835
4836                 for (i = 0; i < ret; i++)
4837                         gang[i] = btrfs_grab_root(gang[i]);
4838                 spin_unlock(&fs_info->fs_roots_radix_lock);
4839
4840                 for (i = 0; i < ret; i++) {
4841                         if (!gang[i])
4842                                 continue;
4843                         root_objectid = gang[i]->root_key.objectid;
4844                         btrfs_free_log(NULL, gang[i]);
4845                         btrfs_put_root(gang[i]);
4846                 }
4847                 root_objectid++;
4848                 spin_lock(&fs_info->fs_roots_radix_lock);
4849         }
4850         spin_unlock(&fs_info->fs_roots_radix_lock);
4851         btrfs_free_log_root_tree(NULL, fs_info);
4852 }
4853
4854 static void btrfs_destroy_ordered_extents(struct btrfs_root *root)
4855 {
4856         struct btrfs_ordered_extent *ordered;
4857
4858         spin_lock(&root->ordered_extent_lock);
4859         /*
4860          * This will just short circuit the ordered completion stuff which will
4861          * make sure the ordered extent gets properly cleaned up.
4862          */
4863         list_for_each_entry(ordered, &root->ordered_extents,
4864                             root_extent_list)
4865                 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
4866         spin_unlock(&root->ordered_extent_lock);
4867 }
4868
4869 static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info)
4870 {
4871         struct btrfs_root *root;
4872         struct list_head splice;
4873
4874         INIT_LIST_HEAD(&splice);
4875
4876         spin_lock(&fs_info->ordered_root_lock);
4877         list_splice_init(&fs_info->ordered_roots, &splice);
4878         while (!list_empty(&splice)) {
4879                 root = list_first_entry(&splice, struct btrfs_root,
4880                                         ordered_root);
4881                 list_move_tail(&root->ordered_root,
4882                                &fs_info->ordered_roots);
4883
4884                 spin_unlock(&fs_info->ordered_root_lock);
4885                 btrfs_destroy_ordered_extents(root);
4886
4887                 cond_resched();
4888                 spin_lock(&fs_info->ordered_root_lock);
4889         }
4890         spin_unlock(&fs_info->ordered_root_lock);
4891
4892         /*
4893          * We need this here because if we've been flipped read-only we won't
4894          * get sync() from the umount, so we need to make sure any ordered
4895          * extents that haven't had their dirty pages IO start writeout yet
4896          * actually get run and error out properly.
4897          */
4898         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
4899 }
4900
4901 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
4902                                       struct btrfs_fs_info *fs_info)
4903 {
4904         struct rb_node *node;
4905         struct btrfs_delayed_ref_root *delayed_refs;
4906         struct btrfs_delayed_ref_node *ref;
4907         int ret = 0;
4908
4909         delayed_refs = &trans->delayed_refs;
4910
4911         spin_lock(&delayed_refs->lock);
4912         if (atomic_read(&delayed_refs->num_entries) == 0) {
4913                 spin_unlock(&delayed_refs->lock);
4914                 btrfs_debug(fs_info, "delayed_refs has NO entry");
4915                 return ret;
4916         }
4917
4918         while ((node = rb_first_cached(&delayed_refs->href_root)) != NULL) {
4919                 struct btrfs_delayed_ref_head *head;
4920                 struct rb_node *n;
4921                 bool pin_bytes = false;
4922
4923                 head = rb_entry(node, struct btrfs_delayed_ref_head,
4924                                 href_node);
4925                 if (btrfs_delayed_ref_lock(delayed_refs, head))
4926                         continue;
4927
4928                 spin_lock(&head->lock);
4929                 while ((n = rb_first_cached(&head->ref_tree)) != NULL) {
4930                         ref = rb_entry(n, struct btrfs_delayed_ref_node,
4931                                        ref_node);
4932                         ref->in_tree = 0;
4933                         rb_erase_cached(&ref->ref_node, &head->ref_tree);
4934                         RB_CLEAR_NODE(&ref->ref_node);
4935                         if (!list_empty(&ref->add_list))
4936                                 list_del(&ref->add_list);
4937                         atomic_dec(&delayed_refs->num_entries);
4938                         btrfs_put_delayed_ref(ref);
4939                 }
4940                 if (head->must_insert_reserved)
4941                         pin_bytes = true;
4942                 btrfs_free_delayed_extent_op(head->extent_op);
4943                 btrfs_delete_ref_head(delayed_refs, head);
4944                 spin_unlock(&head->lock);
4945                 spin_unlock(&delayed_refs->lock);
4946                 mutex_unlock(&head->mutex);
4947
4948                 if (pin_bytes) {
4949                         struct btrfs_block_group *cache;
4950
4951                         cache = btrfs_lookup_block_group(fs_info, head->bytenr);
4952                         BUG_ON(!cache);
4953
4954                         spin_lock(&cache->space_info->lock);
4955                         spin_lock(&cache->lock);
4956                         cache->pinned += head->num_bytes;
4957                         btrfs_space_info_update_bytes_pinned(fs_info,
4958                                 cache->space_info, head->num_bytes);
4959                         cache->reserved -= head->num_bytes;
4960                         cache->space_info->bytes_reserved -= head->num_bytes;
4961                         spin_unlock(&cache->lock);
4962                         spin_unlock(&cache->space_info->lock);
4963
4964                         btrfs_put_block_group(cache);
4965
4966                         btrfs_error_unpin_extent_range(fs_info, head->bytenr,
4967                                 head->bytenr + head->num_bytes - 1);
4968                 }
4969                 btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head);
4970                 btrfs_put_delayed_ref_head(head);
4971                 cond_resched();
4972                 spin_lock(&delayed_refs->lock);
4973         }
4974         btrfs_qgroup_destroy_extent_records(trans);
4975
4976         spin_unlock(&delayed_refs->lock);
4977
4978         return ret;
4979 }
4980
4981 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root)
4982 {
4983         struct btrfs_inode *btrfs_inode;
4984         struct list_head splice;
4985
4986         INIT_LIST_HEAD(&splice);
4987
4988         spin_lock(&root->delalloc_lock);
4989         list_splice_init(&root->delalloc_inodes, &splice);
4990
4991         while (!list_empty(&splice)) {
4992                 struct inode *inode = NULL;
4993                 btrfs_inode = list_first_entry(&splice, struct btrfs_inode,
4994                                                delalloc_inodes);
4995                 __btrfs_del_delalloc_inode(root, btrfs_inode);
4996                 spin_unlock(&root->delalloc_lock);
4997
4998                 /*
4999                  * Make sure we get a live inode and that it'll not disappear
5000                  * meanwhile.
5001                  */
5002                 inode = igrab(&btrfs_inode->vfs_inode);
5003                 if (inode) {
5004                         invalidate_inode_pages2(inode->i_mapping);
5005                         iput(inode);
5006                 }
5007                 spin_lock(&root->delalloc_lock);
5008         }
5009         spin_unlock(&root->delalloc_lock);
5010 }
5011
5012 static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info)
5013 {
5014         struct btrfs_root *root;
5015         struct list_head splice;
5016
5017         INIT_LIST_HEAD(&splice);
5018
5019         spin_lock(&fs_info->delalloc_root_lock);
5020         list_splice_init(&fs_info->delalloc_roots, &splice);
5021         while (!list_empty(&splice)) {
5022                 root = list_first_entry(&splice, struct btrfs_root,
5023                                          delalloc_root);
5024                 root = btrfs_grab_root(root);
5025                 BUG_ON(!root);
5026                 spin_unlock(&fs_info->delalloc_root_lock);
5027
5028                 btrfs_destroy_delalloc_inodes(root);
5029                 btrfs_put_root(root);
5030
5031                 spin_lock(&fs_info->delalloc_root_lock);
5032         }
5033         spin_unlock(&fs_info->delalloc_root_lock);
5034 }
5035
5036 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
5037                                         struct extent_io_tree *dirty_pages,
5038                                         int mark)
5039 {
5040         int ret;
5041         struct extent_buffer *eb;
5042         u64 start = 0;
5043         u64 end;
5044
5045         while (1) {
5046                 ret = find_first_extent_bit(dirty_pages, start, &start, &end,
5047                                             mark, NULL);
5048                 if (ret)
5049                         break;
5050
5051                 clear_extent_bits(dirty_pages, start, end, mark);
5052                 while (start <= end) {
5053                         eb = find_extent_buffer(fs_info, start);
5054                         start += fs_info->nodesize;
5055                         if (!eb)
5056                                 continue;
5057                         wait_on_extent_buffer_writeback(eb);
5058
5059                         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY,
5060                                                &eb->bflags))
5061                                 clear_extent_buffer_dirty(eb);
5062                         free_extent_buffer_stale(eb);
5063                 }
5064         }
5065
5066         return ret;
5067 }
5068
5069 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
5070                                        struct extent_io_tree *unpin)
5071 {
5072         u64 start;
5073         u64 end;
5074         int ret;
5075
5076         while (1) {
5077                 struct extent_state *cached_state = NULL;
5078
5079                 /*
5080                  * The btrfs_finish_extent_commit() may get the same range as
5081                  * ours between find_first_extent_bit and clear_extent_dirty.
5082                  * Hence, hold the unused_bg_unpin_mutex to avoid double unpin
5083                  * the same extent range.
5084                  */
5085                 mutex_lock(&fs_info->unused_bg_unpin_mutex);
5086                 ret = find_first_extent_bit(unpin, 0, &start, &end,
5087                                             EXTENT_DIRTY, &cached_state);
5088                 if (ret) {
5089                         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5090                         break;
5091                 }
5092
5093                 clear_extent_dirty(unpin, start, end, &cached_state);
5094                 free_extent_state(cached_state);
5095                 btrfs_error_unpin_extent_range(fs_info, start, end);
5096                 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5097                 cond_resched();
5098         }
5099
5100         return 0;
5101 }
5102
5103 static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache)
5104 {
5105         struct inode *inode;
5106
5107         inode = cache->io_ctl.inode;
5108         if (inode) {
5109                 invalidate_inode_pages2(inode->i_mapping);
5110                 BTRFS_I(inode)->generation = 0;
5111                 cache->io_ctl.inode = NULL;
5112                 iput(inode);
5113         }
5114         ASSERT(cache->io_ctl.pages == NULL);
5115         btrfs_put_block_group(cache);
5116 }
5117
5118 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
5119                              struct btrfs_fs_info *fs_info)
5120 {
5121         struct btrfs_block_group *cache;
5122
5123         spin_lock(&cur_trans->dirty_bgs_lock);
5124         while (!list_empty(&cur_trans->dirty_bgs)) {
5125                 cache = list_first_entry(&cur_trans->dirty_bgs,
5126                                          struct btrfs_block_group,
5127                                          dirty_list);
5128
5129                 if (!list_empty(&cache->io_list)) {
5130                         spin_unlock(&cur_trans->dirty_bgs_lock);
5131                         list_del_init(&cache->io_list);
5132                         btrfs_cleanup_bg_io(cache);
5133                         spin_lock(&cur_trans->dirty_bgs_lock);
5134                 }
5135
5136                 list_del_init(&cache->dirty_list);
5137                 spin_lock(&cache->lock);
5138                 cache->disk_cache_state = BTRFS_DC_ERROR;
5139                 spin_unlock(&cache->lock);
5140
5141                 spin_unlock(&cur_trans->dirty_bgs_lock);
5142                 btrfs_put_block_group(cache);
5143                 btrfs_delayed_refs_rsv_release(fs_info, 1);
5144                 spin_lock(&cur_trans->dirty_bgs_lock);
5145         }
5146         spin_unlock(&cur_trans->dirty_bgs_lock);
5147
5148         /*
5149          * Refer to the definition of io_bgs member for details why it's safe
5150          * to use it without any locking
5151          */
5152         while (!list_empty(&cur_trans->io_bgs)) {
5153                 cache = list_first_entry(&cur_trans->io_bgs,
5154                                          struct btrfs_block_group,
5155                                          io_list);
5156
5157                 list_del_init(&cache->io_list);
5158                 spin_lock(&cache->lock);
5159                 cache->disk_cache_state = BTRFS_DC_ERROR;
5160                 spin_unlock(&cache->lock);
5161                 btrfs_cleanup_bg_io(cache);
5162         }
5163 }
5164
5165 void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans,
5166                                    struct btrfs_fs_info *fs_info)
5167 {
5168         struct btrfs_device *dev, *tmp;
5169
5170         btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
5171         ASSERT(list_empty(&cur_trans->dirty_bgs));
5172         ASSERT(list_empty(&cur_trans->io_bgs));
5173
5174         list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list,
5175                                  post_commit_list) {
5176                 list_del_init(&dev->post_commit_list);
5177         }
5178
5179         btrfs_destroy_delayed_refs(cur_trans, fs_info);
5180
5181         cur_trans->state = TRANS_STATE_COMMIT_START;
5182         wake_up(&fs_info->transaction_blocked_wait);
5183
5184         cur_trans->state = TRANS_STATE_UNBLOCKED;
5185         wake_up(&fs_info->transaction_wait);
5186
5187         btrfs_destroy_delayed_inodes(fs_info);
5188
5189         btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages,
5190                                      EXTENT_DIRTY);
5191         btrfs_destroy_pinned_extent(fs_info, &cur_trans->pinned_extents);
5192
5193         btrfs_free_redirty_list(cur_trans);
5194
5195         cur_trans->state =TRANS_STATE_COMPLETED;
5196         wake_up(&cur_trans->commit_wait);
5197 }
5198
5199 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info)
5200 {
5201         struct btrfs_transaction *t;
5202
5203         mutex_lock(&fs_info->transaction_kthread_mutex);
5204
5205         spin_lock(&fs_info->trans_lock);
5206         while (!list_empty(&fs_info->trans_list)) {
5207                 t = list_first_entry(&fs_info->trans_list,
5208                                      struct btrfs_transaction, list);
5209                 if (t->state >= TRANS_STATE_COMMIT_START) {
5210                         refcount_inc(&t->use_count);
5211                         spin_unlock(&fs_info->trans_lock);
5212                         btrfs_wait_for_commit(fs_info, t->transid);
5213                         btrfs_put_transaction(t);
5214                         spin_lock(&fs_info->trans_lock);
5215                         continue;
5216                 }
5217                 if (t == fs_info->running_transaction) {
5218                         t->state = TRANS_STATE_COMMIT_DOING;
5219                         spin_unlock(&fs_info->trans_lock);
5220                         /*
5221                          * We wait for 0 num_writers since we don't hold a trans
5222                          * handle open currently for this transaction.
5223                          */
5224                         wait_event(t->writer_wait,
5225                                    atomic_read(&t->num_writers) == 0);
5226                 } else {
5227                         spin_unlock(&fs_info->trans_lock);
5228                 }
5229                 btrfs_cleanup_one_transaction(t, fs_info);
5230
5231                 spin_lock(&fs_info->trans_lock);
5232                 if (t == fs_info->running_transaction)
5233                         fs_info->running_transaction = NULL;
5234                 list_del_init(&t->list);
5235                 spin_unlock(&fs_info->trans_lock);
5236
5237                 btrfs_put_transaction(t);
5238                 trace_btrfs_transaction_commit(fs_info);
5239                 spin_lock(&fs_info->trans_lock);
5240         }
5241         spin_unlock(&fs_info->trans_lock);
5242         btrfs_destroy_all_ordered_extents(fs_info);
5243         btrfs_destroy_delayed_inodes(fs_info);
5244         btrfs_assert_delayed_root_empty(fs_info);
5245         btrfs_destroy_all_delalloc_inodes(fs_info);
5246         btrfs_drop_all_logs(fs_info);
5247         mutex_unlock(&fs_info->transaction_kthread_mutex);
5248
5249         return 0;
5250 }
5251
5252 int btrfs_init_root_free_objectid(struct btrfs_root *root)
5253 {
5254         struct btrfs_path *path;
5255         int ret;
5256         struct extent_buffer *l;
5257         struct btrfs_key search_key;
5258         struct btrfs_key found_key;
5259         int slot;
5260
5261         path = btrfs_alloc_path();
5262         if (!path)
5263                 return -ENOMEM;
5264
5265         search_key.objectid = BTRFS_LAST_FREE_OBJECTID;
5266         search_key.type = -1;
5267         search_key.offset = (u64)-1;
5268         ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0);
5269         if (ret < 0)
5270                 goto error;
5271         BUG_ON(ret == 0); /* Corruption */
5272         if (path->slots[0] > 0) {
5273                 slot = path->slots[0] - 1;
5274                 l = path->nodes[0];
5275                 btrfs_item_key_to_cpu(l, &found_key, slot);
5276                 root->free_objectid = max_t(u64, found_key.objectid + 1,
5277                                             BTRFS_FIRST_FREE_OBJECTID);
5278         } else {
5279                 root->free_objectid = BTRFS_FIRST_FREE_OBJECTID;
5280         }
5281         ret = 0;
5282 error:
5283         btrfs_free_path(path);
5284         return ret;
5285 }
5286
5287 int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid)
5288 {
5289         int ret;
5290         mutex_lock(&root->objectid_mutex);
5291
5292         if (unlikely(root->free_objectid >= BTRFS_LAST_FREE_OBJECTID)) {
5293                 btrfs_warn(root->fs_info,
5294                            "the objectid of root %llu reaches its highest value",
5295                            root->root_key.objectid);
5296                 ret = -ENOSPC;
5297                 goto out;
5298         }
5299
5300         *objectid = root->free_objectid++;
5301         ret = 0;
5302 out:
5303         mutex_unlock(&root->objectid_mutex);
5304         return ret;
5305 }