fs/btrfs/disk-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/blkdev.h>
   8 #include <linux/radix-tree.h>
   9 #include <linux/writeback.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/kthread.h>
  12 #include <linux/slab.h>
  13 #include <linux/migrate.h>
  14 #include <linux/ratelimit.h>
  15 #include <linux/uuid.h>
  16 #include <linux/semaphore.h>
  17 #include <linux/error-injection.h>
  18 #include <linux/crc32c.h>
  19 #include <linux/sched/mm.h>
  20 #include <asm/unaligned.h>
  21 #include <crypto/hash.h>
  22 #include "ctree.h"
  23 #include "disk-io.h"
  24 #include "transaction.h"
  25 #include "btrfs_inode.h"
  26 #include "volumes.h"
  27 #include "print-tree.h"
  28 #include "locking.h"
  29 #include "tree-log.h"
  30 #include "free-space-cache.h"
  31 #include "free-space-tree.h"
  32 #include "check-integrity.h"
  33 #include "rcu-string.h"
  34 #include "dev-replace.h"
  35 #include "raid56.h"
  36 #include "sysfs.h"
  37 #include "qgroup.h"
  38 #include "compression.h"
  39 #include "tree-checker.h"
  40 #include "ref-verify.h"
  41 #include "block-group.h"
  42 #include "discard.h"
  43 #include "space-info.h"
  44 #include "zoned.h"
  45 #include "subpage.h"
  46
  47 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
  48                                  BTRFS_HEADER_FLAG_RELOC |\
  49                                  BTRFS_SUPER_FLAG_ERROR |\
  50                                  BTRFS_SUPER_FLAG_SEEDING |\
  51                                  BTRFS_SUPER_FLAG_METADUMP |\
  52                                  BTRFS_SUPER_FLAG_METADUMP_V2)
  53
  54 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
  55 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
  56                                       struct btrfs_fs_info *fs_info);
  57 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
  58 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
  59                                         struct extent_io_tree *dirty_pages,
  60                                         int mark);
  61 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
  62                                        struct extent_io_tree *pinned_extents);
  63 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
  64 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
  65
  66 static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
  67 {
  68         if (fs_info->csum_shash)
  69                 crypto_free_shash(fs_info->csum_shash);
  70 }
  71
  72 /*
  73  * async submit bios are used to offload expensive checksumming
  74  * onto the worker threads.  They checksum file and metadata bios
  75  * just before they are sent down the IO stack.
  76  */
  77 struct async_submit_bio {
  78         struct inode *inode;
  79         struct bio *bio;
  80         extent_submit_bio_start_t *submit_bio_start;
  81         int mirror_num;
  82
  83         /* Optional parameter for submit_bio_start used by direct io */
  84         u64 dio_file_offset;
  85         struct btrfs_work work;
  86         blk_status_t status;
  87 };
  88
  89 /*
  90  * Compute the csum of a btree block and store the result to provided buffer.
  91  */
  92 static void csum_tree_block(struct extent_buffer *buf, u8 *result)
  93 {
  94         struct btrfs_fs_info *fs_info = buf->fs_info;
  95         const int num_pages = num_extent_pages(buf);
  96         const int first_page_part = min_t(u32, PAGE_SIZE, fs_info->nodesize);
  97         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
  98         char *kaddr;
  99         int i;
 100
 101         shash->tfm = fs_info->csum_shash;
 102         crypto_shash_init(shash);
 103         kaddr = page_address(buf->pages[0]) + offset_in_page(buf->start);
 104         crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
 105                             first_page_part - BTRFS_CSUM_SIZE);
 106
 107         for (i = 1; i < num_pages; i++) {
 108                 kaddr = page_address(buf->pages[i]);
 109                 crypto_shash_update(shash, kaddr, PAGE_SIZE);
 110         }
 111         memset(result, 0, BTRFS_CSUM_SIZE);
 112         crypto_shash_final(shash, result);
 113 }
 114
 115 /*
 116  * we can't consider a given block up to date unless the transid of the
 117  * block matches the transid in the parent node's pointer.  This is how we
 118  * detect blocks that either didn't get written at all or got written
 119  * in the wrong place.
 120  */
 121 static int verify_parent_transid(struct extent_io_tree *io_tree,
 122                                  struct extent_buffer *eb, u64 parent_transid,
 123                                  int atomic)
 124 {
 125         struct extent_state *cached_state = NULL;
 126         int ret;
 127
 128         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
 129                 return 0;
 130
 131         if (atomic)
 132                 return -EAGAIN;
 133
 134         lock_extent(io_tree, eb->start, eb->start + eb->len - 1, &cached_state);
 135         if (extent_buffer_uptodate(eb) &&
 136             btrfs_header_generation(eb) == parent_transid) {
 137                 ret = 0;
 138                 goto out;
 139         }
 140         btrfs_err_rl(eb->fs_info,
 141 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
 142                         eb->start, eb->read_mirror,
 143                         parent_transid, btrfs_header_generation(eb));
 144         ret = 1;
 145         clear_extent_buffer_uptodate(eb);
 146 out:
 147         unlock_extent(io_tree, eb->start, eb->start + eb->len - 1,
 148                       &cached_state);
 149         return ret;
 150 }
 151
 152 static bool btrfs_supported_super_csum(u16 csum_type)
 153 {
 154         switch (csum_type) {
 155         case BTRFS_CSUM_TYPE_CRC32:
 156         case BTRFS_CSUM_TYPE_XXHASH:
 157         case BTRFS_CSUM_TYPE_SHA256:
 158         case BTRFS_CSUM_TYPE_BLAKE2:
 159                 return true;
 160         default:
 161                 return false;
 162         }
 163 }
 164
 165 /*
 166  * Return 0 if the superblock checksum type matches the checksum value of that
 167  * algorithm. Pass the raw disk superblock data.
 168  */
 169 int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 170                            const struct btrfs_super_block *disk_sb)
 171 {
 172         char result[BTRFS_CSUM_SIZE];
 173         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 174
 175         shash->tfm = fs_info->csum_shash;
 176
 177         /*
 178          * The super_block structure does not span the whole
 179          * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
 180          * filled with zeros and is included in the checksum.
 181          */
 182         crypto_shash_digest(shash, (const u8 *)disk_sb + BTRFS_CSUM_SIZE,
 183                             BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
 184
 185         if (memcmp(disk_sb->csum, result, fs_info->csum_size))
 186                 return 1;
 187
 188         return 0;
 189 }
 190
 191 int btrfs_verify_level_key(struct extent_buffer *eb, int level,
 192                            struct btrfs_key *first_key, u64 parent_transid)
 193 {
 194         struct btrfs_fs_info *fs_info = eb->fs_info;
 195         int found_level;
 196         struct btrfs_key found_key;
 197         int ret;
 198
 199         found_level = btrfs_header_level(eb);
 200         if (found_level != level) {
 201                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 202                      KERN_ERR "BTRFS: tree level check failed\n");
 203                 btrfs_err(fs_info,
 204 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
 205                           eb->start, level, found_level);
 206                 return -EIO;
 207         }
 208
 209         if (!first_key)
 210                 return 0;
 211
 212         /*
 213          * For live tree block (new tree blocks in current transaction),
 214          * we need proper lock context to avoid race, which is impossible here.
 215          * So we only checks tree blocks which is read from disk, whose
 216          * generation <= fs_info->last_trans_committed.
 217          */
 218         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
 219                 return 0;
 220
 221         /* We have @first_key, so this @eb must have at least one item */
 222         if (btrfs_header_nritems(eb) == 0) {
 223                 btrfs_err(fs_info,
 224                 "invalid tree nritems, bytenr=%llu nritems=0 expect >0",
 225                           eb->start);
 226                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 227                 return -EUCLEAN;
 228         }
 229
 230         if (found_level)
 231                 btrfs_node_key_to_cpu(eb, &found_key, 0);
 232         else
 233                 btrfs_item_key_to_cpu(eb, &found_key, 0);
 234         ret = btrfs_comp_cpu_keys(first_key, &found_key);
 235
 236         if (ret) {
 237                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 238                      KERN_ERR "BTRFS: tree first key check failed\n");
 239                 btrfs_err(fs_info,
 240 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 241                           eb->start, parent_transid, first_key->objectid,
 242                           first_key->type, first_key->offset,
 243                           found_key.objectid, found_key.type,
 244                           found_key.offset);
 245         }
 246         return ret;
 247 }
 248
 249 /*
 250  * helper to read a given tree block, doing retries as required when
 251  * the checksums don't match and we have alternate mirrors to try.
 252  *
 253  * @parent_transid:     expected transid, skip check if 0
 254  * @level:              expected level, mandatory check
 255  * @first_key:          expected key of first slot, skip check if NULL
 256  */
 257 int btrfs_read_extent_buffer(struct extent_buffer *eb,
 258                              u64 parent_transid, int level,
 259                              struct btrfs_key *first_key)
 260 {
 261         struct btrfs_fs_info *fs_info = eb->fs_info;
 262         struct extent_io_tree *io_tree;
 263         int failed = 0;
 264         int ret;
 265         int num_copies = 0;
 266         int mirror_num = 0;
 267         int failed_mirror = 0;
 268
 269         io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
 270         while (1) {
 271                 clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 272                 ret = read_extent_buffer_pages(eb, WAIT_COMPLETE, mirror_num);
 273                 if (!ret) {
 274                         if (verify_parent_transid(io_tree, eb,
 275                                                    parent_transid, 0))
 276                                 ret = -EIO;
 277                         else if (btrfs_verify_level_key(eb, level,
 278                                                 first_key, parent_transid))
 279                                 ret = -EUCLEAN;
 280                         else
 281                                 break;
 282                 }
 283
 284                 num_copies = btrfs_num_copies(fs_info,
 285                                               eb->start, eb->len);
 286                 if (num_copies == 1)
 287                         break;
 288
 289                 if (!failed_mirror) {
 290                         failed = 1;
 291                         failed_mirror = eb->read_mirror;
 292                 }
 293
 294                 mirror_num++;
 295                 if (mirror_num == failed_mirror)
 296                         mirror_num++;
 297
 298                 if (mirror_num > num_copies)
 299                         break;
 300         }
 301
 302         if (failed && !ret && failed_mirror)
 303                 btrfs_repair_eb_io_failure(eb, failed_mirror);
 304
 305         return ret;
 306 }
 307
 308 static int csum_one_extent_buffer(struct extent_buffer *eb)
 309 {
 310         struct btrfs_fs_info *fs_info = eb->fs_info;
 311         u8 result[BTRFS_CSUM_SIZE];
 312         int ret;
 313
 314         ASSERT(memcmp_extent_buffer(eb, fs_info->fs_devices->metadata_uuid,
 315                                     offsetof(struct btrfs_header, fsid),
 316                                     BTRFS_FSID_SIZE) == 0);
 317         csum_tree_block(eb, result);
 318
 319         if (btrfs_header_level(eb))
 320                 ret = btrfs_check_node(eb);
 321         else
 322                 ret = btrfs_check_leaf_full(eb);
 323
 324         if (ret < 0)
 325                 goto error;
 326
 327         /*
 328          * Also check the generation, the eb reached here must be newer than
 329          * last committed. Or something seriously wrong happened.
 330          */
 331         if (unlikely(btrfs_header_generation(eb) <= fs_info->last_trans_committed)) {
 332                 ret = -EUCLEAN;
 333                 btrfs_err(fs_info,
 334                         "block=%llu bad generation, have %llu expect > %llu",
 335                           eb->start, btrfs_header_generation(eb),
 336                           fs_info->last_trans_committed);
 337                 goto error;
 338         }
 339         write_extent_buffer(eb, result, 0, fs_info->csum_size);
 340
 341         return 0;
 342
 343 error:
 344         btrfs_print_tree(eb, 0);
 345         btrfs_err(fs_info, "block=%llu write time tree block corruption detected",
 346                   eb->start);
 347         WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 348         return ret;
 349 }
 350
 351 /* Checksum all dirty extent buffers in one bio_vec */
 352 static int csum_dirty_subpage_buffers(struct btrfs_fs_info *fs_info,
 353                                       struct bio_vec *bvec)
 354 {
 355         struct page *page = bvec->bv_page;
 356         u64 bvec_start = page_offset(page) + bvec->bv_offset;
 357         u64 cur;
 358         int ret = 0;
 359
 360         for (cur = bvec_start; cur < bvec_start + bvec->bv_len;
 361              cur += fs_info->nodesize) {
 362                 struct extent_buffer *eb;
 363                 bool uptodate;
 364
 365                 eb = find_extent_buffer(fs_info, cur);
 366                 uptodate = btrfs_subpage_test_uptodate(fs_info, page, cur,
 367                                                        fs_info->nodesize);
 368
 369                 /* A dirty eb shouldn't disappear from buffer_radix */
 370                 if (WARN_ON(!eb))
 371                         return -EUCLEAN;
 372
 373                 if (WARN_ON(cur != btrfs_header_bytenr(eb))) {
 374                         free_extent_buffer(eb);
 375                         return -EUCLEAN;
 376                 }
 377                 if (WARN_ON(!uptodate)) {
 378                         free_extent_buffer(eb);
 379                         return -EUCLEAN;
 380                 }
 381
 382                 ret = csum_one_extent_buffer(eb);
 383                 free_extent_buffer(eb);
 384                 if (ret < 0)
 385                         return ret;
 386         }
 387         return ret;
 388 }
 389
 390 /*
 391  * Checksum a dirty tree block before IO.  This has extra checks to make sure
 392  * we only fill in the checksum field in the first page of a multi-page block.
 393  * For subpage extent buffers we need bvec to also read the offset in the page.
 394  */
 395 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec)
 396 {
 397         struct page *page = bvec->bv_page;
 398         u64 start = page_offset(page);
 399         u64 found_start;
 400         struct extent_buffer *eb;
 401
 402         if (fs_info->nodesize < PAGE_SIZE)
 403                 return csum_dirty_subpage_buffers(fs_info, bvec);
 404
 405         eb = (struct extent_buffer *)page->private;
 406         if (page != eb->pages[0])
 407                 return 0;
 408
 409         found_start = btrfs_header_bytenr(eb);
 410
 411         if (test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)) {
 412                 WARN_ON(found_start != 0);
 413                 return 0;
 414         }
 415
 416         /*
 417          * Please do not consolidate these warnings into a single if.
 418          * It is useful to know what went wrong.
 419          */
 420         if (WARN_ON(found_start != start))
 421                 return -EUCLEAN;
 422         if (WARN_ON(!PageUptodate(page)))
 423                 return -EUCLEAN;
 424
 425         return csum_one_extent_buffer(eb);
 426 }
 427
 428 static int check_tree_block_fsid(struct extent_buffer *eb)
 429 {
 430         struct btrfs_fs_info *fs_info = eb->fs_info;
 431         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
 432         u8 fsid[BTRFS_FSID_SIZE];
 433         u8 *metadata_uuid;
 434
 435         read_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid),
 436                            BTRFS_FSID_SIZE);
 437         /*
 438          * Checking the incompat flag is only valid for the current fs. For
 439          * seed devices it's forbidden to have their uuid changed so reading
 440          * ->fsid in this case is fine
 441          */
 442         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
 443                 metadata_uuid = fs_devices->metadata_uuid;
 444         else
 445                 metadata_uuid = fs_devices->fsid;
 446
 447         if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE))
 448                 return 0;
 449
 450         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list)
 451                 if (!memcmp(fsid, seed_devs->fsid, BTRFS_FSID_SIZE))
 452                         return 0;
 453
 454         return 1;
 455 }
 456
 457 /* Do basic extent buffer checks at read time */
 458 static int validate_extent_buffer(struct extent_buffer *eb)
 459 {
 460         struct btrfs_fs_info *fs_info = eb->fs_info;
 461         u64 found_start;
 462         const u32 csum_size = fs_info->csum_size;
 463         u8 found_level;
 464         u8 result[BTRFS_CSUM_SIZE];
 465         const u8 *header_csum;
 466         int ret = 0;
 467
 468         found_start = btrfs_header_bytenr(eb);
 469         if (found_start != eb->start) {
 470                 btrfs_err_rl(fs_info,
 471                         "bad tree block start, mirror %u want %llu have %llu",
 472                              eb->read_mirror, eb->start, found_start);
 473                 ret = -EIO;
 474                 goto out;
 475         }
 476         if (check_tree_block_fsid(eb)) {
 477                 btrfs_err_rl(fs_info, "bad fsid on logical %llu mirror %u",
 478                              eb->start, eb->read_mirror);
 479                 ret = -EIO;
 480                 goto out;
 481         }
 482         found_level = btrfs_header_level(eb);
 483         if (found_level >= BTRFS_MAX_LEVEL) {
 484                 btrfs_err(fs_info,
 485                         "bad tree block level, mirror %u level %d on logical %llu",
 486                         eb->read_mirror, btrfs_header_level(eb), eb->start);
 487                 ret = -EIO;
 488                 goto out;
 489         }
 490
 491         csum_tree_block(eb, result);
 492         header_csum = page_address(eb->pages[0]) +
 493                 get_eb_offset_in_page(eb, offsetof(struct btrfs_header, csum));
 494
 495         if (memcmp(result, header_csum, csum_size) != 0) {
 496                 btrfs_warn_rl(fs_info,
 497 "checksum verify failed on logical %llu mirror %u wanted " CSUM_FMT " found " CSUM_FMT " level %d",
 498                               eb->start, eb->read_mirror,
 499                               CSUM_FMT_VALUE(csum_size, header_csum),
 500                               CSUM_FMT_VALUE(csum_size, result),
 501                               btrfs_header_level(eb));
 502                 ret = -EUCLEAN;
 503                 goto out;
 504         }
 505
 506         /*
 507          * If this is a leaf block and it is corrupt, set the corrupt bit so
 508          * that we don't try and read the other copies of this block, just
 509          * return -EIO.
 510          */
 511         if (found_level == 0 && btrfs_check_leaf_full(eb)) {
 512                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 513                 ret = -EIO;
 514         }
 515
 516         if (found_level > 0 && btrfs_check_node(eb))
 517                 ret = -EIO;
 518
 519         if (!ret)
 520                 set_extent_buffer_uptodate(eb);
 521         else
 522                 btrfs_err(fs_info,
 523                 "read time tree block corruption detected on logical %llu mirror %u",
 524                           eb->start, eb->read_mirror);
 525 out:
 526         return ret;
 527 }
 528
 529 static int validate_subpage_buffer(struct page *page, u64 start, u64 end,
 530                                    int mirror)
 531 {
 532         struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
 533         struct extent_buffer *eb;
 534         bool reads_done;
 535         int ret = 0;
 536
 537         /*
 538          * We don't allow bio merge for subpage metadata read, so we should
 539          * only get one eb for each endio hook.
 540          */
 541         ASSERT(end == start + fs_info->nodesize - 1);
 542         ASSERT(PagePrivate(page));
 543
 544         eb = find_extent_buffer(fs_info, start);
 545         /*
 546          * When we are reading one tree block, eb must have been inserted into
 547          * the radix tree. If not, something is wrong.
 548          */
 549         ASSERT(eb);
 550
 551         reads_done = atomic_dec_and_test(&eb->io_pages);
 552         /* Subpage read must finish in page read */
 553         ASSERT(reads_done);
 554
 555         eb->read_mirror = mirror;
 556         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 557                 ret = -EIO;
 558                 goto err;
 559         }
 560         ret = validate_extent_buffer(eb);
 561         if (ret < 0)
 562                 goto err;
 563
 564         set_extent_buffer_uptodate(eb);
 565
 566         free_extent_buffer(eb);
 567         return ret;
 568 err:
 569         /*
 570          * end_bio_extent_readpage decrements io_pages in case of error,
 571          * make sure it has something to decrement.
 572          */
 573         atomic_inc(&eb->io_pages);
 574         clear_extent_buffer_uptodate(eb);
 575         free_extent_buffer(eb);
 576         return ret;
 577 }
 578
 579 int btrfs_validate_metadata_buffer(struct btrfs_bio *bbio,
 580                                    struct page *page, u64 start, u64 end,
 581                                    int mirror)
 582 {
 583         struct extent_buffer *eb;
 584         int ret = 0;
 585         int reads_done;
 586
 587         ASSERT(page->private);
 588
 589         if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
 590                 return validate_subpage_buffer(page, start, end, mirror);
 591
 592         eb = (struct extent_buffer *)page->private;
 593
 594         /*
 595          * The pending IO might have been the only thing that kept this buffer
 596          * in memory.  Make sure we have a ref for all this other checks
 597          */
 598         atomic_inc(&eb->refs);
 599
 600         reads_done = atomic_dec_and_test(&eb->io_pages);
 601         if (!reads_done)
 602                 goto err;
 603
 604         eb->read_mirror = mirror;
 605         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 606                 ret = -EIO;
 607                 goto err;
 608         }
 609         ret = validate_extent_buffer(eb);
 610 err:
 611         if (ret) {
 612                 /*
 613                  * our io error hook is going to dec the io pages
 614                  * again, we have to make sure it has something
 615                  * to decrement
 616                  */
 617                 atomic_inc(&eb->io_pages);
 618                 clear_extent_buffer_uptodate(eb);
 619         }
 620         free_extent_buffer(eb);
 621
 622         return ret;
 623 }
 624
 625 static void run_one_async_start(struct btrfs_work *work)
 626 {
 627         struct async_submit_bio *async;
 628         blk_status_t ret;
 629
 630         async = container_of(work, struct  async_submit_bio, work);
 631         ret = async->submit_bio_start(async->inode, async->bio,
 632                                       async->dio_file_offset);
 633         if (ret)
 634                 async->status = ret;
 635 }
 636
 637 /*
 638  * In order to insert checksums into the metadata in large chunks, we wait
 639  * until bio submission time.   All the pages in the bio are checksummed and
 640  * sums are attached onto the ordered extent record.
 641  *
 642  * At IO completion time the csums attached on the ordered extent record are
 643  * inserted into the tree.
 644  */
 645 static void run_one_async_done(struct btrfs_work *work)
 646 {
 647         struct async_submit_bio *async =
 648                 container_of(work, struct  async_submit_bio, work);
 649         struct inode *inode = async->inode;
 650         struct btrfs_bio *bbio = btrfs_bio(async->bio);
 651
 652         /* If an error occurred we just want to clean up the bio and move on */
 653         if (async->status) {
 654                 btrfs_bio_end_io(bbio, async->status);
 655                 return;
 656         }
 657
 658         /*
 659          * All of the bios that pass through here are from async helpers.
 660          * Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context.
 661          * This changes nothing when cgroups aren't in use.
 662          */
 663         async->bio->bi_opf |= REQ_CGROUP_PUNT;
 664         btrfs_submit_bio(btrfs_sb(inode->i_sb), async->bio, async->mirror_num);
 665 }
 666
 667 static void run_one_async_free(struct btrfs_work *work)
 668 {
 669         struct async_submit_bio *async;
 670
 671         async = container_of(work, struct  async_submit_bio, work);
 672         kfree(async);
 673 }
 674
 675 /*
 676  * Submit bio to an async queue.
 677  *
 678  * Retrun:
 679  * - true if the work has been succesfuly submitted
 680  * - false in case of error
 681  */
 682 bool btrfs_wq_submit_bio(struct inode *inode, struct bio *bio, int mirror_num,
 683                          u64 dio_file_offset,
 684                          extent_submit_bio_start_t *submit_bio_start)
 685 {
 686         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
 687         struct async_submit_bio *async;
 688
 689         async = kmalloc(sizeof(*async), GFP_NOFS);
 690         if (!async)
 691                 return false;
 692
 693         async->inode = inode;
 694         async->bio = bio;
 695         async->mirror_num = mirror_num;
 696         async->submit_bio_start = submit_bio_start;
 697
 698         btrfs_init_work(&async->work, run_one_async_start, run_one_async_done,
 699                         run_one_async_free);
 700
 701         async->dio_file_offset = dio_file_offset;
 702
 703         async->status = 0;
 704
 705         if (op_is_sync(bio->bi_opf))
 706                 btrfs_queue_work(fs_info->hipri_workers, &async->work);
 707         else
 708                 btrfs_queue_work(fs_info->workers, &async->work);
 709         return true;
 710 }
 711
 712 static blk_status_t btree_csum_one_bio(struct bio *bio)
 713 {
 714         struct bio_vec *bvec;
 715         struct btrfs_root *root;
 716         int ret = 0;
 717         struct bvec_iter_all iter_all;
 718
 719         ASSERT(!bio_flagged(bio, BIO_CLONED));
 720         bio_for_each_segment_all(bvec, bio, iter_all) {
 721                 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
 722                 ret = csum_dirty_buffer(root->fs_info, bvec);
 723                 if (ret)
 724                         break;
 725         }
 726
 727         return errno_to_blk_status(ret);
 728 }
 729
 730 static blk_status_t btree_submit_bio_start(struct inode *inode, struct bio *bio,
 731                                            u64 dio_file_offset)
 732 {
 733         /*
 734          * when we're called for a write, we're already in the async
 735          * submission context.  Just jump into btrfs_submit_bio.
 736          */
 737         return btree_csum_one_bio(bio);
 738 }
 739
 740 static bool should_async_write(struct btrfs_fs_info *fs_info,
 741                              struct btrfs_inode *bi)
 742 {
 743         if (btrfs_is_zoned(fs_info))
 744                 return false;
 745         if (atomic_read(&bi->sync_writers))
 746                 return false;
 747         if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags))
 748                 return false;
 749         return true;
 750 }
 751
 752 void btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int mirror_num)
 753 {
 754         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 755         struct btrfs_bio *bbio = btrfs_bio(bio);
 756         blk_status_t ret;
 757
 758         bio->bi_opf |= REQ_META;
 759
 760         if (btrfs_op(bio) != BTRFS_MAP_WRITE) {
 761                 btrfs_submit_bio(fs_info, bio, mirror_num);
 762                 return;
 763         }
 764
 765         /*
 766          * Kthread helpers are used to submit writes so that checksumming can
 767          * happen in parallel across all CPUs.
 768          */
 769         if (should_async_write(fs_info, BTRFS_I(inode)) &&
 770             btrfs_wq_submit_bio(inode, bio, mirror_num, 0, btree_submit_bio_start))
 771                 return;
 772
 773         ret = btree_csum_one_bio(bio);
 774         if (ret) {
 775                 btrfs_bio_end_io(bbio, ret);
 776                 return;
 777         }
 778
 779         btrfs_submit_bio(fs_info, bio, mirror_num);
 780 }
 781
 782 #ifdef CONFIG_MIGRATION
 783 static int btree_migrate_folio(struct address_space *mapping,
 784                 struct folio *dst, struct folio *src, enum migrate_mode mode)
 785 {
 786         /*
 787          * we can't safely write a btree page from here,
 788          * we haven't done the locking hook
 789          */
 790         if (folio_test_dirty(src))
 791                 return -EAGAIN;
 792         /*
 793          * Buffers may be managed in a filesystem specific way.
 794          * We must have no buffers or drop them.
 795          */
 796         if (folio_get_private(src) &&
 797             !filemap_release_folio(src, GFP_KERNEL))
 798                 return -EAGAIN;
 799         return migrate_folio(mapping, dst, src, mode);
 800 }
 801 #else
 802 #define btree_migrate_folio NULL
 803 #endif
 804
 805 static int btree_writepages(struct address_space *mapping,
 806                             struct writeback_control *wbc)
 807 {
 808         struct btrfs_fs_info *fs_info;
 809         int ret;
 810
 811         if (wbc->sync_mode == WB_SYNC_NONE) {
 812
 813                 if (wbc->for_kupdate)
 814                         return 0;
 815
 816                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
 817                 /* this is a bit racy, but that's ok */
 818                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
 819                                              BTRFS_DIRTY_METADATA_THRESH,
 820                                              fs_info->dirty_metadata_batch);
 821                 if (ret < 0)
 822                         return 0;
 823         }
 824         return btree_write_cache_pages(mapping, wbc);
 825 }
 826
 827 static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags)
 828 {
 829         if (folio_test_writeback(folio) || folio_test_dirty(folio))
 830                 return false;
 831
 832         return try_release_extent_buffer(&folio->page);
 833 }
 834
 835 static void btree_invalidate_folio(struct folio *folio, size_t offset,
 836                                  size_t length)
 837 {
 838         struct extent_io_tree *tree;
 839         tree = &BTRFS_I(folio->mapping->host)->io_tree;
 840         extent_invalidate_folio(tree, folio, offset);
 841         btree_release_folio(folio, GFP_NOFS);
 842         if (folio_get_private(folio)) {
 843                 btrfs_warn(BTRFS_I(folio->mapping->host)->root->fs_info,
 844                            "folio private not zero on folio %llu",
 845                            (unsigned long long)folio_pos(folio));
 846                 folio_detach_private(folio);
 847         }
 848 }
 849
 850 #ifdef DEBUG
 851 static bool btree_dirty_folio(struct address_space *mapping,
 852                 struct folio *folio)
 853 {
 854         struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
 855         struct btrfs_subpage *subpage;
 856         struct extent_buffer *eb;
 857         int cur_bit = 0;
 858         u64 page_start = folio_pos(folio);
 859
 860         if (fs_info->sectorsize == PAGE_SIZE) {
 861                 eb = folio_get_private(folio);
 862                 BUG_ON(!eb);
 863                 BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 864                 BUG_ON(!atomic_read(&eb->refs));
 865                 btrfs_assert_tree_write_locked(eb);
 866                 return filemap_dirty_folio(mapping, folio);
 867         }
 868         subpage = folio_get_private(folio);
 869
 870         ASSERT(subpage->dirty_bitmap);
 871         while (cur_bit < BTRFS_SUBPAGE_BITMAP_SIZE) {
 872                 unsigned long flags;
 873                 u64 cur;
 874                 u16 tmp = (1 << cur_bit);
 875
 876                 spin_lock_irqsave(&subpage->lock, flags);
 877                 if (!(tmp & subpage->dirty_bitmap)) {
 878                         spin_unlock_irqrestore(&subpage->lock, flags);
 879                         cur_bit++;
 880                         continue;
 881                 }
 882                 spin_unlock_irqrestore(&subpage->lock, flags);
 883                 cur = page_start + cur_bit * fs_info->sectorsize;
 884
 885                 eb = find_extent_buffer(fs_info, cur);
 886                 ASSERT(eb);
 887                 ASSERT(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 888                 ASSERT(atomic_read(&eb->refs));
 889                 btrfs_assert_tree_write_locked(eb);
 890                 free_extent_buffer(eb);
 891
 892                 cur_bit += (fs_info->nodesize >> fs_info->sectorsize_bits);
 893         }
 894         return filemap_dirty_folio(mapping, folio);
 895 }
 896 #else
 897 #define btree_dirty_folio filemap_dirty_folio
 898 #endif
 899
 900 static const struct address_space_operations btree_aops = {
 901         .writepages     = btree_writepages,
 902         .release_folio  = btree_release_folio,
 903         .invalidate_folio = btree_invalidate_folio,
 904         .migrate_folio  = btree_migrate_folio,
 905         .dirty_folio    = btree_dirty_folio,
 906 };
 907
 908 struct extent_buffer *btrfs_find_create_tree_block(
 909                                                 struct btrfs_fs_info *fs_info,
 910                                                 u64 bytenr, u64 owner_root,
 911                                                 int level)
 912 {
 913         if (btrfs_is_testing(fs_info))
 914                 return alloc_test_extent_buffer(fs_info, bytenr);
 915         return alloc_extent_buffer(fs_info, bytenr, owner_root, level);
 916 }
 917
 918 /*
 919  * Read tree block at logical address @bytenr and do variant basic but critical
 920  * verification.
 921  *
 922  * @owner_root:         the objectid of the root owner for this block.
 923  * @parent_transid:     expected transid of this tree block, skip check if 0
 924  * @level:              expected level, mandatory check
 925  * @first_key:          expected key in slot 0, skip check if NULL
 926  */
 927 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
 928                                       u64 owner_root, u64 parent_transid,
 929                                       int level, struct btrfs_key *first_key)
 930 {
 931         struct extent_buffer *buf = NULL;
 932         int ret;
 933
 934         buf = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
 935         if (IS_ERR(buf))
 936                 return buf;
 937
 938         ret = btrfs_read_extent_buffer(buf, parent_transid, level, first_key);
 939         if (ret) {
 940                 free_extent_buffer_stale(buf);
 941                 return ERR_PTR(ret);
 942         }
 943         if (btrfs_check_eb_owner(buf, owner_root)) {
 944                 free_extent_buffer_stale(buf);
 945                 return ERR_PTR(-EUCLEAN);
 946         }
 947         return buf;
 948
 949 }
 950
 951 void btrfs_clean_tree_block(struct extent_buffer *buf)
 952 {
 953         struct btrfs_fs_info *fs_info = buf->fs_info;
 954         if (btrfs_header_generation(buf) ==
 955             fs_info->running_transaction->transid) {
 956                 btrfs_assert_tree_write_locked(buf);
 957
 958                 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) {
 959                         percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
 960                                                  -buf->len,
 961                                                  fs_info->dirty_metadata_batch);
 962                         clear_extent_buffer_dirty(buf);
 963                 }
 964         }
 965 }
 966
 967 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
 968                          u64 objectid)
 969 {
 970         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
 971
 972         memset(&root->root_key, 0, sizeof(root->root_key));
 973         memset(&root->root_item, 0, sizeof(root->root_item));
 974         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
 975         root->fs_info = fs_info;
 976         root->root_key.objectid = objectid;
 977         root->node = NULL;
 978         root->commit_root = NULL;
 979         root->state = 0;
 980         RB_CLEAR_NODE(&root->rb_node);
 981
 982         root->last_trans = 0;
 983         root->free_objectid = 0;
 984         root->nr_delalloc_inodes = 0;
 985         root->nr_ordered_extents = 0;
 986         root->inode_tree = RB_ROOT;
 987         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
 988
 989         btrfs_init_root_block_rsv(root);
 990
 991         INIT_LIST_HEAD(&root->dirty_list);
 992         INIT_LIST_HEAD(&root->root_list);
 993         INIT_LIST_HEAD(&root->delalloc_inodes);
 994         INIT_LIST_HEAD(&root->delalloc_root);
 995         INIT_LIST_HEAD(&root->ordered_extents);
 996         INIT_LIST_HEAD(&root->ordered_root);
 997         INIT_LIST_HEAD(&root->reloc_dirty_list);
 998         INIT_LIST_HEAD(&root->logged_list[0]);
 999         INIT_LIST_HEAD(&root->logged_list[1]);
1000         spin_lock_init(&root->inode_lock);
1001         spin_lock_init(&root->delalloc_lock);
1002         spin_lock_init(&root->ordered_extent_lock);
1003         spin_lock_init(&root->accounting_lock);
1004         spin_lock_init(&root->log_extents_lock[0]);
1005         spin_lock_init(&root->log_extents_lock[1]);
1006         spin_lock_init(&root->qgroup_meta_rsv_lock);
1007         mutex_init(&root->objectid_mutex);
1008         mutex_init(&root->log_mutex);
1009         mutex_init(&root->ordered_extent_mutex);
1010         mutex_init(&root->delalloc_mutex);
1011         init_waitqueue_head(&root->qgroup_flush_wait);
1012         init_waitqueue_head(&root->log_writer_wait);
1013         init_waitqueue_head(&root->log_commit_wait[0]);
1014         init_waitqueue_head(&root->log_commit_wait[1]);
1015         INIT_LIST_HEAD(&root->log_ctxs[0]);
1016         INIT_LIST_HEAD(&root->log_ctxs[1]);
1017         atomic_set(&root->log_commit[0], 0);
1018         atomic_set(&root->log_commit[1], 0);
1019         atomic_set(&root->log_writers, 0);
1020         atomic_set(&root->log_batch, 0);
1021         refcount_set(&root->refs, 1);
1022         atomic_set(&root->snapshot_force_cow, 0);
1023         atomic_set(&root->nr_swapfiles, 0);
1024         root->log_transid = 0;
1025         root->log_transid_committed = -1;
1026         root->last_log_commit = 0;
1027         root->anon_dev = 0;
1028         if (!dummy) {
1029                 extent_io_tree_init(fs_info, &root->dirty_log_pages,
1030                                     IO_TREE_ROOT_DIRTY_LOG_PAGES, NULL);
1031                 extent_io_tree_init(fs_info, &root->log_csum_range,
1032                                     IO_TREE_LOG_CSUM_RANGE, NULL);
1033         }
1034
1035         spin_lock_init(&root->root_item_lock);
1036         btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks);
1037 #ifdef CONFIG_BTRFS_DEBUG
1038         INIT_LIST_HEAD(&root->leak_list);
1039         spin_lock(&fs_info->fs_roots_radix_lock);
1040         list_add_tail(&root->leak_list, &fs_info->allocated_roots);
1041         spin_unlock(&fs_info->fs_roots_radix_lock);
1042 #endif
1043 }
1044
1045 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
1046                                            u64 objectid, gfp_t flags)
1047 {
1048         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
1049         if (root)
1050                 __setup_root(root, fs_info, objectid);
1051         return root;
1052 }
1053
1054 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1055 /* Should only be used by the testing infrastructure */
1056 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
1057 {
1058         struct btrfs_root *root;
1059
1060         if (!fs_info)
1061                 return ERR_PTR(-EINVAL);
1062
1063         root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, GFP_KERNEL);
1064         if (!root)
1065                 return ERR_PTR(-ENOMEM);
1066
1067         /* We don't use the stripesize in selftest, set it as sectorsize */
1068         root->alloc_bytenr = 0;
1069
1070         return root;
1071 }
1072 #endif
1073
1074 static int global_root_cmp(struct rb_node *a_node, const struct rb_node *b_node)
1075 {
1076         const struct btrfs_root *a = rb_entry(a_node, struct btrfs_root, rb_node);
1077         const struct btrfs_root *b = rb_entry(b_node, struct btrfs_root, rb_node);
1078
1079         return btrfs_comp_cpu_keys(&a->root_key, &b->root_key);
1080 }
1081
1082 static int global_root_key_cmp(const void *k, const struct rb_node *node)
1083 {
1084         const struct btrfs_key *key = k;
1085         const struct btrfs_root *root = rb_entry(node, struct btrfs_root, rb_node);
1086
1087         return btrfs_comp_cpu_keys(key, &root->root_key);
1088 }
1089
1090 int btrfs_global_root_insert(struct btrfs_root *root)
1091 {
1092         struct btrfs_fs_info *fs_info = root->fs_info;
1093         struct rb_node *tmp;
1094
1095         write_lock(&fs_info->global_root_lock);
1096         tmp = rb_find_add(&root->rb_node, &fs_info->global_root_tree, global_root_cmp);
1097         write_unlock(&fs_info->global_root_lock);
1098         ASSERT(!tmp);
1099
1100         return tmp ? -EEXIST : 0;
1101 }
1102
1103 void btrfs_global_root_delete(struct btrfs_root *root)
1104 {
1105         struct btrfs_fs_info *fs_info = root->fs_info;
1106
1107         write_lock(&fs_info->global_root_lock);
1108         rb_erase(&root->rb_node, &fs_info->global_root_tree);
1109         write_unlock(&fs_info->global_root_lock);
1110 }
1111
1112 struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info,
1113                                      struct btrfs_key *key)
1114 {
1115         struct rb_node *node;
1116         struct btrfs_root *root = NULL;
1117
1118         read_lock(&fs_info->global_root_lock);
1119         node = rb_find(key, &fs_info->global_root_tree, global_root_key_cmp);
1120         if (node)
1121                 root = container_of(node, struct btrfs_root, rb_node);
1122         read_unlock(&fs_info->global_root_lock);
1123
1124         return root;
1125 }
1126
1127 static u64 btrfs_global_root_id(struct btrfs_fs_info *fs_info, u64 bytenr)
1128 {
1129         struct btrfs_block_group *block_group;
1130         u64 ret;
1131
1132         if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))
1133                 return 0;
1134
1135         if (bytenr)
1136                 block_group = btrfs_lookup_block_group(fs_info, bytenr);
1137         else
1138                 block_group = btrfs_lookup_first_block_group(fs_info, bytenr);
1139         ASSERT(block_group);
1140         if (!block_group)
1141                 return 0;
1142         ret = block_group->global_root_id;
1143         btrfs_put_block_group(block_group);
1144
1145         return ret;
1146 }
1147
1148 struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1149 {
1150         struct btrfs_key key = {
1151                 .objectid = BTRFS_CSUM_TREE_OBJECTID,
1152                 .type = BTRFS_ROOT_ITEM_KEY,
1153                 .offset = btrfs_global_root_id(fs_info, bytenr),
1154         };
1155
1156         return btrfs_global_root(fs_info, &key);
1157 }
1158
1159 struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1160 {
1161         struct btrfs_key key = {
1162                 .objectid = BTRFS_EXTENT_TREE_OBJECTID,
1163                 .type = BTRFS_ROOT_ITEM_KEY,
1164                 .offset = btrfs_global_root_id(fs_info, bytenr),
1165         };
1166
1167         return btrfs_global_root(fs_info, &key);
1168 }
1169
1170 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1171                                      u64 objectid)
1172 {
1173         struct btrfs_fs_info *fs_info = trans->fs_info;
1174         struct extent_buffer *leaf;
1175         struct btrfs_root *tree_root = fs_info->tree_root;
1176         struct btrfs_root *root;
1177         struct btrfs_key key;
1178         unsigned int nofs_flag;
1179         int ret = 0;
1180
1181         /*
1182          * We're holding a transaction handle, so use a NOFS memory allocation
1183          * context to avoid deadlock if reclaim happens.
1184          */
1185         nofs_flag = memalloc_nofs_save();
1186         root = btrfs_alloc_root(fs_info, objectid, GFP_KERNEL);
1187         memalloc_nofs_restore(nofs_flag);
1188         if (!root)
1189                 return ERR_PTR(-ENOMEM);
1190
1191         root->root_key.objectid = objectid;
1192         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1193         root->root_key.offset = 0;
1194
1195         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0,
1196                                       BTRFS_NESTING_NORMAL);
1197         if (IS_ERR(leaf)) {
1198                 ret = PTR_ERR(leaf);
1199                 leaf = NULL;
1200                 goto fail_unlock;
1201         }
1202
1203         root->node = leaf;
1204         btrfs_mark_buffer_dirty(leaf);
1205
1206         root->commit_root = btrfs_root_node(root);
1207         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1208
1209         btrfs_set_root_flags(&root->root_item, 0);
1210         btrfs_set_root_limit(&root->root_item, 0);
1211         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1212         btrfs_set_root_generation(&root->root_item, trans->transid);
1213         btrfs_set_root_level(&root->root_item, 0);
1214         btrfs_set_root_refs(&root->root_item, 1);
1215         btrfs_set_root_used(&root->root_item, leaf->len);
1216         btrfs_set_root_last_snapshot(&root->root_item, 0);
1217         btrfs_set_root_dirid(&root->root_item, 0);
1218         if (is_fstree(objectid))
1219                 generate_random_guid(root->root_item.uuid);
1220         else
1221                 export_guid(root->root_item.uuid, &guid_null);
1222         btrfs_set_root_drop_level(&root->root_item, 0);
1223
1224         btrfs_tree_unlock(leaf);
1225
1226         key.objectid = objectid;
1227         key.type = BTRFS_ROOT_ITEM_KEY;
1228         key.offset = 0;
1229         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1230         if (ret)
1231                 goto fail;
1232
1233         return root;
1234
1235 fail_unlock:
1236         if (leaf)
1237                 btrfs_tree_unlock(leaf);
1238 fail:
1239         btrfs_put_root(root);
1240
1241         return ERR_PTR(ret);
1242 }
1243
1244 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1245                                          struct btrfs_fs_info *fs_info)
1246 {
1247         struct btrfs_root *root;
1248
1249         root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS);
1250         if (!root)
1251                 return ERR_PTR(-ENOMEM);
1252
1253         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1254         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1255         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1256
1257         return root;
1258 }
1259
1260 int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans,
1261                               struct btrfs_root *root)
1262 {
1263         struct extent_buffer *leaf;
1264
1265         /*
1266          * DON'T set SHAREABLE bit for log trees.
1267          *
1268          * Log trees are not exposed to user space thus can't be snapshotted,
1269          * and they go away before a real commit is actually done.
1270          *
1271          * They do store pointers to file data extents, and those reference
1272          * counts still get updated (along with back refs to the log tree).
1273          */
1274
1275         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1276                         NULL, 0, 0, 0, BTRFS_NESTING_NORMAL);
1277         if (IS_ERR(leaf))
1278                 return PTR_ERR(leaf);
1279
1280         root->node = leaf;
1281
1282         btrfs_mark_buffer_dirty(root->node);
1283         btrfs_tree_unlock(root->node);
1284
1285         return 0;
1286 }
1287
1288 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1289                              struct btrfs_fs_info *fs_info)
1290 {
1291         struct btrfs_root *log_root;
1292
1293         log_root = alloc_log_tree(trans, fs_info);
1294         if (IS_ERR(log_root))
1295                 return PTR_ERR(log_root);
1296
1297         if (!btrfs_is_zoned(fs_info)) {
1298                 int ret = btrfs_alloc_log_tree_node(trans, log_root);
1299
1300                 if (ret) {
1301                         btrfs_put_root(log_root);
1302                         return ret;
1303                 }
1304         }
1305
1306         WARN_ON(fs_info->log_root_tree);
1307         fs_info->log_root_tree = log_root;
1308         return 0;
1309 }
1310
1311 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1312                        struct btrfs_root *root)
1313 {
1314         struct btrfs_fs_info *fs_info = root->fs_info;
1315         struct btrfs_root *log_root;
1316         struct btrfs_inode_item *inode_item;
1317         int ret;
1318
1319         log_root = alloc_log_tree(trans, fs_info);
1320         if (IS_ERR(log_root))
1321                 return PTR_ERR(log_root);
1322
1323         ret = btrfs_alloc_log_tree_node(trans, log_root);
1324         if (ret) {
1325                 btrfs_put_root(log_root);
1326                 return ret;
1327         }
1328
1329         log_root->last_trans = trans->transid;
1330         log_root->root_key.offset = root->root_key.objectid;
1331
1332         inode_item = &log_root->root_item.inode;
1333         btrfs_set_stack_inode_generation(inode_item, 1);
1334         btrfs_set_stack_inode_size(inode_item, 3);
1335         btrfs_set_stack_inode_nlink(inode_item, 1);
1336         btrfs_set_stack_inode_nbytes(inode_item,
1337                                      fs_info->nodesize);
1338         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1339
1340         btrfs_set_root_node(&log_root->root_item, log_root->node);
1341
1342         WARN_ON(root->log_root);
1343         root->log_root = log_root;
1344         root->log_transid = 0;
1345         root->log_transid_committed = -1;
1346         root->last_log_commit = 0;
1347         return 0;
1348 }
1349
1350 static struct btrfs_root *read_tree_root_path(struct btrfs_root *tree_root,
1351                                               struct btrfs_path *path,
1352                                               struct btrfs_key *key)
1353 {
1354         struct btrfs_root *root;
1355         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1356         u64 generation;
1357         int ret;
1358         int level;
1359
1360         root = btrfs_alloc_root(fs_info, key->objectid, GFP_NOFS);
1361         if (!root)
1362                 return ERR_PTR(-ENOMEM);
1363
1364         ret = btrfs_find_root(tree_root, key, path,
1365                               &root->root_item, &root->root_key);
1366         if (ret) {
1367                 if (ret > 0)
1368                         ret = -ENOENT;
1369                 goto fail;
1370         }
1371
1372         generation = btrfs_root_generation(&root->root_item);
1373         level = btrfs_root_level(&root->root_item);
1374         root->node = read_tree_block(fs_info,
1375                                      btrfs_root_bytenr(&root->root_item),
1376                                      key->objectid, generation, level, NULL);
1377         if (IS_ERR(root->node)) {
1378                 ret = PTR_ERR(root->node);
1379                 root->node = NULL;
1380                 goto fail;
1381         }
1382         if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1383                 ret = -EIO;
1384                 goto fail;
1385         }
1386
1387         /*
1388          * For real fs, and not log/reloc trees, root owner must
1389          * match its root node owner
1390          */
1391         if (!test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state) &&
1392             root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1393             root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
1394             root->root_key.objectid != btrfs_header_owner(root->node)) {
1395                 btrfs_crit(fs_info,
1396 "root=%llu block=%llu, tree root owner mismatch, have %llu expect %llu",
1397                            root->root_key.objectid, root->node->start,
1398                            btrfs_header_owner(root->node),
1399                            root->root_key.objectid);
1400                 ret = -EUCLEAN;
1401                 goto fail;
1402         }
1403         root->commit_root = btrfs_root_node(root);
1404         return root;
1405 fail:
1406         btrfs_put_root(root);
1407         return ERR_PTR(ret);
1408 }
1409
1410 struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1411                                         struct btrfs_key *key)
1412 {
1413         struct btrfs_root *root;
1414         struct btrfs_path *path;
1415
1416         path = btrfs_alloc_path();
1417         if (!path)
1418                 return ERR_PTR(-ENOMEM);
1419         root = read_tree_root_path(tree_root, path, key);
1420         btrfs_free_path(path);
1421
1422         return root;
1423 }
1424
1425 /*
1426  * Initialize subvolume root in-memory structure
1427  *
1428  * @anon_dev:   anonymous device to attach to the root, if zero, allocate new
1429  */
1430 static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev)
1431 {
1432         int ret;
1433         unsigned int nofs_flag;
1434
1435         /*
1436          * We might be called under a transaction (e.g. indirect backref
1437          * resolution) which could deadlock if it triggers memory reclaim
1438          */
1439         nofs_flag = memalloc_nofs_save();
1440         ret = btrfs_drew_lock_init(&root->snapshot_lock);
1441         memalloc_nofs_restore(nofs_flag);
1442         if (ret)
1443                 goto fail;
1444
1445         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1446             !btrfs_is_data_reloc_root(root)) {
1447                 set_bit(BTRFS_ROOT_SHAREABLE, &root->state);
1448                 btrfs_check_and_init_root_item(&root->root_item);
1449         }
1450
1451         /*
1452          * Don't assign anonymous block device to roots that are not exposed to
1453          * userspace, the id pool is limited to 1M
1454          */
1455         if (is_fstree(root->root_key.objectid) &&
1456             btrfs_root_refs(&root->root_item) > 0) {
1457                 if (!anon_dev) {
1458                         ret = get_anon_bdev(&root->anon_dev);
1459                         if (ret)
1460                                 goto fail;
1461                 } else {
1462                         root->anon_dev = anon_dev;
1463                 }
1464         }
1465
1466         mutex_lock(&root->objectid_mutex);
1467         ret = btrfs_init_root_free_objectid(root);
1468         if (ret) {
1469                 mutex_unlock(&root->objectid_mutex);
1470                 goto fail;
1471         }
1472
1473         ASSERT(root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
1474
1475         mutex_unlock(&root->objectid_mutex);
1476
1477         return 0;
1478 fail:
1479         /* The caller is responsible to call btrfs_free_fs_root */
1480         return ret;
1481 }
1482
1483 static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1484                                                u64 root_id)
1485 {
1486         struct btrfs_root *root;
1487
1488         spin_lock(&fs_info->fs_roots_radix_lock);
1489         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1490                                  (unsigned long)root_id);
1491         if (root)
1492                 root = btrfs_grab_root(root);
1493         spin_unlock(&fs_info->fs_roots_radix_lock);
1494         return root;
1495 }
1496
1497 static struct btrfs_root *btrfs_get_global_root(struct btrfs_fs_info *fs_info,
1498                                                 u64 objectid)
1499 {
1500         struct btrfs_key key = {
1501                 .objectid = objectid,
1502                 .type = BTRFS_ROOT_ITEM_KEY,
1503                 .offset = 0,
1504         };
1505
1506         if (objectid == BTRFS_ROOT_TREE_OBJECTID)
1507                 return btrfs_grab_root(fs_info->tree_root);
1508         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
1509                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1510         if (objectid == BTRFS_CHUNK_TREE_OBJECTID)
1511                 return btrfs_grab_root(fs_info->chunk_root);
1512         if (objectid == BTRFS_DEV_TREE_OBJECTID)
1513                 return btrfs_grab_root(fs_info->dev_root);
1514         if (objectid == BTRFS_CSUM_TREE_OBJECTID)
1515                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1516         if (objectid == BTRFS_QUOTA_TREE_OBJECTID)
1517                 return btrfs_grab_root(fs_info->quota_root) ?
1518                         fs_info->quota_root : ERR_PTR(-ENOENT);
1519         if (objectid == BTRFS_UUID_TREE_OBJECTID)
1520                 return btrfs_grab_root(fs_info->uuid_root) ?
1521                         fs_info->uuid_root : ERR_PTR(-ENOENT);
1522         if (objectid == BTRFS_BLOCK_GROUP_TREE_OBJECTID)
1523                 return btrfs_grab_root(fs_info->block_group_root) ?
1524                         fs_info->block_group_root : ERR_PTR(-ENOENT);
1525         if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) {
1526                 struct btrfs_root *root = btrfs_global_root(fs_info, &key);
1527
1528                 return btrfs_grab_root(root) ? root : ERR_PTR(-ENOENT);
1529         }
1530         return NULL;
1531 }
1532
1533 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1534                          struct btrfs_root *root)
1535 {
1536         int ret;
1537
1538         ret = radix_tree_preload(GFP_NOFS);
1539         if (ret)
1540                 return ret;
1541
1542         spin_lock(&fs_info->fs_roots_radix_lock);
1543         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1544                                 (unsigned long)root->root_key.objectid,
1545                                 root);
1546         if (ret == 0) {
1547                 btrfs_grab_root(root);
1548                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1549         }
1550         spin_unlock(&fs_info->fs_roots_radix_lock);
1551         radix_tree_preload_end();
1552
1553         return ret;
1554 }
1555
1556 void btrfs_check_leaked_roots(struct btrfs_fs_info *fs_info)
1557 {
1558 #ifdef CONFIG_BTRFS_DEBUG
1559         struct btrfs_root *root;
1560
1561         while (!list_empty(&fs_info->allocated_roots)) {
1562                 char buf[BTRFS_ROOT_NAME_BUF_LEN];
1563
1564                 root = list_first_entry(&fs_info->allocated_roots,
1565                                         struct btrfs_root, leak_list);
1566                 btrfs_err(fs_info, "leaked root %s refcount %d",
1567                           btrfs_root_name(&root->root_key, buf),
1568                           refcount_read(&root->refs));
1569                 while (refcount_read(&root->refs) > 1)
1570                         btrfs_put_root(root);
1571                 btrfs_put_root(root);
1572         }
1573 #endif
1574 }
1575
1576 static void free_global_roots(struct btrfs_fs_info *fs_info)
1577 {
1578         struct btrfs_root *root;
1579         struct rb_node *node;
1580
1581         while ((node = rb_first_postorder(&fs_info->global_root_tree)) != NULL) {
1582                 root = rb_entry(node, struct btrfs_root, rb_node);
1583                 rb_erase(&root->rb_node, &fs_info->global_root_tree);
1584                 btrfs_put_root(root);
1585         }
1586 }
1587
1588 void btrfs_free_fs_info(struct btrfs_fs_info *fs_info)
1589 {
1590         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
1591         percpu_counter_destroy(&fs_info->delalloc_bytes);
1592         percpu_counter_destroy(&fs_info->ordered_bytes);
1593         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
1594         btrfs_free_csum_hash(fs_info);
1595         btrfs_free_stripe_hash_table(fs_info);
1596         btrfs_free_ref_cache(fs_info);
1597         kfree(fs_info->balance_ctl);
1598         kfree(fs_info->delayed_root);
1599         free_global_roots(fs_info);
1600         btrfs_put_root(fs_info->tree_root);
1601         btrfs_put_root(fs_info->chunk_root);
1602         btrfs_put_root(fs_info->dev_root);
1603         btrfs_put_root(fs_info->quota_root);
1604         btrfs_put_root(fs_info->uuid_root);
1605         btrfs_put_root(fs_info->fs_root);
1606         btrfs_put_root(fs_info->data_reloc_root);
1607         btrfs_put_root(fs_info->block_group_root);
1608         btrfs_check_leaked_roots(fs_info);
1609         btrfs_extent_buffer_leak_debug_check(fs_info);
1610         kfree(fs_info->super_copy);
1611         kfree(fs_info->super_for_commit);
1612         kfree(fs_info->subpage_info);
1613         kvfree(fs_info);
1614 }
1615
1616
1617 /*
1618  * Get an in-memory reference of a root structure.
1619  *
1620  * For essential trees like root/extent tree, we grab it from fs_info directly.
1621  * For subvolume trees, we check the cached filesystem roots first. If not
1622  * found, then read it from disk and add it to cached fs roots.
1623  *
1624  * Caller should release the root by calling btrfs_put_root() after the usage.
1625  *
1626  * NOTE: Reloc and log trees can't be read by this function as they share the
1627  *       same root objectid.
1628  *
1629  * @objectid:   root id
1630  * @anon_dev:   preallocated anonymous block device number for new roots,
1631  *              pass 0 for new allocation.
1632  * @check_ref:  whether to check root item references, If true, return -ENOENT
1633  *              for orphan roots
1634  */
1635 static struct btrfs_root *btrfs_get_root_ref(struct btrfs_fs_info *fs_info,
1636                                              u64 objectid, dev_t anon_dev,
1637                                              bool check_ref)
1638 {
1639         struct btrfs_root *root;
1640         struct btrfs_path *path;
1641         struct btrfs_key key;
1642         int ret;
1643
1644         root = btrfs_get_global_root(fs_info, objectid);
1645         if (root)
1646                 return root;
1647 again:
1648         root = btrfs_lookup_fs_root(fs_info, objectid);
1649         if (root) {
1650                 /* Shouldn't get preallocated anon_dev for cached roots */
1651                 ASSERT(!anon_dev);
1652                 if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1653                         btrfs_put_root(root);
1654                         return ERR_PTR(-ENOENT);
1655                 }
1656                 return root;
1657         }
1658
1659         key.objectid = objectid;
1660         key.type = BTRFS_ROOT_ITEM_KEY;
1661         key.offset = (u64)-1;
1662         root = btrfs_read_tree_root(fs_info->tree_root, &key);
1663         if (IS_ERR(root))
1664                 return root;
1665
1666         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1667                 ret = -ENOENT;
1668                 goto fail;
1669         }
1670
1671         ret = btrfs_init_fs_root(root, anon_dev);
1672         if (ret)
1673                 goto fail;
1674
1675         path = btrfs_alloc_path();
1676         if (!path) {
1677                 ret = -ENOMEM;
1678                 goto fail;
1679         }
1680         key.objectid = BTRFS_ORPHAN_OBJECTID;
1681         key.type = BTRFS_ORPHAN_ITEM_KEY;
1682         key.offset = objectid;
1683
1684         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1685         btrfs_free_path(path);
1686         if (ret < 0)
1687                 goto fail;
1688         if (ret == 0)
1689                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1690
1691         ret = btrfs_insert_fs_root(fs_info, root);
1692         if (ret) {
1693                 if (ret == -EEXIST) {
1694                         btrfs_put_root(root);
1695                         goto again;
1696                 }
1697                 goto fail;
1698         }
1699         return root;
1700 fail:
1701         /*
1702          * If our caller provided us an anonymous device, then it's his
1703          * responsibility to free it in case we fail. So we have to set our
1704          * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root()
1705          * and once again by our caller.
1706          */
1707         if (anon_dev)
1708                 root->anon_dev = 0;
1709         btrfs_put_root(root);
1710         return ERR_PTR(ret);
1711 }
1712
1713 /*
1714  * Get in-memory reference of a root structure
1715  *
1716  * @objectid:   tree objectid
1717  * @check_ref:  if set, verify that the tree exists and the item has at least
1718  *              one reference
1719  */
1720 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1721                                      u64 objectid, bool check_ref)
1722 {
1723         return btrfs_get_root_ref(fs_info, objectid, 0, check_ref);
1724 }
1725
1726 /*
1727  * Get in-memory reference of a root structure, created as new, optionally pass
1728  * the anonymous block device id
1729  *
1730  * @objectid:   tree objectid
1731  * @anon_dev:   if zero, allocate a new anonymous block device or use the
1732  *              parameter value
1733  */
1734 struct btrfs_root *btrfs_get_new_fs_root(struct btrfs_fs_info *fs_info,
1735                                          u64 objectid, dev_t anon_dev)
1736 {
1737         return btrfs_get_root_ref(fs_info, objectid, anon_dev, true);
1738 }
1739
1740 /*
1741  * btrfs_get_fs_root_commit_root - return a root for the given objectid
1742  * @fs_info:    the fs_info
1743  * @objectid:   the objectid we need to lookup
1744  *
1745  * This is exclusively used for backref walking, and exists specifically because
1746  * of how qgroups does lookups.  Qgroups will do a backref lookup at delayed ref
1747  * creation time, which means we may have to read the tree_root in order to look
1748  * up a fs root that is not in memory.  If the root is not in memory we will
1749  * read the tree root commit root and look up the fs root from there.  This is a
1750  * temporary root, it will not be inserted into the radix tree as it doesn't
1751  * have the most uptodate information, it'll simply be discarded once the
1752  * backref code is finished using the root.
1753  */
1754 struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info,
1755                                                  struct btrfs_path *path,
1756                                                  u64 objectid)
1757 {
1758         struct btrfs_root *root;
1759         struct btrfs_key key;
1760
1761         ASSERT(path->search_commit_root && path->skip_locking);
1762
1763         /*
1764          * This can return -ENOENT if we ask for a root that doesn't exist, but
1765          * since this is called via the backref walking code we won't be looking
1766          * up a root that doesn't exist, unless there's corruption.  So if root
1767          * != NULL just return it.
1768          */
1769         root = btrfs_get_global_root(fs_info, objectid);
1770         if (root)
1771                 return root;
1772
1773         root = btrfs_lookup_fs_root(fs_info, objectid);
1774         if (root)
1775                 return root;
1776
1777         key.objectid = objectid;
1778         key.type = BTRFS_ROOT_ITEM_KEY;
1779         key.offset = (u64)-1;
1780         root = read_tree_root_path(fs_info->tree_root, path, &key);
1781         btrfs_release_path(path);
1782
1783         return root;
1784 }
1785
1786 static int cleaner_kthread(void *arg)
1787 {
1788         struct btrfs_fs_info *fs_info = arg;
1789         int again;
1790
1791         while (1) {
1792                 again = 0;
1793
1794                 set_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1795
1796                 /* Make the cleaner go to sleep early. */
1797                 if (btrfs_need_cleaner_sleep(fs_info))
1798                         goto sleep;
1799
1800                 /*
1801                  * Do not do anything if we might cause open_ctree() to block
1802                  * before we have finished mounting the filesystem.
1803                  */
1804                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1805                         goto sleep;
1806
1807                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1808                         goto sleep;
1809
1810                 /*
1811                  * Avoid the problem that we change the status of the fs
1812                  * during the above check and trylock.
1813                  */
1814                 if (btrfs_need_cleaner_sleep(fs_info)) {
1815                         mutex_unlock(&fs_info->cleaner_mutex);
1816                         goto sleep;
1817                 }
1818
1819                 btrfs_run_delayed_iputs(fs_info);
1820
1821                 again = btrfs_clean_one_deleted_snapshot(fs_info);
1822                 mutex_unlock(&fs_info->cleaner_mutex);
1823
1824                 /*
1825                  * The defragger has dealt with the R/O remount and umount,
1826                  * needn't do anything special here.
1827                  */
1828                 btrfs_run_defrag_inodes(fs_info);
1829
1830                 /*
1831                  * Acquires fs_info->reclaim_bgs_lock to avoid racing
1832                  * with relocation (btrfs_relocate_chunk) and relocation
1833                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1834                  * after acquiring fs_info->reclaim_bgs_lock. So we
1835                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1836                  * unused block groups.
1837                  */
1838                 btrfs_delete_unused_bgs(fs_info);
1839
1840                 /*
1841                  * Reclaim block groups in the reclaim_bgs list after we deleted
1842                  * all unused block_groups. This possibly gives us some more free
1843                  * space.
1844                  */
1845                 btrfs_reclaim_bgs(fs_info);
1846 sleep:
1847                 clear_and_wake_up_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1848                 if (kthread_should_park())
1849                         kthread_parkme();
1850                 if (kthread_should_stop())
1851                         return 0;
1852                 if (!again) {
1853                         set_current_state(TASK_INTERRUPTIBLE);
1854                         schedule();
1855                         __set_current_state(TASK_RUNNING);
1856                 }
1857         }
1858 }
1859
1860 static int transaction_kthread(void *arg)
1861 {
1862         struct btrfs_root *root = arg;
1863         struct btrfs_fs_info *fs_info = root->fs_info;
1864         struct btrfs_trans_handle *trans;
1865         struct btrfs_transaction *cur;
1866         u64 transid;
1867         time64_t delta;
1868         unsigned long delay;
1869         bool cannot_commit;
1870
1871         do {
1872                 cannot_commit = false;
1873                 delay = msecs_to_jiffies(fs_info->commit_interval * 1000);
1874                 mutex_lock(&fs_info->transaction_kthread_mutex);
1875
1876                 spin_lock(&fs_info->trans_lock);
1877                 cur = fs_info->running_transaction;
1878                 if (!cur) {
1879                         spin_unlock(&fs_info->trans_lock);
1880                         goto sleep;
1881                 }
1882
1883                 delta = ktime_get_seconds() - cur->start_time;
1884                 if (!test_and_clear_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags) &&
1885                     cur->state < TRANS_STATE_COMMIT_START &&
1886                     delta < fs_info->commit_interval) {
1887                         spin_unlock(&fs_info->trans_lock);
1888                         delay -= msecs_to_jiffies((delta - 1) * 1000);
1889                         delay = min(delay,
1890                                     msecs_to_jiffies(fs_info->commit_interval * 1000));
1891                         goto sleep;
1892                 }
1893                 transid = cur->transid;
1894                 spin_unlock(&fs_info->trans_lock);
1895
1896                 /* If the file system is aborted, this will always fail. */
1897                 trans = btrfs_attach_transaction(root);
1898                 if (IS_ERR(trans)) {
1899                         if (PTR_ERR(trans) != -ENOENT)
1900                                 cannot_commit = true;
1901                         goto sleep;
1902                 }
1903                 if (transid == trans->transid) {
1904                         btrfs_commit_transaction(trans);
1905                 } else {
1906                         btrfs_end_transaction(trans);
1907                 }
1908 sleep:
1909                 wake_up_process(fs_info->cleaner_kthread);
1910                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1911
1912                 if (BTRFS_FS_ERROR(fs_info))
1913                         btrfs_cleanup_transaction(fs_info);
1914                 if (!kthread_should_stop() &&
1915                                 (!btrfs_transaction_blocked(fs_info) ||
1916                                  cannot_commit))
1917                         schedule_timeout_interruptible(delay);
1918         } while (!kthread_should_stop());
1919         return 0;
1920 }
1921
1922 /*
1923  * This will find the highest generation in the array of root backups.  The
1924  * index of the highest array is returned, or -EINVAL if we can't find
1925  * anything.
1926  *
1927  * We check to make sure the array is valid by comparing the
1928  * generation of the latest  root in the array with the generation
1929  * in the super block.  If they don't match we pitch it.
1930  */
1931 static int find_newest_super_backup(struct btrfs_fs_info *info)
1932 {
1933         const u64 newest_gen = btrfs_super_generation(info->super_copy);
1934         u64 cur;
1935         struct btrfs_root_backup *root_backup;
1936         int i;
1937
1938         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1939                 root_backup = info->super_copy->super_roots + i;
1940                 cur = btrfs_backup_tree_root_gen(root_backup);
1941                 if (cur == newest_gen)
1942                         return i;
1943         }
1944
1945         return -EINVAL;
1946 }
1947
1948 /*
1949  * copy all the root pointers into the super backup array.
1950  * this will bump the backup pointer by one when it is
1951  * done
1952  */
1953 static void backup_super_roots(struct btrfs_fs_info *info)
1954 {
1955         const int next_backup = info->backup_root_index;
1956         struct btrfs_root_backup *root_backup;
1957
1958         root_backup = info->super_for_commit->super_roots + next_backup;
1959
1960         /*
1961          * make sure all of our padding and empty slots get zero filled
1962          * regardless of which ones we use today
1963          */
1964         memset(root_backup, 0, sizeof(*root_backup));
1965
1966         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1967
1968         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1969         btrfs_set_backup_tree_root_gen(root_backup,
1970                                btrfs_header_generation(info->tree_root->node));
1971
1972         btrfs_set_backup_tree_root_level(root_backup,
1973                                btrfs_header_level(info->tree_root->node));
1974
1975         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1976         btrfs_set_backup_chunk_root_gen(root_backup,
1977                                btrfs_header_generation(info->chunk_root->node));
1978         btrfs_set_backup_chunk_root_level(root_backup,
1979                                btrfs_header_level(info->chunk_root->node));
1980
1981         if (!btrfs_fs_compat_ro(info, BLOCK_GROUP_TREE)) {
1982                 struct btrfs_root *extent_root = btrfs_extent_root(info, 0);
1983                 struct btrfs_root *csum_root = btrfs_csum_root(info, 0);
1984
1985                 btrfs_set_backup_extent_root(root_backup,
1986                                              extent_root->node->start);
1987                 btrfs_set_backup_extent_root_gen(root_backup,
1988                                 btrfs_header_generation(extent_root->node));
1989                 btrfs_set_backup_extent_root_level(root_backup,
1990                                         btrfs_header_level(extent_root->node));
1991
1992                 btrfs_set_backup_csum_root(root_backup, csum_root->node->start);
1993                 btrfs_set_backup_csum_root_gen(root_backup,
1994                                                btrfs_header_generation(csum_root->node));
1995                 btrfs_set_backup_csum_root_level(root_backup,
1996                                                  btrfs_header_level(csum_root->node));
1997         }
1998
1999         /*
2000          * we might commit during log recovery, which happens before we set
2001          * the fs_root.  Make sure it is valid before we fill it in.
2002          */
2003         if (info->fs_root && info->fs_root->node) {
2004                 btrfs_set_backup_fs_root(root_backup,
2005                                          info->fs_root->node->start);
2006                 btrfs_set_backup_fs_root_gen(root_backup,
2007                                btrfs_header_generation(info->fs_root->node));
2008                 btrfs_set_backup_fs_root_level(root_backup,
2009                                btrfs_header_level(info->fs_root->node));
2010         }
2011
2012         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
2013         btrfs_set_backup_dev_root_gen(root_backup,
2014                                btrfs_header_generation(info->dev_root->node));
2015         btrfs_set_backup_dev_root_level(root_backup,
2016                                        btrfs_header_level(info->dev_root->node));
2017
2018         btrfs_set_backup_total_bytes(root_backup,
2019                              btrfs_super_total_bytes(info->super_copy));
2020         btrfs_set_backup_bytes_used(root_backup,
2021                              btrfs_super_bytes_used(info->super_copy));
2022         btrfs_set_backup_num_devices(root_backup,
2023                              btrfs_super_num_devices(info->super_copy));
2024
2025         /*
2026          * if we don't copy this out to the super_copy, it won't get remembered
2027          * for the next commit
2028          */
2029         memcpy(&info->super_copy->super_roots,
2030                &info->super_for_commit->super_roots,
2031                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
2032 }
2033
2034 /*
2035  * read_backup_root - Reads a backup root based on the passed priority. Prio 0
2036  * is the newest, prio 1/2/3 are 2nd newest/3rd newest/4th (oldest) backup roots
2037  *
2038  * fs_info - filesystem whose backup roots need to be read
2039  * priority - priority of backup root required
2040  *
2041  * Returns backup root index on success and -EINVAL otherwise.
2042  */
2043 static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority)
2044 {
2045         int backup_index = find_newest_super_backup(fs_info);
2046         struct btrfs_super_block *super = fs_info->super_copy;
2047         struct btrfs_root_backup *root_backup;
2048
2049         if (priority < BTRFS_NUM_BACKUP_ROOTS && backup_index >= 0) {
2050                 if (priority == 0)
2051                         return backup_index;
2052
2053                 backup_index = backup_index + BTRFS_NUM_BACKUP_ROOTS - priority;
2054                 backup_index %= BTRFS_NUM_BACKUP_ROOTS;
2055         } else {
2056                 return -EINVAL;
2057         }
2058
2059         root_backup = super->super_roots + backup_index;
2060
2061         btrfs_set_super_generation(super,
2062                                    btrfs_backup_tree_root_gen(root_backup));
2063         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
2064         btrfs_set_super_root_level(super,
2065                                    btrfs_backup_tree_root_level(root_backup));
2066         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
2067
2068         /*
2069          * Fixme: the total bytes and num_devices need to match or we should
2070          * need a fsck
2071          */
2072         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
2073         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
2074
2075         return backup_index;
2076 }
2077
2078 /* helper to cleanup workers */
2079 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
2080 {
2081         btrfs_destroy_workqueue(fs_info->fixup_workers);
2082         btrfs_destroy_workqueue(fs_info->delalloc_workers);
2083         btrfs_destroy_workqueue(fs_info->hipri_workers);
2084         btrfs_destroy_workqueue(fs_info->workers);
2085         if (fs_info->endio_workers)
2086                 destroy_workqueue(fs_info->endio_workers);
2087         if (fs_info->endio_raid56_workers)
2088                 destroy_workqueue(fs_info->endio_raid56_workers);
2089         if (fs_info->rmw_workers)
2090                 destroy_workqueue(fs_info->rmw_workers);
2091         if (fs_info->compressed_write_workers)
2092                 destroy_workqueue(fs_info->compressed_write_workers);
2093         btrfs_destroy_workqueue(fs_info->endio_write_workers);
2094         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
2095         btrfs_destroy_workqueue(fs_info->delayed_workers);
2096         btrfs_destroy_workqueue(fs_info->caching_workers);
2097         btrfs_destroy_workqueue(fs_info->flush_workers);
2098         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
2099         if (fs_info->discard_ctl.discard_workers)
2100                 destroy_workqueue(fs_info->discard_ctl.discard_workers);
2101         /*
2102          * Now that all other work queues are destroyed, we can safely destroy
2103          * the queues used for metadata I/O, since tasks from those other work
2104          * queues can do metadata I/O operations.
2105          */
2106         if (fs_info->endio_meta_workers)
2107                 destroy_workqueue(fs_info->endio_meta_workers);
2108 }
2109
2110 static void free_root_extent_buffers(struct btrfs_root *root)
2111 {
2112         if (root) {
2113                 free_extent_buffer(root->node);
2114                 free_extent_buffer(root->commit_root);
2115                 root->node = NULL;
2116                 root->commit_root = NULL;
2117         }
2118 }
2119
2120 static void free_global_root_pointers(struct btrfs_fs_info *fs_info)
2121 {
2122         struct btrfs_root *root, *tmp;
2123
2124         rbtree_postorder_for_each_entry_safe(root, tmp,
2125                                              &fs_info->global_root_tree,
2126                                              rb_node)
2127                 free_root_extent_buffers(root);
2128 }
2129
2130 /* helper to cleanup tree roots */
2131 static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root)
2132 {
2133         free_root_extent_buffers(info->tree_root);
2134
2135         free_global_root_pointers(info);
2136         free_root_extent_buffers(info->dev_root);
2137         free_root_extent_buffers(info->quota_root);
2138         free_root_extent_buffers(info->uuid_root);
2139         free_root_extent_buffers(info->fs_root);
2140         free_root_extent_buffers(info->data_reloc_root);
2141         free_root_extent_buffers(info->block_group_root);
2142         if (free_chunk_root)
2143                 free_root_extent_buffers(info->chunk_root);
2144 }
2145
2146 void btrfs_put_root(struct btrfs_root *root)
2147 {
2148         if (!root)
2149                 return;
2150
2151         if (refcount_dec_and_test(&root->refs)) {
2152                 WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
2153                 WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state));
2154                 if (root->anon_dev)
2155                         free_anon_bdev(root->anon_dev);
2156                 btrfs_drew_lock_destroy(&root->snapshot_lock);
2157                 free_root_extent_buffers(root);
2158 #ifdef CONFIG_BTRFS_DEBUG
2159                 spin_lock(&root->fs_info->fs_roots_radix_lock);
2160                 list_del_init(&root->leak_list);
2161                 spin_unlock(&root->fs_info->fs_roots_radix_lock);
2162 #endif
2163                 kfree(root);
2164         }
2165 }
2166
2167 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
2168 {
2169         int ret;
2170         struct btrfs_root *gang[8];
2171         int i;
2172
2173         while (!list_empty(&fs_info->dead_roots)) {
2174                 gang[0] = list_entry(fs_info->dead_roots.next,
2175                                      struct btrfs_root, root_list);
2176                 list_del(&gang[0]->root_list);
2177
2178                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state))
2179                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
2180                 btrfs_put_root(gang[0]);
2181         }
2182
2183         while (1) {
2184                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
2185                                              (void **)gang, 0,
2186                                              ARRAY_SIZE(gang));
2187                 if (!ret)
2188                         break;
2189                 for (i = 0; i < ret; i++)
2190                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2191         }
2192 }
2193
2194 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2195 {
2196         mutex_init(&fs_info->scrub_lock);
2197         atomic_set(&fs_info->scrubs_running, 0);
2198         atomic_set(&fs_info->scrub_pause_req, 0);
2199         atomic_set(&fs_info->scrubs_paused, 0);
2200         atomic_set(&fs_info->scrub_cancel_req, 0);
2201         init_waitqueue_head(&fs_info->scrub_pause_wait);
2202         refcount_set(&fs_info->scrub_workers_refcnt, 0);
2203 }
2204
2205 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2206 {
2207         spin_lock_init(&fs_info->balance_lock);
2208         mutex_init(&fs_info->balance_mutex);
2209         atomic_set(&fs_info->balance_pause_req, 0);
2210         atomic_set(&fs_info->balance_cancel_req, 0);
2211         fs_info->balance_ctl = NULL;
2212         init_waitqueue_head(&fs_info->balance_wait_q);
2213         atomic_set(&fs_info->reloc_cancel_req, 0);
2214 }
2215
2216 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2217 {
2218         struct inode *inode = fs_info->btree_inode;
2219         unsigned long hash = btrfs_inode_hash(BTRFS_BTREE_INODE_OBJECTID,
2220                                               fs_info->tree_root);
2221
2222         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2223         set_nlink(inode, 1);
2224         /*
2225          * we set the i_size on the btree inode to the max possible int.
2226          * the real end of the address space is determined by all of
2227          * the devices in the system
2228          */
2229         inode->i_size = OFFSET_MAX;
2230         inode->i_mapping->a_ops = &btree_aops;
2231
2232         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2233         extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
2234                             IO_TREE_BTREE_INODE_IO, NULL);
2235         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2236
2237         BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root);
2238         BTRFS_I(inode)->location.objectid = BTRFS_BTREE_INODE_OBJECTID;
2239         BTRFS_I(inode)->location.type = 0;
2240         BTRFS_I(inode)->location.offset = 0;
2241         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2242         __insert_inode_hash(inode, hash);
2243 }
2244
2245 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2246 {
2247         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2248         init_rwsem(&fs_info->dev_replace.rwsem);
2249         init_waitqueue_head(&fs_info->dev_replace.replace_wait);
2250 }
2251
2252 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2253 {
2254         spin_lock_init(&fs_info->qgroup_lock);
2255         mutex_init(&fs_info->qgroup_ioctl_lock);
2256         fs_info->qgroup_tree = RB_ROOT;
2257         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2258         fs_info->qgroup_seq = 1;
2259         fs_info->qgroup_ulist = NULL;
2260         fs_info->qgroup_rescan_running = false;
2261         fs_info->qgroup_drop_subtree_thres = BTRFS_MAX_LEVEL;
2262         mutex_init(&fs_info->qgroup_rescan_lock);
2263 }
2264
2265 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
2266 {
2267         u32 max_active = fs_info->thread_pool_size;
2268         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2269
2270         fs_info->workers =
2271                 btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16);
2272         fs_info->hipri_workers =
2273                 btrfs_alloc_workqueue(fs_info, "worker-high",
2274                                       flags | WQ_HIGHPRI, max_active, 16);
2275
2276         fs_info->delalloc_workers =
2277                 btrfs_alloc_workqueue(fs_info, "delalloc",
2278                                       flags, max_active, 2);
2279
2280         fs_info->flush_workers =
2281                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2282                                       flags, max_active, 0);
2283
2284         fs_info->caching_workers =
2285                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2286
2287         fs_info->fixup_workers =
2288                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2289
2290         fs_info->endio_workers =
2291                 alloc_workqueue("btrfs-endio", flags, max_active);
2292         fs_info->endio_meta_workers =
2293                 alloc_workqueue("btrfs-endio-meta", flags, max_active);
2294         fs_info->endio_raid56_workers =
2295                 alloc_workqueue("btrfs-endio-raid56", flags, max_active);
2296         fs_info->rmw_workers = alloc_workqueue("btrfs-rmw", flags, max_active);
2297         fs_info->endio_write_workers =
2298                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2299                                       max_active, 2);
2300         fs_info->compressed_write_workers =
2301                 alloc_workqueue("btrfs-compressed-write", flags, max_active);
2302         fs_info->endio_freespace_worker =
2303                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2304                                       max_active, 0);
2305         fs_info->delayed_workers =
2306                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2307                                       max_active, 0);
2308         fs_info->qgroup_rescan_workers =
2309                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2310         fs_info->discard_ctl.discard_workers =
2311                 alloc_workqueue("btrfs_discard", WQ_UNBOUND | WQ_FREEZABLE, 1);
2312
2313         if (!(fs_info->workers && fs_info->hipri_workers &&
2314               fs_info->delalloc_workers && fs_info->flush_workers &&
2315               fs_info->endio_workers && fs_info->endio_meta_workers &&
2316               fs_info->compressed_write_workers &&
2317               fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
2318               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2319               fs_info->caching_workers && fs_info->fixup_workers &&
2320               fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
2321               fs_info->discard_ctl.discard_workers)) {
2322                 return -ENOMEM;
2323         }
2324
2325         return 0;
2326 }
2327
2328 static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
2329 {
2330         struct crypto_shash *csum_shash;
2331         const char *csum_driver = btrfs_super_csum_driver(csum_type);
2332
2333         csum_shash = crypto_alloc_shash(csum_driver, 0, 0);
2334
2335         if (IS_ERR(csum_shash)) {
2336                 btrfs_err(fs_info, "error allocating %s hash for checksum",
2337                           csum_driver);
2338                 return PTR_ERR(csum_shash);
2339         }
2340
2341         fs_info->csum_shash = csum_shash;
2342
2343         btrfs_info(fs_info, "using %s (%s) checksum algorithm",
2344                         btrfs_super_csum_name(csum_type),
2345                         crypto_shash_driver_name(csum_shash));
2346         return 0;
2347 }
2348
2349 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2350                             struct btrfs_fs_devices *fs_devices)
2351 {
2352         int ret;
2353         struct btrfs_root *log_tree_root;
2354         struct btrfs_super_block *disk_super = fs_info->super_copy;
2355         u64 bytenr = btrfs_super_log_root(disk_super);
2356         int level = btrfs_super_log_root_level(disk_super);
2357
2358         if (fs_devices->rw_devices == 0) {
2359                 btrfs_warn(fs_info, "log replay required on RO media");
2360                 return -EIO;
2361         }
2362
2363         log_tree_root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID,
2364                                          GFP_KERNEL);
2365         if (!log_tree_root)
2366                 return -ENOMEM;
2367
2368         log_tree_root->node = read_tree_block(fs_info, bytenr,
2369                                               BTRFS_TREE_LOG_OBJECTID,
2370                                               fs_info->generation + 1, level,
2371                                               NULL);
2372         if (IS_ERR(log_tree_root->node)) {
2373                 btrfs_warn(fs_info, "failed to read log tree");
2374                 ret = PTR_ERR(log_tree_root->node);
2375                 log_tree_root->node = NULL;
2376                 btrfs_put_root(log_tree_root);
2377                 return ret;
2378         }
2379         if (!extent_buffer_uptodate(log_tree_root->node)) {
2380                 btrfs_err(fs_info, "failed to read log tree");
2381                 btrfs_put_root(log_tree_root);
2382                 return -EIO;
2383         }
2384
2385         /* returns with log_tree_root freed on success */
2386         ret = btrfs_recover_log_trees(log_tree_root);
2387         if (ret) {
2388                 btrfs_handle_fs_error(fs_info, ret,
2389                                       "Failed to recover log tree");
2390                 btrfs_put_root(log_tree_root);
2391                 return ret;
2392         }
2393
2394         if (sb_rdonly(fs_info->sb)) {
2395                 ret = btrfs_commit_super(fs_info);
2396                 if (ret)
2397                         return ret;
2398         }
2399
2400         return 0;
2401 }
2402
2403 static int load_global_roots_objectid(struct btrfs_root *tree_root,
2404                                       struct btrfs_path *path, u64 objectid,
2405                                       const char *name)
2406 {
2407         struct btrfs_fs_info *fs_info = tree_root->fs_info;
2408         struct btrfs_root *root;
2409         u64 max_global_id = 0;
2410         int ret;
2411         struct btrfs_key key = {
2412                 .objectid = objectid,
2413                 .type = BTRFS_ROOT_ITEM_KEY,
2414                 .offset = 0,
2415         };
2416         bool found = false;
2417
2418         /* If we have IGNOREDATACSUMS skip loading these roots. */
2419         if (objectid == BTRFS_CSUM_TREE_OBJECTID &&
2420             btrfs_test_opt(fs_info, IGNOREDATACSUMS)) {
2421                 set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2422                 return 0;
2423         }
2424
2425         while (1) {
2426                 ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0);
2427                 if (ret < 0)
2428                         break;
2429
2430                 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2431                         ret = btrfs_next_leaf(tree_root, path);
2432                         if (ret) {
2433                                 if (ret > 0)
2434                                         ret = 0;
2435                                 break;
2436                         }
2437                 }
2438                 ret = 0;
2439
2440                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2441                 if (key.objectid != objectid)
2442                         break;
2443                 btrfs_release_path(path);
2444
2445                 /*
2446                  * Just worry about this for extent tree, it'll be the same for
2447                  * everybody.
2448                  */
2449                 if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2450                         max_global_id = max(max_global_id, key.offset);
2451
2452                 found = true;
2453                 root = read_tree_root_path(tree_root, path, &key);
2454                 if (IS_ERR(root)) {
2455                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2456                                 ret = PTR_ERR(root);
2457                         break;
2458                 }
2459                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2460                 ret = btrfs_global_root_insert(root);
2461                 if (ret) {
2462                         btrfs_put_root(root);
2463                         break;
2464                 }
2465                 key.offset++;
2466         }
2467         btrfs_release_path(path);
2468
2469         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2470                 fs_info->nr_global_roots = max_global_id + 1;
2471
2472         if (!found || ret) {
2473                 if (objectid == BTRFS_CSUM_TREE_OBJECTID)
2474                         set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2475
2476                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2477                         ret = ret ? ret : -ENOENT;
2478                 else
2479                         ret = 0;
2480                 btrfs_err(fs_info, "failed to load root %s", name);
2481         }
2482         return ret;
2483 }
2484
2485 static int load_global_roots(struct btrfs_root *tree_root)
2486 {
2487         struct btrfs_path *path;
2488         int ret = 0;
2489
2490         path = btrfs_alloc_path();
2491         if (!path)
2492                 return -ENOMEM;
2493
2494         ret = load_global_roots_objectid(tree_root, path,
2495                                          BTRFS_EXTENT_TREE_OBJECTID, "extent");
2496         if (ret)
2497                 goto out;
2498         ret = load_global_roots_objectid(tree_root, path,
2499                                          BTRFS_CSUM_TREE_OBJECTID, "csum");
2500         if (ret)
2501                 goto out;
2502         if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE))
2503                 goto out;
2504         ret = load_global_roots_objectid(tree_root, path,
2505                                          BTRFS_FREE_SPACE_TREE_OBJECTID,
2506                                          "free space");
2507 out:
2508         btrfs_free_path(path);
2509         return ret;
2510 }
2511
2512 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2513 {
2514         struct btrfs_root *tree_root = fs_info->tree_root;
2515         struct btrfs_root *root;
2516         struct btrfs_key location;
2517         int ret;
2518
2519         BUG_ON(!fs_info->tree_root);
2520
2521         ret = load_global_roots(tree_root);
2522         if (ret)
2523                 return ret;
2524
2525         location.type = BTRFS_ROOT_ITEM_KEY;
2526         location.offset = 0;
2527
2528         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) {
2529                 location.objectid = BTRFS_BLOCK_GROUP_TREE_OBJECTID;
2530                 root = btrfs_read_tree_root(tree_root, &location);
2531                 if (IS_ERR(root)) {
2532                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2533                                 ret = PTR_ERR(root);
2534                                 goto out;
2535                         }
2536                 } else {
2537                         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2538                         fs_info->block_group_root = root;
2539                 }
2540         }
2541
2542         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2543         root = btrfs_read_tree_root(tree_root, &location);
2544         if (IS_ERR(root)) {
2545                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2546                         ret = PTR_ERR(root);
2547                         goto out;
2548                 }
2549         } else {
2550                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2551                 fs_info->dev_root = root;
2552         }
2553         /* Initialize fs_info for all devices in any case */
2554         ret = btrfs_init_devices_late(fs_info);
2555         if (ret)
2556                 goto out;
2557
2558         /*
2559          * This tree can share blocks with some other fs tree during relocation
2560          * and we need a proper setup by btrfs_get_fs_root
2561          */
2562         root = btrfs_get_fs_root(tree_root->fs_info,
2563                                  BTRFS_DATA_RELOC_TREE_OBJECTID, true);
2564         if (IS_ERR(root)) {
2565                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2566                         ret = PTR_ERR(root);
2567                         goto out;
2568                 }
2569         } else {
2570                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2571                 fs_info->data_reloc_root = root;
2572         }
2573
2574         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2575         root = btrfs_read_tree_root(tree_root, &location);
2576         if (!IS_ERR(root)) {
2577                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2578                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2579                 fs_info->quota_root = root;
2580         }
2581
2582         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2583         root = btrfs_read_tree_root(tree_root, &location);
2584         if (IS_ERR(root)) {
2585                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2586                         ret = PTR_ERR(root);
2587                         if (ret != -ENOENT)
2588                                 goto out;
2589                 }
2590         } else {
2591                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2592                 fs_info->uuid_root = root;
2593         }
2594
2595         return 0;
2596 out:
2597         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2598                    location.objectid, ret);
2599         return ret;
2600 }
2601
2602 /*
2603  * Real super block validation
2604  * NOTE: super csum type and incompat features will not be checked here.
2605  *
2606  * @sb:         super block to check
2607  * @mirror_num: the super block number to check its bytenr:
2608  *              0       the primary (1st) sb
2609  *              1, 2    2nd and 3rd backup copy
2610  *             -1       skip bytenr check
2611  */
2612 int btrfs_validate_super(struct btrfs_fs_info *fs_info,
2613                          struct btrfs_super_block *sb, int mirror_num)
2614 {
2615         u64 nodesize = btrfs_super_nodesize(sb);
2616         u64 sectorsize = btrfs_super_sectorsize(sb);
2617         int ret = 0;
2618
2619         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2620                 btrfs_err(fs_info, "no valid FS found");
2621                 ret = -EINVAL;
2622         }
2623         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2624                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2625                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2626                 ret = -EINVAL;
2627         }
2628         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2629                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2630                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2631                 ret = -EINVAL;
2632         }
2633         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2634                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2635                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2636                 ret = -EINVAL;
2637         }
2638         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2639                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2640                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2641                 ret = -EINVAL;
2642         }
2643
2644         /*
2645          * Check sectorsize and nodesize first, other check will need it.
2646          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2647          */
2648         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2649             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2650                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2651                 ret = -EINVAL;
2652         }
2653
2654         /*
2655          * We only support at most two sectorsizes: 4K and PAGE_SIZE.
2656          *
2657          * We can support 16K sectorsize with 64K page size without problem,
2658          * but such sectorsize/pagesize combination doesn't make much sense.
2659          * 4K will be our future standard, PAGE_SIZE is supported from the very
2660          * beginning.
2661          */
2662         if (sectorsize > PAGE_SIZE || (sectorsize != SZ_4K && sectorsize != PAGE_SIZE)) {
2663                 btrfs_err(fs_info,
2664                         "sectorsize %llu not yet supported for page size %lu",
2665                         sectorsize, PAGE_SIZE);
2666                 ret = -EINVAL;
2667         }
2668
2669         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2670             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2671                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2672                 ret = -EINVAL;
2673         }
2674         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2675                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2676                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2677                 ret = -EINVAL;
2678         }
2679
2680         /* Root alignment check */
2681         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2682                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2683                            btrfs_super_root(sb));
2684                 ret = -EINVAL;
2685         }
2686         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2687                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2688                            btrfs_super_chunk_root(sb));
2689                 ret = -EINVAL;
2690         }
2691         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2692                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2693                            btrfs_super_log_root(sb));
2694                 ret = -EINVAL;
2695         }
2696
2697         if (memcmp(fs_info->fs_devices->fsid, fs_info->super_copy->fsid,
2698                    BTRFS_FSID_SIZE)) {
2699                 btrfs_err(fs_info,
2700                 "superblock fsid doesn't match fsid of fs_devices: %pU != %pU",
2701                         fs_info->super_copy->fsid, fs_info->fs_devices->fsid);
2702                 ret = -EINVAL;
2703         }
2704
2705         if (btrfs_fs_incompat(fs_info, METADATA_UUID) &&
2706             memcmp(fs_info->fs_devices->metadata_uuid,
2707                    fs_info->super_copy->metadata_uuid, BTRFS_FSID_SIZE)) {
2708                 btrfs_err(fs_info,
2709 "superblock metadata_uuid doesn't match metadata uuid of fs_devices: %pU != %pU",
2710                         fs_info->super_copy->metadata_uuid,
2711                         fs_info->fs_devices->metadata_uuid);
2712                 ret = -EINVAL;
2713         }
2714
2715         /*
2716          * Artificial requirement for block-group-tree to force newer features
2717          * (free-space-tree, no-holes) so the test matrix is smaller.
2718          */
2719         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
2720             (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
2721              !btrfs_fs_incompat(fs_info, NO_HOLES))) {
2722                 btrfs_err(fs_info,
2723                 "block-group-tree feature requires fres-space-tree and no-holes");
2724                 ret = -EINVAL;
2725         }
2726
2727         if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
2728                    BTRFS_FSID_SIZE) != 0) {
2729                 btrfs_err(fs_info,
2730                         "dev_item UUID does not match metadata fsid: %pU != %pU",
2731                         fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid);
2732                 ret = -EINVAL;
2733         }
2734
2735         /*
2736          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2737          * done later
2738          */
2739         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2740                 btrfs_err(fs_info, "bytes_used is too small %llu",
2741                           btrfs_super_bytes_used(sb));
2742                 ret = -EINVAL;
2743         }
2744         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2745                 btrfs_err(fs_info, "invalid stripesize %u",
2746                           btrfs_super_stripesize(sb));
2747                 ret = -EINVAL;
2748         }
2749         if (btrfs_super_num_devices(sb) > (1UL << 31))
2750                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2751                            btrfs_super_num_devices(sb));
2752         if (btrfs_super_num_devices(sb) == 0) {
2753                 btrfs_err(fs_info, "number of devices is 0");
2754                 ret = -EINVAL;
2755         }
2756
2757         if (mirror_num >= 0 &&
2758             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2759                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2760                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2761                 ret = -EINVAL;
2762         }
2763
2764         /*
2765          * Obvious sys_chunk_array corruptions, it must hold at least one key
2766          * and one chunk
2767          */
2768         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2769                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2770                           btrfs_super_sys_array_size(sb),
2771                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2772                 ret = -EINVAL;
2773         }
2774         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2775                         + sizeof(struct btrfs_chunk)) {
2776                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2777                           btrfs_super_sys_array_size(sb),
2778                           sizeof(struct btrfs_disk_key)
2779                           + sizeof(struct btrfs_chunk));
2780                 ret = -EINVAL;
2781         }
2782
2783         /*
2784          * The generation is a global counter, we'll trust it more than the others
2785          * but it's still possible that it's the one that's wrong.
2786          */
2787         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2788                 btrfs_warn(fs_info,
2789                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2790                         btrfs_super_generation(sb),
2791                         btrfs_super_chunk_root_generation(sb));
2792         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2793             && btrfs_super_cache_generation(sb) != (u64)-1)
2794                 btrfs_warn(fs_info,
2795                         "suspicious: generation < cache_generation: %llu < %llu",
2796                         btrfs_super_generation(sb),
2797                         btrfs_super_cache_generation(sb));
2798
2799         return ret;
2800 }
2801
2802 /*
2803  * Validation of super block at mount time.
2804  * Some checks already done early at mount time, like csum type and incompat
2805  * flags will be skipped.
2806  */
2807 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2808 {
2809         return btrfs_validate_super(fs_info, fs_info->super_copy, 0);
2810 }
2811
2812 /*
2813  * Validation of super block at write time.
2814  * Some checks like bytenr check will be skipped as their values will be
2815  * overwritten soon.
2816  * Extra checks like csum type and incompat flags will be done here.
2817  */
2818 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2819                                       struct btrfs_super_block *sb)
2820 {
2821         int ret;
2822
2823         ret = btrfs_validate_super(fs_info, sb, -1);
2824         if (ret < 0)
2825                 goto out;
2826         if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) {
2827                 ret = -EUCLEAN;
2828                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2829                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2830                 goto out;
2831         }
2832         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2833                 ret = -EUCLEAN;
2834                 btrfs_err(fs_info,
2835                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2836                           btrfs_super_incompat_flags(sb),
2837                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2838                 goto out;
2839         }
2840 out:
2841         if (ret < 0)
2842                 btrfs_err(fs_info,
2843                 "super block corruption detected before writing it to disk");
2844         return ret;
2845 }
2846
2847 static int load_super_root(struct btrfs_root *root, u64 bytenr, u64 gen, int level)
2848 {
2849         int ret = 0;
2850
2851         root->node = read_tree_block(root->fs_info, bytenr,
2852                                      root->root_key.objectid, gen, level, NULL);
2853         if (IS_ERR(root->node)) {
2854                 ret = PTR_ERR(root->node);
2855                 root->node = NULL;
2856                 return ret;
2857         }
2858         if (!extent_buffer_uptodate(root->node)) {
2859                 free_extent_buffer(root->node);
2860                 root->node = NULL;
2861                 return -EIO;
2862         }
2863
2864         btrfs_set_root_node(&root->root_item, root->node);
2865         root->commit_root = btrfs_root_node(root);
2866         btrfs_set_root_refs(&root->root_item, 1);
2867         return ret;
2868 }
2869
2870 static int load_important_roots(struct btrfs_fs_info *fs_info)
2871 {
2872         struct btrfs_super_block *sb = fs_info->super_copy;
2873         u64 gen, bytenr;
2874         int level, ret;
2875
2876         bytenr = btrfs_super_root(sb);
2877         gen = btrfs_super_generation(sb);
2878         level = btrfs_super_root_level(sb);
2879         ret = load_super_root(fs_info->tree_root, bytenr, gen, level);
2880         if (ret) {
2881                 btrfs_warn(fs_info, "couldn't read tree root");
2882                 return ret;
2883         }
2884         return 0;
2885 }
2886
2887 static int __cold init_tree_roots(struct btrfs_fs_info *fs_info)
2888 {
2889         int backup_index = find_newest_super_backup(fs_info);
2890         struct btrfs_super_block *sb = fs_info->super_copy;
2891         struct btrfs_root *tree_root = fs_info->tree_root;
2892         bool handle_error = false;
2893         int ret = 0;
2894         int i;
2895
2896         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
2897                 if (handle_error) {
2898                         if (!IS_ERR(tree_root->node))
2899                                 free_extent_buffer(tree_root->node);
2900                         tree_root->node = NULL;
2901
2902                         if (!btrfs_test_opt(fs_info, USEBACKUPROOT))
2903                                 break;
2904
2905                         free_root_pointers(fs_info, 0);
2906
2907                         /*
2908                          * Don't use the log in recovery mode, it won't be
2909                          * valid
2910                          */
2911                         btrfs_set_super_log_root(sb, 0);
2912
2913                         /* We can't trust the free space cache either */
2914                         btrfs_set_opt(fs_info->mount_opt, CLEAR_CACHE);
2915
2916                         ret = read_backup_root(fs_info, i);
2917                         backup_index = ret;
2918                         if (ret < 0)
2919                                 return ret;
2920                 }
2921
2922                 ret = load_important_roots(fs_info);
2923                 if (ret) {
2924                         handle_error = true;
2925                         continue;
2926                 }
2927
2928                 /*
2929                  * No need to hold btrfs_root::objectid_mutex since the fs
2930                  * hasn't been fully initialised and we are the only user
2931                  */
2932                 ret = btrfs_init_root_free_objectid(tree_root);
2933                 if (ret < 0) {
2934                         handle_error = true;
2935                         continue;
2936                 }
2937
2938                 ASSERT(tree_root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
2939
2940                 ret = btrfs_read_roots(fs_info);
2941                 if (ret < 0) {
2942                         handle_error = true;
2943                         continue;
2944                 }
2945
2946                 /* All successful */
2947                 fs_info->generation = btrfs_header_generation(tree_root->node);
2948                 fs_info->last_trans_committed = fs_info->generation;
2949                 fs_info->last_reloc_trans = 0;
2950
2951                 /* Always begin writing backup roots after the one being used */
2952                 if (backup_index < 0) {
2953                         fs_info->backup_root_index = 0;
2954                 } else {
2955                         fs_info->backup_root_index = backup_index + 1;
2956                         fs_info->backup_root_index %= BTRFS_NUM_BACKUP_ROOTS;
2957                 }
2958                 break;
2959         }
2960
2961         return ret;
2962 }
2963
2964 void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
2965 {
2966         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2967         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2968         INIT_LIST_HEAD(&fs_info->trans_list);
2969         INIT_LIST_HEAD(&fs_info->dead_roots);
2970         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2971         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2972         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2973         spin_lock_init(&fs_info->delalloc_root_lock);
2974         spin_lock_init(&fs_info->trans_lock);
2975         spin_lock_init(&fs_info->fs_roots_radix_lock);
2976         spin_lock_init(&fs_info->delayed_iput_lock);
2977         spin_lock_init(&fs_info->defrag_inodes_lock);
2978         spin_lock_init(&fs_info->super_lock);
2979         spin_lock_init(&fs_info->buffer_lock);
2980         spin_lock_init(&fs_info->unused_bgs_lock);
2981         spin_lock_init(&fs_info->treelog_bg_lock);
2982         spin_lock_init(&fs_info->zone_active_bgs_lock);
2983         spin_lock_init(&fs_info->relocation_bg_lock);
2984         rwlock_init(&fs_info->tree_mod_log_lock);
2985         rwlock_init(&fs_info->global_root_lock);
2986         mutex_init(&fs_info->unused_bg_unpin_mutex);
2987         mutex_init(&fs_info->reclaim_bgs_lock);
2988         mutex_init(&fs_info->reloc_mutex);
2989         mutex_init(&fs_info->delalloc_root_mutex);
2990         mutex_init(&fs_info->zoned_meta_io_lock);
2991         mutex_init(&fs_info->zoned_data_reloc_io_lock);
2992         seqlock_init(&fs_info->profiles_lock);
2993
2994         btrfs_lockdep_init_map(fs_info, btrfs_trans_num_writers);
2995         btrfs_lockdep_init_map(fs_info, btrfs_trans_num_extwriters);
2996         btrfs_lockdep_init_map(fs_info, btrfs_trans_pending_ordered);
2997         btrfs_lockdep_init_map(fs_info, btrfs_ordered_extent);
2998         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_commit_start,
2999                                      BTRFS_LOCKDEP_TRANS_COMMIT_START);
3000         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_unblocked,
3001                                      BTRFS_LOCKDEP_TRANS_UNBLOCKED);
3002         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_super_committed,
3003                                      BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED);
3004         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_completed,
3005                                      BTRFS_LOCKDEP_TRANS_COMPLETED);
3006
3007         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
3008         INIT_LIST_HEAD(&fs_info->space_info);
3009         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
3010         INIT_LIST_HEAD(&fs_info->unused_bgs);
3011         INIT_LIST_HEAD(&fs_info->reclaim_bgs);
3012         INIT_LIST_HEAD(&fs_info->zone_active_bgs);
3013 #ifdef CONFIG_BTRFS_DEBUG
3014         INIT_LIST_HEAD(&fs_info->allocated_roots);
3015         INIT_LIST_HEAD(&fs_info->allocated_ebs);
3016         spin_lock_init(&fs_info->eb_leak_lock);
3017 #endif
3018         extent_map_tree_init(&fs_info->mapping_tree);
3019         btrfs_init_block_rsv(&fs_info->global_block_rsv,
3020                              BTRFS_BLOCK_RSV_GLOBAL);
3021         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
3022         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
3023         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
3024         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
3025                              BTRFS_BLOCK_RSV_DELOPS);
3026         btrfs_init_block_rsv(&fs_info->delayed_refs_rsv,
3027                              BTRFS_BLOCK_RSV_DELREFS);
3028
3029         atomic_set(&fs_info->async_delalloc_pages, 0);
3030         atomic_set(&fs_info->defrag_running, 0);
3031         atomic_set(&fs_info->nr_delayed_iputs, 0);
3032         atomic64_set(&fs_info->tree_mod_seq, 0);
3033         fs_info->global_root_tree = RB_ROOT;
3034         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
3035         fs_info->metadata_ratio = 0;
3036         fs_info->defrag_inodes = RB_ROOT;
3037         atomic64_set(&fs_info->free_chunk_space, 0);
3038         fs_info->tree_mod_log = RB_ROOT;
3039         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
3040         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
3041         btrfs_init_ref_verify(fs_info);
3042
3043         fs_info->thread_pool_size = min_t(unsigned long,
3044                                           num_online_cpus() + 2, 8);
3045
3046         INIT_LIST_HEAD(&fs_info->ordered_roots);
3047         spin_lock_init(&fs_info->ordered_root_lock);
3048
3049         btrfs_init_scrub(fs_info);
3050 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3051         fs_info->check_integrity_print_mask = 0;
3052 #endif
3053         btrfs_init_balance(fs_info);
3054         btrfs_init_async_reclaim_work(fs_info);
3055
3056         rwlock_init(&fs_info->block_group_cache_lock);
3057         fs_info->block_group_cache_tree = RB_ROOT_CACHED;
3058
3059         extent_io_tree_init(fs_info, &fs_info->excluded_extents,
3060                             IO_TREE_FS_EXCLUDED_EXTENTS, NULL);
3061
3062         mutex_init(&fs_info->ordered_operations_mutex);
3063         mutex_init(&fs_info->tree_log_mutex);
3064         mutex_init(&fs_info->chunk_mutex);
3065         mutex_init(&fs_info->transaction_kthread_mutex);
3066         mutex_init(&fs_info->cleaner_mutex);
3067         mutex_init(&fs_info->ro_block_group_mutex);
3068         init_rwsem(&fs_info->commit_root_sem);
3069         init_rwsem(&fs_info->cleanup_work_sem);
3070         init_rwsem(&fs_info->subvol_sem);
3071         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
3072
3073         btrfs_init_dev_replace_locks(fs_info);
3074         btrfs_init_qgroup(fs_info);
3075         btrfs_discard_init(fs_info);
3076
3077         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
3078         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
3079
3080         init_waitqueue_head(&fs_info->transaction_throttle);
3081         init_waitqueue_head(&fs_info->transaction_wait);
3082         init_waitqueue_head(&fs_info->transaction_blocked_wait);
3083         init_waitqueue_head(&fs_info->async_submit_wait);
3084         init_waitqueue_head(&fs_info->delayed_iputs_wait);
3085
3086         /* Usable values until the real ones are cached from the superblock */
3087         fs_info->nodesize = 4096;
3088         fs_info->sectorsize = 4096;
3089         fs_info->sectorsize_bits = ilog2(4096);
3090         fs_info->stripesize = 4096;
3091
3092         fs_info->max_extent_size = BTRFS_MAX_EXTENT_SIZE;
3093
3094         spin_lock_init(&fs_info->swapfile_pins_lock);
3095         fs_info->swapfile_pins = RB_ROOT;
3096
3097         fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH;
3098         INIT_WORK(&fs_info->reclaim_bgs_work, btrfs_reclaim_bgs_work);
3099 }
3100
3101 static int init_mount_fs_info(struct btrfs_fs_info *fs_info, struct super_block *sb)
3102 {
3103         int ret;
3104
3105         fs_info->sb = sb;
3106         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
3107         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
3108
3109         ret = percpu_counter_init(&fs_info->ordered_bytes, 0, GFP_KERNEL);
3110         if (ret)
3111                 return ret;
3112
3113         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
3114         if (ret)
3115                 return ret;
3116
3117         fs_info->dirty_metadata_batch = PAGE_SIZE *
3118                                         (1 + ilog2(nr_cpu_ids));
3119
3120         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
3121         if (ret)
3122                 return ret;
3123
3124         ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0,
3125                         GFP_KERNEL);
3126         if (ret)
3127                 return ret;
3128
3129         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
3130                                         GFP_KERNEL);
3131         if (!fs_info->delayed_root)
3132                 return -ENOMEM;
3133         btrfs_init_delayed_root(fs_info->delayed_root);
3134
3135         if (sb_rdonly(sb))
3136                 set_bit(BTRFS_FS_STATE_RO, &fs_info->fs_state);
3137
3138         return btrfs_alloc_stripe_hash_table(fs_info);
3139 }
3140
3141 static int btrfs_uuid_rescan_kthread(void *data)
3142 {
3143         struct btrfs_fs_info *fs_info = data;
3144         int ret;
3145
3146         /*
3147          * 1st step is to iterate through the existing UUID tree and
3148          * to delete all entries that contain outdated data.
3149          * 2nd step is to add all missing entries to the UUID tree.
3150          */
3151         ret = btrfs_uuid_tree_iterate(fs_info);
3152         if (ret < 0) {
3153                 if (ret != -EINTR)
3154                         btrfs_warn(fs_info, "iterating uuid_tree failed %d",
3155                                    ret);
3156                 up(&fs_info->uuid_tree_rescan_sem);
3157                 return ret;
3158         }
3159         return btrfs_uuid_scan_kthread(data);
3160 }
3161
3162 static int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info)
3163 {
3164         struct task_struct *task;
3165
3166         down(&fs_info->uuid_tree_rescan_sem);
3167         task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid");
3168         if (IS_ERR(task)) {
3169                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
3170                 btrfs_warn(fs_info, "failed to start uuid_rescan task");
3171                 up(&fs_info->uuid_tree_rescan_sem);
3172                 return PTR_ERR(task);
3173         }
3174
3175         return 0;
3176 }
3177
3178 /*
3179  * Some options only have meaning at mount time and shouldn't persist across
3180  * remounts, or be displayed. Clear these at the end of mount and remount
3181  * code paths.
3182  */
3183 void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info)
3184 {
3185         btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT);
3186         btrfs_clear_opt(fs_info->mount_opt, CLEAR_CACHE);
3187 }
3188
3189 /*
3190  * Mounting logic specific to read-write file systems. Shared by open_ctree
3191  * and btrfs_remount when remounting from read-only to read-write.
3192  */
3193 int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info)
3194 {
3195         int ret;
3196         const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE);
3197         bool clear_free_space_tree = false;
3198
3199         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3200             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3201                 clear_free_space_tree = true;
3202         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3203                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3204                 btrfs_warn(fs_info, "free space tree is invalid");
3205                 clear_free_space_tree = true;
3206         }
3207
3208         if (clear_free_space_tree) {
3209                 btrfs_info(fs_info, "clearing free space tree");
3210                 ret = btrfs_clear_free_space_tree(fs_info);
3211                 if (ret) {
3212                         btrfs_warn(fs_info,
3213                                    "failed to clear free space tree: %d", ret);
3214                         goto out;
3215                 }
3216         }
3217
3218         /*
3219          * btrfs_find_orphan_roots() is responsible for finding all the dead
3220          * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load
3221          * them into the fs_info->fs_roots_radix tree. This must be done before
3222          * calling btrfs_orphan_cleanup() on the tree root. If we don't do it
3223          * first, then btrfs_orphan_cleanup() will delete a dead root's orphan
3224          * item before the root's tree is deleted - this means that if we unmount
3225          * or crash before the deletion completes, on the next mount we will not
3226          * delete what remains of the tree because the orphan item does not
3227          * exists anymore, which is what tells us we have a pending deletion.
3228          */
3229         ret = btrfs_find_orphan_roots(fs_info);
3230         if (ret)
3231                 goto out;
3232
3233         ret = btrfs_cleanup_fs_roots(fs_info);
3234         if (ret)
3235                 goto out;
3236
3237         down_read(&fs_info->cleanup_work_sem);
3238         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3239             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3240                 up_read(&fs_info->cleanup_work_sem);
3241                 goto out;
3242         }
3243         up_read(&fs_info->cleanup_work_sem);
3244
3245         mutex_lock(&fs_info->cleaner_mutex);
3246         ret = btrfs_recover_relocation(fs_info);
3247         mutex_unlock(&fs_info->cleaner_mutex);
3248         if (ret < 0) {
3249                 btrfs_warn(fs_info, "failed to recover relocation: %d", ret);
3250                 goto out;
3251         }
3252
3253         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3254             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3255                 btrfs_info(fs_info, "creating free space tree");
3256                 ret = btrfs_create_free_space_tree(fs_info);
3257                 if (ret) {
3258                         btrfs_warn(fs_info,
3259                                 "failed to create free space tree: %d", ret);
3260                         goto out;
3261                 }
3262         }
3263
3264         if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) {
3265                 ret = btrfs_set_free_space_cache_v1_active(fs_info, cache_opt);
3266                 if (ret)
3267                         goto out;
3268         }
3269
3270         ret = btrfs_resume_balance_async(fs_info);
3271         if (ret)
3272                 goto out;
3273
3274         ret = btrfs_resume_dev_replace_async(fs_info);
3275         if (ret) {
3276                 btrfs_warn(fs_info, "failed to resume dev_replace");
3277                 goto out;
3278         }
3279
3280         btrfs_qgroup_rescan_resume(fs_info);
3281
3282         if (!fs_info->uuid_root) {
3283                 btrfs_info(fs_info, "creating UUID tree");
3284                 ret = btrfs_create_uuid_tree(fs_info);
3285                 if (ret) {
3286                         btrfs_warn(fs_info,
3287                                    "failed to create the UUID tree %d", ret);
3288                         goto out;
3289                 }
3290         }
3291
3292 out:
3293         return ret;
3294 }
3295
3296 /*
3297  * Do various sanity and dependency checks of different features.
3298  *
3299  * This is the place for less strict checks (like for subpage or artificial
3300  * feature dependencies).
3301  *
3302  * For strict checks or possible corruption detection, see
3303  * btrfs_validate_super().
3304  *
3305  * This should be called after btrfs_parse_options(), as some mount options
3306  * (space cache related) can modify on-disk format like free space tree and
3307  * screw up certain feature dependencies.
3308  */
3309 int btrfs_check_features(struct btrfs_fs_info *fs_info, struct super_block *sb)
3310 {
3311         struct btrfs_super_block *disk_super = fs_info->super_copy;
3312         u64 incompat = btrfs_super_incompat_flags(disk_super);
3313         const u64 compat_ro = btrfs_super_compat_ro_flags(disk_super);
3314         const u64 compat_ro_unsupp = (compat_ro & ~BTRFS_FEATURE_COMPAT_RO_SUPP);
3315
3316         if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
3317                 btrfs_err(fs_info,
3318                 "cannot mount because of unknown incompat features (0x%llx)",
3319                     incompat);
3320                 return -EINVAL;
3321         }
3322
3323         /* Runtime limitation for mixed block groups. */
3324         if ((incompat & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
3325             (fs_info->sectorsize != fs_info->nodesize)) {
3326                 btrfs_err(fs_info,
3327 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
3328                         fs_info->nodesize, fs_info->sectorsize);
3329                 return -EINVAL;
3330         }
3331
3332         /* Mixed backref is an always-enabled feature. */
3333         incompat |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
3334
3335         /* Set compression related flags just in case. */
3336         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
3337                 incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
3338         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
3339                 incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
3340
3341         /*
3342          * An ancient flag, which should really be marked deprecated.
3343          * Such runtime limitation doesn't really need a incompat flag.
3344          */
3345         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE)
3346                 incompat |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
3347
3348         if (compat_ro_unsupp && !sb_rdonly(sb)) {
3349                 btrfs_err(fs_info,
3350         "cannot mount read-write because of unknown compat_ro features (0x%llx)",
3351                        compat_ro);
3352                 return -EINVAL;
3353         }
3354
3355         /*
3356          * We have unsupported RO compat features, although RO mounted, we
3357          * should not cause any metadata writes, including log replay.
3358          * Or we could screw up whatever the new feature requires.
3359          */
3360         if (compat_ro_unsupp && btrfs_super_log_root(disk_super) &&
3361             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3362                 btrfs_err(fs_info,
3363 "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay",
3364                           compat_ro);
3365                 return -EINVAL;
3366         }
3367
3368         /*
3369          * Artificial limitations for block group tree, to force
3370          * block-group-tree to rely on no-holes and free-space-tree.
3371          */
3372         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
3373             (!btrfs_fs_incompat(fs_info, NO_HOLES) ||
3374              !btrfs_test_opt(fs_info, FREE_SPACE_TREE))) {
3375                 btrfs_err(fs_info,
3376 "block-group-tree feature requires no-holes and free-space-tree features");
3377                 return -EINVAL;
3378         }
3379
3380         /*
3381          * Subpage runtime limitation on v1 cache.
3382          *
3383          * V1 space cache still has some hard codeed PAGE_SIZE usage, while
3384          * we're already defaulting to v2 cache, no need to bother v1 as it's
3385          * going to be deprecated anyway.
3386          */
3387         if (fs_info->sectorsize < PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) {
3388                 btrfs_warn(fs_info,
3389         "v1 space cache is not supported for page size %lu with sectorsize %u",
3390                            PAGE_SIZE, fs_info->sectorsize);
3391                 return -EINVAL;
3392         }
3393
3394         /* This can be called by remount, we need to protect the super block. */
3395         spin_lock(&fs_info->super_lock);
3396         btrfs_set_super_incompat_flags(disk_super, incompat);
3397         spin_unlock(&fs_info->super_lock);
3398
3399         return 0;
3400 }
3401
3402 int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices,
3403                       char *options)
3404 {
3405         u32 sectorsize;
3406         u32 nodesize;
3407         u32 stripesize;
3408         u64 generation;
3409         u64 features;
3410         u16 csum_type;
3411         struct btrfs_super_block *disk_super;
3412         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
3413         struct btrfs_root *tree_root;
3414         struct btrfs_root *chunk_root;
3415         int ret;
3416         int err = -EINVAL;
3417         int level;
3418
3419         ret = init_mount_fs_info(fs_info, sb);
3420         if (ret) {
3421                 err = ret;
3422                 goto fail;
3423         }
3424
3425         /* These need to be init'ed before we start creating inodes and such. */
3426         tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID,
3427                                      GFP_KERNEL);
3428         fs_info->tree_root = tree_root;
3429         chunk_root = btrfs_alloc_root(fs_info, BTRFS_CHUNK_TREE_OBJECTID,
3430                                       GFP_KERNEL);
3431         fs_info->chunk_root = chunk_root;
3432         if (!tree_root || !chunk_root) {
3433                 err = -ENOMEM;
3434                 goto fail;
3435         }
3436
3437         fs_info->btree_inode = new_inode(sb);
3438         if (!fs_info->btree_inode) {
3439                 err = -ENOMEM;
3440                 goto fail;
3441         }
3442         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
3443         btrfs_init_btree_inode(fs_info);
3444
3445         invalidate_bdev(fs_devices->latest_dev->bdev);
3446
3447         /*
3448          * Read super block and check the signature bytes only
3449          */
3450         disk_super = btrfs_read_dev_super(fs_devices->latest_dev->bdev);
3451         if (IS_ERR(disk_super)) {
3452                 err = PTR_ERR(disk_super);
3453                 goto fail_alloc;
3454         }
3455
3456         /*
3457          * Verify the type first, if that or the checksum value are
3458          * corrupted, we'll find out
3459          */
3460         csum_type = btrfs_super_csum_type(disk_super);
3461         if (!btrfs_supported_super_csum(csum_type)) {
3462                 btrfs_err(fs_info, "unsupported checksum algorithm: %u",
3463                           csum_type);
3464                 err = -EINVAL;
3465                 btrfs_release_disk_super(disk_super);
3466                 goto fail_alloc;
3467         }
3468
3469         fs_info->csum_size = btrfs_super_csum_size(disk_super);
3470
3471         ret = btrfs_init_csum_hash(fs_info, csum_type);
3472         if (ret) {
3473                 err = ret;
3474                 btrfs_release_disk_super(disk_super);
3475                 goto fail_alloc;
3476         }
3477
3478         /*
3479          * We want to check superblock checksum, the type is stored inside.
3480          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
3481          */
3482         if (btrfs_check_super_csum(fs_info, disk_super)) {
3483                 btrfs_err(fs_info, "superblock checksum mismatch");
3484                 err = -EINVAL;
3485                 btrfs_release_disk_super(disk_super);
3486                 goto fail_alloc;
3487         }
3488
3489         /*
3490          * super_copy is zeroed at allocation time and we never touch the
3491          * following bytes up to INFO_SIZE, the checksum is calculated from
3492          * the whole block of INFO_SIZE
3493          */
3494         memcpy(fs_info->super_copy, disk_super, sizeof(*fs_info->super_copy));
3495         btrfs_release_disk_super(disk_super);
3496
3497         disk_super = fs_info->super_copy;
3498
3499
3500         features = btrfs_super_flags(disk_super);
3501         if (features & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
3502                 features &= ~BTRFS_SUPER_FLAG_CHANGING_FSID_V2;
3503                 btrfs_set_super_flags(disk_super, features);
3504                 btrfs_info(fs_info,
3505                         "found metadata UUID change in progress flag, clearing");
3506         }
3507
3508         memcpy(fs_info->super_for_commit, fs_info->super_copy,
3509                sizeof(*fs_info->super_for_commit));
3510
3511         ret = btrfs_validate_mount_super(fs_info);
3512         if (ret) {
3513                 btrfs_err(fs_info, "superblock contains fatal errors");
3514                 err = -EINVAL;
3515                 goto fail_alloc;
3516         }
3517
3518         if (!btrfs_super_root(disk_super))
3519                 goto fail_alloc;
3520
3521         /* check FS state, whether FS is broken. */
3522         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
3523                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
3524
3525         /*
3526          * In the long term, we'll store the compression type in the super
3527          * block, and it'll be used for per file compression control.
3528          */
3529         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
3530
3531
3532         /* Set up fs_info before parsing mount options */
3533         nodesize = btrfs_super_nodesize(disk_super);
3534         sectorsize = btrfs_super_sectorsize(disk_super);
3535         stripesize = sectorsize;
3536         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
3537         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
3538
3539         fs_info->nodesize = nodesize;
3540         fs_info->sectorsize = sectorsize;
3541         fs_info->sectorsize_bits = ilog2(sectorsize);
3542         fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size;
3543         fs_info->stripesize = stripesize;
3544
3545         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
3546         if (ret) {
3547                 err = ret;
3548                 goto fail_alloc;
3549         }
3550
3551         ret = btrfs_check_features(fs_info, sb);
3552         if (ret < 0) {
3553                 err = ret;
3554                 goto fail_alloc;
3555         }
3556
3557         if (sectorsize < PAGE_SIZE) {
3558                 struct btrfs_subpage_info *subpage_info;
3559
3560                 /*
3561                  * V1 space cache has some hardcoded PAGE_SIZE usage, and is
3562                  * going to be deprecated.
3563                  *
3564                  * Force to use v2 cache for subpage case.
3565                  */
3566                 btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
3567                 btrfs_set_and_info(fs_info, FREE_SPACE_TREE,
3568                         "forcing free space tree for sector size %u with page size %lu",
3569                         sectorsize, PAGE_SIZE);
3570
3571                 btrfs_warn(fs_info,
3572                 "read-write for sector size %u with page size %lu is experimental",
3573                            sectorsize, PAGE_SIZE);
3574                 subpage_info = kzalloc(sizeof(*subpage_info), GFP_KERNEL);
3575                 if (!subpage_info)
3576                         goto fail_alloc;
3577                 btrfs_init_subpage_info(subpage_info, sectorsize);
3578                 fs_info->subpage_info = subpage_info;
3579         }
3580
3581         ret = btrfs_init_workqueues(fs_info);
3582         if (ret) {
3583                 err = ret;
3584                 goto fail_sb_buffer;
3585         }
3586
3587         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
3588         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
3589
3590         sb->s_blocksize = sectorsize;
3591         sb->s_blocksize_bits = blksize_bits(sectorsize);
3592         memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE);
3593
3594         mutex_lock(&fs_info->chunk_mutex);
3595         ret = btrfs_read_sys_array(fs_info);
3596         mutex_unlock(&fs_info->chunk_mutex);
3597         if (ret) {
3598                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
3599                 goto fail_sb_buffer;
3600         }
3601
3602         generation = btrfs_super_chunk_root_generation(disk_super);
3603         level = btrfs_super_chunk_root_level(disk_super);
3604         ret = load_super_root(chunk_root, btrfs_super_chunk_root(disk_super),
3605                               generation, level);
3606         if (ret) {
3607                 btrfs_err(fs_info, "failed to read chunk root");
3608                 goto fail_tree_roots;
3609         }
3610
3611         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
3612                            offsetof(struct btrfs_header, chunk_tree_uuid),
3613                            BTRFS_UUID_SIZE);
3614
3615         ret = btrfs_read_chunk_tree(fs_info);
3616         if (ret) {
3617                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
3618                 goto fail_tree_roots;
3619         }
3620
3621         /*
3622          * At this point we know all the devices that make this filesystem,
3623          * including the seed devices but we don't know yet if the replace
3624          * target is required. So free devices that are not part of this
3625          * filesystem but skip the replace target device which is checked
3626          * below in btrfs_init_dev_replace().
3627          */
3628         btrfs_free_extra_devids(fs_devices);
3629         if (!fs_devices->latest_dev->bdev) {
3630                 btrfs_err(fs_info, "failed to read devices");
3631                 goto fail_tree_roots;
3632         }
3633
3634         ret = init_tree_roots(fs_info);
3635         if (ret)
3636                 goto fail_tree_roots;
3637
3638         /*
3639          * Get zone type information of zoned block devices. This will also
3640          * handle emulation of a zoned filesystem if a regular device has the
3641          * zoned incompat feature flag set.
3642          */
3643         ret = btrfs_get_dev_zone_info_all_devices(fs_info);
3644         if (ret) {
3645                 btrfs_err(fs_info,
3646                           "zoned: failed to read device zone info: %d",
3647                           ret);
3648                 goto fail_block_groups;
3649         }
3650
3651         /*
3652          * If we have a uuid root and we're not being told to rescan we need to
3653          * check the generation here so we can set the
3654          * BTRFS_FS_UPDATE_UUID_TREE_GEN bit.  Otherwise we could commit the
3655          * transaction during a balance or the log replay without updating the
3656          * uuid generation, and then if we crash we would rescan the uuid tree,
3657          * even though it was perfectly fine.
3658          */
3659         if (fs_info->uuid_root && !btrfs_test_opt(fs_info, RESCAN_UUID_TREE) &&
3660             fs_info->generation == btrfs_super_uuid_tree_generation(disk_super))
3661                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
3662
3663         ret = btrfs_verify_dev_extents(fs_info);
3664         if (ret) {
3665                 btrfs_err(fs_info,
3666                           "failed to verify dev extents against chunks: %d",
3667                           ret);
3668                 goto fail_block_groups;
3669         }
3670         ret = btrfs_recover_balance(fs_info);
3671         if (ret) {
3672                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3673                 goto fail_block_groups;
3674         }
3675
3676         ret = btrfs_init_dev_stats(fs_info);
3677         if (ret) {
3678                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3679                 goto fail_block_groups;
3680         }
3681
3682         ret = btrfs_init_dev_replace(fs_info);
3683         if (ret) {
3684                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3685                 goto fail_block_groups;
3686         }
3687
3688         ret = btrfs_check_zoned_mode(fs_info);
3689         if (ret) {
3690                 btrfs_err(fs_info, "failed to initialize zoned mode: %d",
3691                           ret);
3692                 goto fail_block_groups;
3693         }
3694
3695         ret = btrfs_sysfs_add_fsid(fs_devices);
3696         if (ret) {
3697                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3698                                 ret);
3699                 goto fail_block_groups;
3700         }
3701
3702         ret = btrfs_sysfs_add_mounted(fs_info);
3703         if (ret) {
3704                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3705                 goto fail_fsdev_sysfs;
3706         }
3707
3708         ret = btrfs_init_space_info(fs_info);
3709         if (ret) {
3710                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3711                 goto fail_sysfs;
3712         }
3713
3714         ret = btrfs_read_block_groups(fs_info);
3715         if (ret) {
3716                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3717                 goto fail_sysfs;
3718         }
3719
3720         btrfs_free_zone_cache(fs_info);
3721
3722         if (!sb_rdonly(sb) && fs_info->fs_devices->missing_devices &&
3723             !btrfs_check_rw_degradable(fs_info, NULL)) {
3724                 btrfs_warn(fs_info,
3725                 "writable mount is not allowed due to too many missing devices");
3726                 goto fail_sysfs;
3727         }
3728
3729         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, fs_info,
3730                                                "btrfs-cleaner");
3731         if (IS_ERR(fs_info->cleaner_kthread))
3732                 goto fail_sysfs;
3733
3734         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3735                                                    tree_root,
3736                                                    "btrfs-transaction");
3737         if (IS_ERR(fs_info->transaction_kthread))
3738                 goto fail_cleaner;
3739
3740         if (!btrfs_test_opt(fs_info, NOSSD) &&
3741             !fs_info->fs_devices->rotating) {
3742                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3743         }
3744
3745         /*
3746          * Mount does not set all options immediately, we can do it now and do
3747          * not have to wait for transaction commit
3748          */
3749         btrfs_apply_pending_changes(fs_info);
3750
3751 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3752         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3753                 ret = btrfsic_mount(fs_info, fs_devices,
3754                                     btrfs_test_opt(fs_info,
3755                                         CHECK_INTEGRITY_DATA) ? 1 : 0,
3756                                     fs_info->check_integrity_print_mask);
3757                 if (ret)
3758                         btrfs_warn(fs_info,
3759                                 "failed to initialize integrity check module: %d",
3760                                 ret);
3761         }
3762 #endif
3763         ret = btrfs_read_qgroup_config(fs_info);
3764         if (ret)
3765                 goto fail_trans_kthread;
3766
3767         if (btrfs_build_ref_tree(fs_info))
3768                 btrfs_err(fs_info, "couldn't build ref tree");
3769
3770         /* do not make disk changes in broken FS or nologreplay is given */
3771         if (btrfs_super_log_root(disk_super) != 0 &&
3772             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3773                 btrfs_info(fs_info, "start tree-log replay");
3774                 ret = btrfs_replay_log(fs_info, fs_devices);
3775                 if (ret) {
3776                         err = ret;
3777                         goto fail_qgroup;
3778                 }
3779         }
3780
3781         fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true);
3782         if (IS_ERR(fs_info->fs_root)) {
3783                 err = PTR_ERR(fs_info->fs_root);
3784                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3785                 fs_info->fs_root = NULL;
3786                 goto fail_qgroup;
3787         }
3788
3789         if (sb_rdonly(sb))
3790                 goto clear_oneshot;
3791
3792         ret = btrfs_start_pre_rw_mount(fs_info);
3793         if (ret) {
3794                 close_ctree(fs_info);
3795                 return ret;
3796         }
3797         btrfs_discard_resume(fs_info);
3798
3799         if (fs_info->uuid_root &&
3800             (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) ||
3801              fs_info->generation != btrfs_super_uuid_tree_generation(disk_super))) {
3802                 btrfs_info(fs_info, "checking UUID tree");
3803                 ret = btrfs_check_uuid_tree(fs_info);
3804                 if (ret) {
3805                         btrfs_warn(fs_info,
3806                                 "failed to check the UUID tree: %d", ret);
3807                         close_ctree(fs_info);
3808                         return ret;
3809                 }
3810         }
3811
3812         set_bit(BTRFS_FS_OPEN, &fs_info->flags);
3813
3814         /* Kick the cleaner thread so it'll start deleting snapshots. */
3815         if (test_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags))
3816                 wake_up_process(fs_info->cleaner_kthread);
3817
3818 clear_oneshot:
3819         btrfs_clear_oneshot_options(fs_info);
3820         return 0;
3821
3822 fail_qgroup:
3823         btrfs_free_qgroup_config(fs_info);
3824 fail_trans_kthread:
3825         kthread_stop(fs_info->transaction_kthread);
3826         btrfs_cleanup_transaction(fs_info);
3827         btrfs_free_fs_roots(fs_info);
3828 fail_cleaner:
3829         kthread_stop(fs_info->cleaner_kthread);
3830
3831         /*
3832          * make sure we're done with the btree inode before we stop our
3833          * kthreads
3834          */
3835         filemap_write_and_wait(fs_info->btree_inode->i_mapping);
3836
3837 fail_sysfs:
3838         btrfs_sysfs_remove_mounted(fs_info);
3839
3840 fail_fsdev_sysfs:
3841         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3842
3843 fail_block_groups:
3844         btrfs_put_block_group_cache(fs_info);
3845
3846 fail_tree_roots:
3847         if (fs_info->data_reloc_root)
3848                 btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root);
3849         free_root_pointers(fs_info, true);
3850         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
3851
3852 fail_sb_buffer:
3853         btrfs_stop_all_workers(fs_info);
3854         btrfs_free_block_groups(fs_info);
3855 fail_alloc:
3856         btrfs_mapping_tree_free(&fs_info->mapping_tree);
3857
3858         iput(fs_info->btree_inode);
3859 fail:
3860         btrfs_close_devices(fs_info->fs_devices);
3861         return err;
3862 }
3863 ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
3864
3865 static void btrfs_end_super_write(struct bio *bio)
3866 {
3867         struct btrfs_device *device = bio->bi_private;
3868         struct bio_vec *bvec;
3869         struct bvec_iter_all iter_all;
3870         struct page *page;
3871
3872         bio_for_each_segment_all(bvec, bio, iter_all) {
3873                 page = bvec->bv_page;
3874
3875                 if (bio->bi_status) {
3876                         btrfs_warn_rl_in_rcu(device->fs_info,
3877                                 "lost page write due to IO error on %s (%d)",
3878                                 rcu_str_deref(device->name),
3879                                 blk_status_to_errno(bio->bi_status));
3880                         ClearPageUptodate(page);
3881                         SetPageError(page);
3882                         btrfs_dev_stat_inc_and_print(device,
3883                                                      BTRFS_DEV_STAT_WRITE_ERRS);
3884                 } else {
3885                         SetPageUptodate(page);
3886                 }
3887
3888                 put_page(page);
3889                 unlock_page(page);
3890         }
3891
3892         bio_put(bio);
3893 }
3894
3895 struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev,
3896                                                    int copy_num, bool drop_cache)
3897 {
3898         struct btrfs_super_block *super;
3899         struct page *page;
3900         u64 bytenr, bytenr_orig;
3901         struct address_space *mapping = bdev->bd_inode->i_mapping;
3902         int ret;
3903
3904         bytenr_orig = btrfs_sb_offset(copy_num);
3905         ret = btrfs_sb_log_location_bdev(bdev, copy_num, READ, &bytenr);
3906         if (ret == -ENOENT)
3907                 return ERR_PTR(-EINVAL);
3908         else if (ret)
3909                 return ERR_PTR(ret);
3910
3911         if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev))
3912                 return ERR_PTR(-EINVAL);
3913
3914         if (drop_cache) {
3915                 /* This should only be called with the primary sb. */
3916                 ASSERT(copy_num == 0);
3917
3918                 /*
3919                  * Drop the page of the primary superblock, so later read will
3920                  * always read from the device.
3921                  */
3922                 invalidate_inode_pages2_range(mapping,
3923                                 bytenr >> PAGE_SHIFT,
3924                                 (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT);
3925         }
3926
3927         page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS);
3928         if (IS_ERR(page))
3929                 return ERR_CAST(page);
3930
3931         super = page_address(page);
3932         if (btrfs_super_magic(super) != BTRFS_MAGIC) {
3933                 btrfs_release_disk_super(super);
3934                 return ERR_PTR(-ENODATA);
3935         }
3936
3937         if (btrfs_super_bytenr(super) != bytenr_orig) {
3938                 btrfs_release_disk_super(super);
3939                 return ERR_PTR(-EINVAL);
3940         }
3941
3942         return super;
3943 }
3944
3945
3946 struct btrfs_super_block *btrfs_read_dev_super(struct block_device *bdev)
3947 {
3948         struct btrfs_super_block *super, *latest = NULL;
3949         int i;
3950         u64 transid = 0;
3951
3952         /* we would like to check all the supers, but that would make
3953          * a btrfs mount succeed after a mkfs from a different FS.
3954          * So, we need to add a special mount option to scan for
3955          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
3956          */
3957         for (i = 0; i < 1; i++) {
3958                 super = btrfs_read_dev_one_super(bdev, i, false);
3959                 if (IS_ERR(super))
3960                         continue;
3961
3962                 if (!latest || btrfs_super_generation(super) > transid) {
3963                         if (latest)
3964                                 btrfs_release_disk_super(super);
3965
3966                         latest = super;
3967                         transid = btrfs_super_generation(super);
3968                 }
3969         }
3970
3971         return super;
3972 }
3973
3974 /*
3975  * Write superblock @sb to the @device. Do not wait for completion, all the
3976  * pages we use for writing are locked.
3977  *
3978  * Write @max_mirrors copies of the superblock, where 0 means default that fit
3979  * the expected device size at commit time. Note that max_mirrors must be
3980  * same for write and wait phases.
3981  *
3982  * Return number of errors when page is not found or submission fails.
3983  */
3984 static int write_dev_supers(struct btrfs_device *device,
3985                             struct btrfs_super_block *sb, int max_mirrors)
3986 {
3987         struct btrfs_fs_info *fs_info = device->fs_info;
3988         struct address_space *mapping = device->bdev->bd_inode->i_mapping;
3989         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3990         int i;
3991         int errors = 0;
3992         int ret;
3993         u64 bytenr, bytenr_orig;
3994
3995         if (max_mirrors == 0)
3996                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3997
3998         shash->tfm = fs_info->csum_shash;
3999
4000         for (i = 0; i < max_mirrors; i++) {
4001                 struct page *page;
4002                 struct bio *bio;
4003                 struct btrfs_super_block *disk_super;
4004
4005                 bytenr_orig = btrfs_sb_offset(i);
4006                 ret = btrfs_sb_log_location(device, i, WRITE, &bytenr);
4007                 if (ret == -ENOENT) {
4008                         continue;
4009                 } else if (ret < 0) {
4010                         btrfs_err(device->fs_info,
4011                                 "couldn't get super block location for mirror %d",
4012                                 i);
4013                         errors++;
4014                         continue;
4015                 }
4016                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
4017                     device->commit_total_bytes)
4018                         break;
4019
4020                 btrfs_set_super_bytenr(sb, bytenr_orig);
4021
4022                 crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE,
4023                                     BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE,
4024                                     sb->csum);
4025
4026                 page = find_or_create_page(mapping, bytenr >> PAGE_SHIFT,
4027                                            GFP_NOFS);
4028                 if (!page) {
4029                         btrfs_err(device->fs_info,
4030                             "couldn't get super block page for bytenr %llu",
4031                             bytenr);
4032                         errors++;
4033                         continue;
4034                 }
4035
4036                 /* Bump the refcount for wait_dev_supers() */
4037                 get_page(page);
4038
4039                 disk_super = page_address(page);
4040                 memcpy(disk_super, sb, BTRFS_SUPER_INFO_SIZE);
4041
4042                 /*
4043                  * Directly use bios here instead of relying on the page cache
4044                  * to do I/O, so we don't lose the ability to do integrity
4045                  * checking.
4046                  */
4047                 bio = bio_alloc(device->bdev, 1,
4048                                 REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO,
4049                                 GFP_NOFS);
4050                 bio->bi_iter.bi_sector = bytenr >> SECTOR_SHIFT;
4051                 bio->bi_private = device;
4052                 bio->bi_end_io = btrfs_end_super_write;
4053                 __bio_add_page(bio, page, BTRFS_SUPER_INFO_SIZE,
4054                                offset_in_page(bytenr));
4055
4056                 /*
4057                  * We FUA only the first super block.  The others we allow to
4058                  * go down lazy and there's a short window where the on-disk
4059                  * copies might still contain the older version.
4060                  */
4061                 if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER))
4062                         bio->bi_opf |= REQ_FUA;
4063
4064                 btrfsic_check_bio(bio);
4065                 submit_bio(bio);
4066
4067                 if (btrfs_advance_sb_log(device, i))
4068                         errors++;
4069         }
4070         return errors < i ? 0 : -1;
4071 }
4072
4073 /*
4074  * Wait for write completion of superblocks done by write_dev_supers,
4075  * @max_mirrors same for write and wait phases.
4076  *
4077  * Return number of errors when page is not found or not marked up to
4078  * date.
4079  */
4080 static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
4081 {
4082         int i;
4083         int errors = 0;
4084         bool primary_failed = false;
4085         int ret;
4086         u64 bytenr;
4087
4088         if (max_mirrors == 0)
4089                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
4090
4091         for (i = 0; i < max_mirrors; i++) {
4092                 struct page *page;
4093
4094                 ret = btrfs_sb_log_location(device, i, READ, &bytenr);
4095                 if (ret == -ENOENT) {
4096                         break;
4097                 } else if (ret < 0) {
4098                         errors++;
4099                         if (i == 0)
4100                                 primary_failed = true;
4101                         continue;
4102                 }
4103                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
4104                     device->commit_total_bytes)
4105                         break;
4106
4107                 page = find_get_page(device->bdev->bd_inode->i_mapping,
4108                                      bytenr >> PAGE_SHIFT);
4109                 if (!page) {
4110                         errors++;
4111                         if (i == 0)
4112                                 primary_failed = true;
4113                         continue;
4114                 }
4115                 /* Page is submitted locked and unlocked once the IO completes */
4116                 wait_on_page_locked(page);
4117                 if (PageError(page)) {
4118                         errors++;
4119                         if (i == 0)
4120                                 primary_failed = true;
4121                 }
4122
4123                 /* Drop our reference */
4124                 put_page(page);
4125
4126                 /* Drop the reference from the writing run */
4127                 put_page(page);
4128         }
4129
4130         /* log error, force error return */
4131         if (primary_failed) {
4132                 btrfs_err(device->fs_info, "error writing primary super block to device %llu",
4133                           device->devid);
4134                 return -1;
4135         }
4136
4137         return errors < i ? 0 : -1;
4138 }
4139
4140 /*
4141  * endio for the write_dev_flush, this will wake anyone waiting
4142  * for the barrier when it is done
4143  */
4144 static void btrfs_end_empty_barrier(struct bio *bio)
4145 {
4146         bio_uninit(bio);
4147         complete(bio->bi_private);
4148 }
4149
4150 /*
4151  * Submit a flush request to the device if it supports it. Error handling is
4152  * done in the waiting counterpart.
4153  */
4154 static void write_dev_flush(struct btrfs_device *device)
4155 {
4156         struct bio *bio = &device->flush_bio;
4157
4158 #ifndef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4159         /*
4160          * When a disk has write caching disabled, we skip submission of a bio
4161          * with flush and sync requests before writing the superblock, since
4162          * it's not needed. However when the integrity checker is enabled, this
4163          * results in reports that there are metadata blocks referred by a
4164          * superblock that were not properly flushed. So don't skip the bio
4165          * submission only when the integrity checker is enabled for the sake
4166          * of simplicity, since this is a debug tool and not meant for use in
4167          * non-debug builds.
4168          */
4169         if (!bdev_write_cache(device->bdev))
4170                 return;
4171 #endif
4172
4173         bio_init(bio, device->bdev, NULL, 0,
4174                  REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH);
4175         bio->bi_end_io = btrfs_end_empty_barrier;
4176         init_completion(&device->flush_wait);
4177         bio->bi_private = &device->flush_wait;
4178
4179         btrfsic_check_bio(bio);
4180         submit_bio(bio);
4181         set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4182 }
4183
4184 /*
4185  * If the flush bio has been submitted by write_dev_flush, wait for it.
4186  */
4187 static blk_status_t wait_dev_flush(struct btrfs_device *device)
4188 {
4189         struct bio *bio = &device->flush_bio;
4190
4191         if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
4192                 return BLK_STS_OK;
4193
4194         clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4195         wait_for_completion_io(&device->flush_wait);
4196
4197         return bio->bi_status;
4198 }
4199
4200 static int check_barrier_error(struct btrfs_fs_info *fs_info)
4201 {
4202         if (!btrfs_check_rw_degradable(fs_info, NULL))
4203                 return -EIO;
4204         return 0;
4205 }
4206
4207 /*
4208  * send an empty flush down to each device in parallel,
4209  * then wait for them
4210  */
4211 static int barrier_all_devices(struct btrfs_fs_info *info)
4212 {
4213         struct list_head *head;
4214         struct btrfs_device *dev;
4215         int errors_wait = 0;
4216         blk_status_t ret;
4217
4218         lockdep_assert_held(&info->fs_devices->device_list_mutex);
4219         /* send down all the barriers */
4220         head = &info->fs_devices->devices;
4221         list_for_each_entry(dev, head, dev_list) {
4222                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4223                         continue;
4224                 if (!dev->bdev)
4225                         continue;
4226                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4227                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4228                         continue;
4229
4230                 write_dev_flush(dev);
4231                 dev->last_flush_error = BLK_STS_OK;
4232         }
4233
4234         /* wait for all the barriers */
4235         list_for_each_entry(dev, head, dev_list) {
4236                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4237                         continue;
4238                 if (!dev->bdev) {
4239                         errors_wait++;
4240                         continue;
4241                 }
4242                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4243                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4244                         continue;
4245
4246                 ret = wait_dev_flush(dev);
4247                 if (ret) {
4248                         dev->last_flush_error = ret;
4249                         btrfs_dev_stat_inc_and_print(dev,
4250                                         BTRFS_DEV_STAT_FLUSH_ERRS);
4251                         errors_wait++;
4252                 }
4253         }
4254
4255         if (errors_wait) {
4256                 /*
4257                  * At some point we need the status of all disks
4258                  * to arrive at the volume status. So error checking
4259                  * is being pushed to a separate loop.
4260                  */
4261                 return check_barrier_error(info);
4262         }
4263         return 0;
4264 }
4265
4266 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
4267 {
4268         int raid_type;
4269         int min_tolerated = INT_MAX;
4270
4271         if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 ||
4272             (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE))
4273                 min_tolerated = min_t(int, min_tolerated,
4274                                     btrfs_raid_array[BTRFS_RAID_SINGLE].
4275                                     tolerated_failures);
4276
4277         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
4278                 if (raid_type == BTRFS_RAID_SINGLE)
4279                         continue;
4280                 if (!(flags & btrfs_raid_array[raid_type].bg_flag))
4281                         continue;
4282                 min_tolerated = min_t(int, min_tolerated,
4283                                     btrfs_raid_array[raid_type].
4284                                     tolerated_failures);
4285         }
4286
4287         if (min_tolerated == INT_MAX) {
4288                 pr_warn("BTRFS: unknown raid flag: %llu", flags);
4289                 min_tolerated = 0;
4290         }
4291
4292         return min_tolerated;
4293 }
4294
4295 int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
4296 {
4297         struct list_head *head;
4298         struct btrfs_device *dev;
4299         struct btrfs_super_block *sb;
4300         struct btrfs_dev_item *dev_item;
4301         int ret;
4302         int do_barriers;
4303         int max_errors;
4304         int total_errors = 0;
4305         u64 flags;
4306
4307         do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
4308
4309         /*
4310          * max_mirrors == 0 indicates we're from commit_transaction,
4311          * not from fsync where the tree roots in fs_info have not
4312          * been consistent on disk.
4313          */
4314         if (max_mirrors == 0)
4315                 backup_super_roots(fs_info);
4316
4317         sb = fs_info->super_for_commit;
4318         dev_item = &sb->dev_item;
4319
4320         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4321         head = &fs_info->fs_devices->devices;
4322         max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1;
4323
4324         if (do_barriers) {
4325                 ret = barrier_all_devices(fs_info);
4326                 if (ret) {
4327                         mutex_unlock(
4328                                 &fs_info->fs_devices->device_list_mutex);
4329                         btrfs_handle_fs_error(fs_info, ret,
4330                                               "errors while submitting device barriers.");
4331                         return ret;
4332                 }
4333         }
4334
4335         list_for_each_entry(dev, head, dev_list) {
4336                 if (!dev->bdev) {
4337                         total_errors++;
4338                         continue;
4339                 }
4340                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4341                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4342                         continue;
4343
4344                 btrfs_set_stack_device_generation(dev_item, 0);
4345                 btrfs_set_stack_device_type(dev_item, dev->type);
4346                 btrfs_set_stack_device_id(dev_item, dev->devid);
4347                 btrfs_set_stack_device_total_bytes(dev_item,
4348                                                    dev->commit_total_bytes);
4349                 btrfs_set_stack_device_bytes_used(dev_item,
4350                                                   dev->commit_bytes_used);
4351                 btrfs_set_stack_device_io_align(dev_item, dev->io_align);
4352                 btrfs_set_stack_device_io_width(dev_item, dev->io_width);
4353                 btrfs_set_stack_device_sector_size(dev_item, dev->sector_size);
4354                 memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE);
4355                 memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid,
4356                        BTRFS_FSID_SIZE);
4357
4358                 flags = btrfs_super_flags(sb);
4359                 btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
4360
4361                 ret = btrfs_validate_write_super(fs_info, sb);
4362                 if (ret < 0) {
4363                         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4364                         btrfs_handle_fs_error(fs_info, -EUCLEAN,
4365                                 "unexpected superblock corruption detected");
4366                         return -EUCLEAN;
4367                 }
4368
4369                 ret = write_dev_supers(dev, sb, max_mirrors);
4370                 if (ret)
4371                         total_errors++;
4372         }
4373         if (total_errors > max_errors) {
4374                 btrfs_err(fs_info, "%d errors while writing supers",
4375                           total_errors);
4376                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4377
4378                 /* FUA is masked off if unsupported and can't be the reason */
4379                 btrfs_handle_fs_error(fs_info, -EIO,
4380                                       "%d errors while writing supers",
4381                                       total_errors);
4382                 return -EIO;
4383         }
4384
4385         total_errors = 0;
4386         list_for_each_entry(dev, head, dev_list) {
4387                 if (!dev->bdev)
4388                         continue;
4389                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4390                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4391                         continue;
4392
4393                 ret = wait_dev_supers(dev, max_mirrors);
4394                 if (ret)
4395                         total_errors++;
4396         }
4397         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4398         if (total_errors > max_errors) {
4399                 btrfs_handle_fs_error(fs_info, -EIO,
4400                                       "%d errors while writing supers",
4401                                       total_errors);
4402                 return -EIO;
4403         }
4404         return 0;
4405 }
4406
4407 /* Drop a fs root from the radix tree and free it. */
4408 void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
4409                                   struct btrfs_root *root)
4410 {
4411         bool drop_ref = false;
4412
4413         spin_lock(&fs_info->fs_roots_radix_lock);
4414         radix_tree_delete(&fs_info->fs_roots_radix,
4415                           (unsigned long)root->root_key.objectid);
4416         if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state))
4417                 drop_ref = true;
4418         spin_unlock(&fs_info->fs_roots_radix_lock);
4419
4420         if (BTRFS_FS_ERROR(fs_info)) {
4421                 ASSERT(root->log_root == NULL);
4422                 if (root->reloc_root) {
4423                         btrfs_put_root(root->reloc_root);
4424                         root->reloc_root = NULL;
4425                 }
4426         }
4427
4428         if (drop_ref)
4429                 btrfs_put_root(root);
4430 }
4431
4432 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
4433 {
4434         u64 root_objectid = 0;
4435         struct btrfs_root *gang[8];
4436         int i = 0;
4437         int err = 0;
4438         unsigned int ret = 0;
4439
4440         while (1) {
4441                 spin_lock(&fs_info->fs_roots_radix_lock);
4442                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4443                                              (void **)gang, root_objectid,
4444                                              ARRAY_SIZE(gang));
4445                 if (!ret) {
4446                         spin_unlock(&fs_info->fs_roots_radix_lock);
4447                         break;
4448                 }
4449                 root_objectid = gang[ret - 1]->root_key.objectid + 1;
4450
4451                 for (i = 0; i < ret; i++) {
4452                         /* Avoid to grab roots in dead_roots */
4453                         if (btrfs_root_refs(&gang[i]->root_item) == 0) {
4454                                 gang[i] = NULL;
4455                                 continue;
4456                         }
4457                         /* grab all the search result for later use */
4458                         gang[i] = btrfs_grab_root(gang[i]);
4459                 }
4460                 spin_unlock(&fs_info->fs_roots_radix_lock);
4461
4462                 for (i = 0; i < ret; i++) {
4463                         if (!gang[i])
4464                                 continue;
4465                         root_objectid = gang[i]->root_key.objectid;
4466                         err = btrfs_orphan_cleanup(gang[i]);
4467                         if (err)
4468                                 break;
4469                         btrfs_put_root(gang[i]);
4470                 }
4471                 root_objectid++;
4472         }
4473
4474         /* release the uncleaned roots due to error */
4475         for (; i < ret; i++) {
4476                 if (gang[i])
4477                         btrfs_put_root(gang[i]);
4478         }
4479         return err;
4480 }
4481
4482 int btrfs_commit_super(struct btrfs_fs_info *fs_info)
4483 {
4484         struct btrfs_root *root = fs_info->tree_root;
4485         struct btrfs_trans_handle *trans;
4486
4487         mutex_lock(&fs_info->cleaner_mutex);
4488         btrfs_run_delayed_iputs(fs_info);
4489         mutex_unlock(&fs_info->cleaner_mutex);
4490         wake_up_process(fs_info->cleaner_kthread);
4491
4492         /* wait until ongoing cleanup work done */
4493         down_write(&fs_info->cleanup_work_sem);
4494         up_write(&fs_info->cleanup_work_sem);
4495
4496         trans = btrfs_join_transaction(root);
4497         if (IS_ERR(trans))
4498                 return PTR_ERR(trans);
4499         return btrfs_commit_transaction(trans);
4500 }
4501
4502 static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info)
4503 {
4504         struct btrfs_transaction *trans;
4505         struct btrfs_transaction *tmp;
4506         bool found = false;
4507
4508         if (list_empty(&fs_info->trans_list))
4509                 return;
4510
4511         /*
4512          * This function is only called at the very end of close_ctree(),
4513          * thus no other running transaction, no need to take trans_lock.
4514          */
4515         ASSERT(test_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags));
4516         list_for_each_entry_safe(trans, tmp, &fs_info->trans_list, list) {
4517                 struct extent_state *cached = NULL;
4518                 u64 dirty_bytes = 0;
4519                 u64 cur = 0;
4520                 u64 found_start;
4521                 u64 found_end;
4522
4523                 found = true;
4524                 while (!find_first_extent_bit(&trans->dirty_pages, cur,
4525                         &found_start, &found_end, EXTENT_DIRTY, &cached)) {
4526                         dirty_bytes += found_end + 1 - found_start;
4527                         cur = found_end + 1;
4528                 }
4529                 btrfs_warn(fs_info,
4530         "transaction %llu (with %llu dirty metadata bytes) is not committed",
4531                            trans->transid, dirty_bytes);
4532                 btrfs_cleanup_one_transaction(trans, fs_info);
4533
4534                 if (trans == fs_info->running_transaction)
4535                         fs_info->running_transaction = NULL;
4536                 list_del_init(&trans->list);
4537
4538                 btrfs_put_transaction(trans);
4539                 trace_btrfs_transaction_commit(fs_info);
4540         }
4541         ASSERT(!found);
4542 }
4543
4544 void __cold close_ctree(struct btrfs_fs_info *fs_info)
4545 {
4546         int ret;
4547
4548         set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags);
4549
4550         /*
4551          * If we had UNFINISHED_DROPS we could still be processing them, so
4552          * clear that bit and wake up relocation so it can stop.
4553          * We must do this before stopping the block group reclaim task, because
4554          * at btrfs_relocate_block_group() we wait for this bit, and after the
4555          * wait we stop with -EINTR if btrfs_fs_closing() returns non-zero - we
4556          * have just set BTRFS_FS_CLOSING_START, so btrfs_fs_closing() will
4557          * return 1.
4558          */
4559         btrfs_wake_unfinished_drop(fs_info);
4560
4561         /*
4562          * We may have the reclaim task running and relocating a data block group,
4563          * in which case it may create delayed iputs. So stop it before we park
4564          * the cleaner kthread otherwise we can get new delayed iputs after
4565          * parking the cleaner, and that can make the async reclaim task to hang
4566          * if it's waiting for delayed iputs to complete, since the cleaner is
4567          * parked and can not run delayed iputs - this will make us hang when
4568          * trying to stop the async reclaim task.
4569          */
4570         cancel_work_sync(&fs_info->reclaim_bgs_work);
4571         /*
4572          * We don't want the cleaner to start new transactions, add more delayed
4573          * iputs, etc. while we're closing. We can't use kthread_stop() yet
4574          * because that frees the task_struct, and the transaction kthread might
4575          * still try to wake up the cleaner.
4576          */
4577         kthread_park(fs_info->cleaner_kthread);
4578
4579         /* wait for the qgroup rescan worker to stop */
4580         btrfs_qgroup_wait_for_completion(fs_info, false);
4581
4582         /* wait for the uuid_scan task to finish */
4583         down(&fs_info->uuid_tree_rescan_sem);
4584         /* avoid complains from lockdep et al., set sem back to initial state */
4585         up(&fs_info->uuid_tree_rescan_sem);
4586
4587         /* pause restriper - we want to resume on mount */
4588         btrfs_pause_balance(fs_info);
4589
4590         btrfs_dev_replace_suspend_for_unmount(fs_info);
4591
4592         btrfs_scrub_cancel(fs_info);
4593
4594         /* wait for any defraggers to finish */
4595         wait_event(fs_info->transaction_wait,
4596                    (atomic_read(&fs_info->defrag_running) == 0));
4597
4598         /* clear out the rbtree of defraggable inodes */
4599         btrfs_cleanup_defrag_inodes(fs_info);
4600
4601         /*
4602          * After we parked the cleaner kthread, ordered extents may have
4603          * completed and created new delayed iputs. If one of the async reclaim
4604          * tasks is running and in the RUN_DELAYED_IPUTS flush state, then we
4605          * can hang forever trying to stop it, because if a delayed iput is
4606          * added after it ran btrfs_run_delayed_iputs() and before it called
4607          * btrfs_wait_on_delayed_iputs(), it will hang forever since there is
4608          * no one else to run iputs.
4609          *
4610          * So wait for all ongoing ordered extents to complete and then run
4611          * delayed iputs. This works because once we reach this point no one
4612          * can either create new ordered extents nor create delayed iputs
4613          * through some other means.
4614          *
4615          * Also note that btrfs_wait_ordered_roots() is not safe here, because
4616          * it waits for BTRFS_ORDERED_COMPLETE to be set on an ordered extent,
4617          * but the delayed iput for the respective inode is made only when doing
4618          * the final btrfs_put_ordered_extent() (which must happen at
4619          * btrfs_finish_ordered_io() when we are unmounting).
4620          */
4621         btrfs_flush_workqueue(fs_info->endio_write_workers);
4622         /* Ordered extents for free space inodes. */
4623         btrfs_flush_workqueue(fs_info->endio_freespace_worker);
4624         btrfs_run_delayed_iputs(fs_info);
4625
4626         cancel_work_sync(&fs_info->async_reclaim_work);
4627         cancel_work_sync(&fs_info->async_data_reclaim_work);
4628         cancel_work_sync(&fs_info->preempt_reclaim_work);
4629
4630         /* Cancel or finish ongoing discard work */
4631         btrfs_discard_cleanup(fs_info);
4632
4633         if (!sb_rdonly(fs_info->sb)) {
4634                 /*
4635                  * The cleaner kthread is stopped, so do one final pass over
4636                  * unused block groups.
4637                  */
4638                 btrfs_delete_unused_bgs(fs_info);
4639
4640                 /*
4641                  * There might be existing delayed inode workers still running
4642                  * and holding an empty delayed inode item. We must wait for
4643                  * them to complete first because they can create a transaction.
4644                  * This happens when someone calls btrfs_balance_delayed_items()
4645                  * and then a transaction commit runs the same delayed nodes
4646                  * before any delayed worker has done something with the nodes.
4647                  * We must wait for any worker here and not at transaction
4648                  * commit time since that could cause a deadlock.
4649                  * This is a very rare case.
4650                  */
4651                 btrfs_flush_workqueue(fs_info->delayed_workers);
4652
4653                 ret = btrfs_commit_super(fs_info);
4654                 if (ret)
4655                         btrfs_err(fs_info, "commit super ret %d", ret);
4656         }
4657
4658         if (BTRFS_FS_ERROR(fs_info))
4659                 btrfs_error_commit_super(fs_info);
4660
4661         kthread_stop(fs_info->transaction_kthread);
4662         kthread_stop(fs_info->cleaner_kthread);
4663
4664         ASSERT(list_empty(&fs_info->delayed_iputs));
4665         set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags);
4666
4667         if (btrfs_check_quota_leak(fs_info)) {
4668                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4669                 btrfs_err(fs_info, "qgroup reserved space leaked");
4670         }
4671
4672         btrfs_free_qgroup_config(fs_info);
4673         ASSERT(list_empty(&fs_info->delalloc_roots));
4674
4675         if (percpu_counter_sum(&fs_info->delalloc_bytes)) {
4676                 btrfs_info(fs_info, "at unmount delalloc count %lld",
4677                        percpu_counter_sum(&fs_info->delalloc_bytes));
4678         }
4679
4680         if (percpu_counter_sum(&fs_info->ordered_bytes))
4681                 btrfs_info(fs_info, "at unmount dio bytes count %lld",
4682                            percpu_counter_sum(&fs_info->ordered_bytes));
4683
4684         btrfs_sysfs_remove_mounted(fs_info);
4685         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
4686
4687         btrfs_put_block_group_cache(fs_info);
4688
4689         /*
4690          * we must make sure there is not any read request to
4691          * submit after we stopping all workers.
4692          */
4693         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
4694         btrfs_stop_all_workers(fs_info);
4695
4696         /* We shouldn't have any transaction open at this point */
4697         warn_about_uncommitted_trans(fs_info);
4698
4699         clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
4700         free_root_pointers(fs_info, true);
4701         btrfs_free_fs_roots(fs_info);
4702
4703         /*
4704          * We must free the block groups after dropping the fs_roots as we could
4705          * have had an IO error and have left over tree log blocks that aren't
4706          * cleaned up until the fs roots are freed.  This makes the block group
4707          * accounting appear to be wrong because there's pending reserved bytes,
4708          * so make sure we do the block group cleanup afterwards.
4709          */
4710         btrfs_free_block_groups(fs_info);
4711
4712         iput(fs_info->btree_inode);
4713
4714 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4715         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY))
4716                 btrfsic_unmount(fs_info->fs_devices);
4717 #endif
4718
4719         btrfs_mapping_tree_free(&fs_info->mapping_tree);
4720         btrfs_close_devices(fs_info->fs_devices);
4721 }
4722
4723 int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
4724                           int atomic)
4725 {
4726         int ret;
4727         struct inode *btree_inode = buf->pages[0]->mapping->host;
4728
4729         ret = extent_buffer_uptodate(buf);
4730         if (!ret)
4731                 return ret;
4732
4733         ret = verify_parent_transid(&BTRFS_I(btree_inode)->io_tree, buf,
4734                                     parent_transid, atomic);
4735         if (ret == -EAGAIN)
4736                 return ret;
4737         return !ret;
4738 }
4739
4740 void btrfs_mark_buffer_dirty(struct extent_buffer *buf)
4741 {
4742         struct btrfs_fs_info *fs_info = buf->fs_info;
4743         u64 transid = btrfs_header_generation(buf);
4744         int was_dirty;
4745
4746 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4747         /*
4748          * This is a fast path so only do this check if we have sanity tests
4749          * enabled.  Normal people shouldn't be using unmapped buffers as dirty
4750          * outside of the sanity tests.
4751          */
4752         if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags)))
4753                 return;
4754 #endif
4755         btrfs_assert_tree_write_locked(buf);
4756         if (transid != fs_info->generation)
4757                 WARN(1, KERN_CRIT "btrfs transid mismatch buffer %llu, found %llu running %llu\n",
4758                         buf->start, transid, fs_info->generation);
4759         was_dirty = set_extent_buffer_dirty(buf);
4760         if (!was_dirty)
4761                 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4762                                          buf->len,
4763                                          fs_info->dirty_metadata_batch);
4764 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4765         /*
4766          * Since btrfs_mark_buffer_dirty() can be called with item pointer set
4767          * but item data not updated.
4768          * So here we should only check item pointers, not item data.
4769          */
4770         if (btrfs_header_level(buf) == 0 &&
4771             btrfs_check_leaf_relaxed(buf)) {
4772                 btrfs_print_leaf(buf);
4773                 ASSERT(0);
4774         }
4775 #endif
4776 }
4777
4778 static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info,
4779                                         int flush_delayed)
4780 {
4781         /*
4782          * looks as though older kernels can get into trouble with
4783          * this code, they end up stuck in balance_dirty_pages forever
4784          */
4785         int ret;
4786
4787         if (current->flags & PF_MEMALLOC)
4788                 return;
4789
4790         if (flush_delayed)
4791                 btrfs_balance_delayed_items(fs_info);
4792
4793         ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
4794                                      BTRFS_DIRTY_METADATA_THRESH,
4795                                      fs_info->dirty_metadata_batch);
4796         if (ret > 0) {
4797                 balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping);
4798         }
4799 }
4800
4801 void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info)
4802 {
4803         __btrfs_btree_balance_dirty(fs_info, 1);
4804 }
4805
4806 void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info)
4807 {
4808         __btrfs_btree_balance_dirty(fs_info, 0);
4809 }
4810
4811 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info)
4812 {
4813         /* cleanup FS via transaction */
4814         btrfs_cleanup_transaction(fs_info);
4815
4816         mutex_lock(&fs_info->cleaner_mutex);
4817         btrfs_run_delayed_iputs(fs_info);
4818         mutex_unlock(&fs_info->cleaner_mutex);
4819
4820         down_write(&fs_info->cleanup_work_sem);
4821         up_write(&fs_info->cleanup_work_sem);
4822 }
4823
4824 static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info)
4825 {
4826         struct btrfs_root *gang[8];
4827         u64 root_objectid = 0;
4828         int ret;
4829
4830         spin_lock(&fs_info->fs_roots_radix_lock);
4831         while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4832                                              (void **)gang, root_objectid,
4833                                              ARRAY_SIZE(gang))) != 0) {
4834                 int i;
4835
4836                 for (i = 0; i < ret; i++)
4837                         gang[i] = btrfs_grab_root(gang[i]);
4838                 spin_unlock(&fs_info->fs_roots_radix_lock);
4839
4840                 for (i = 0; i < ret; i++) {
4841                         if (!gang[i])
4842                                 continue;
4843                         root_objectid = gang[i]->root_key.objectid;
4844                         btrfs_free_log(NULL, gang[i]);
4845                         btrfs_put_root(gang[i]);
4846                 }
4847                 root_objectid++;
4848                 spin_lock(&fs_info->fs_roots_radix_lock);
4849         }
4850         spin_unlock(&fs_info->fs_roots_radix_lock);
4851         btrfs_free_log_root_tree(NULL, fs_info);
4852 }
4853
4854 static void btrfs_destroy_ordered_extents(struct btrfs_root *root)
4855 {
4856         struct btrfs_ordered_extent *ordered;
4857
4858         spin_lock(&root->ordered_extent_lock);
4859         /*
4860          * This will just short circuit the ordered completion stuff which will
4861          * make sure the ordered extent gets properly cleaned up.
4862          */
4863         list_for_each_entry(ordered, &root->ordered_extents,
4864                             root_extent_list)
4865                 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
4866         spin_unlock(&root->ordered_extent_lock);
4867 }
4868
4869 static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info)
4870 {
4871         struct btrfs_root *root;
4872         struct list_head splice;
4873
4874         INIT_LIST_HEAD(&splice);
4875
4876         spin_lock(&fs_info->ordered_root_lock);
4877         list_splice_init(&fs_info->ordered_roots, &splice);
4878         while (!list_empty(&splice)) {
4879                 root = list_first_entry(&splice, struct btrfs_root,
4880                                         ordered_root);
4881                 list_move_tail(&root->ordered_root,
4882                                &fs_info->ordered_roots);
4883
4884                 spin_unlock(&fs_info->ordered_root_lock);
4885                 btrfs_destroy_ordered_extents(root);
4886
4887                 cond_resched();
4888                 spin_lock(&fs_info->ordered_root_lock);
4889         }
4890         spin_unlock(&fs_info->ordered_root_lock);
4891
4892         /*
4893          * We need this here because if we've been flipped read-only we won't
4894          * get sync() from the umount, so we need to make sure any ordered
4895          * extents that haven't had their dirty pages IO start writeout yet
4896          * actually get run and error out properly.
4897          */
4898         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
4899 }
4900
4901 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
4902                                       struct btrfs_fs_info *fs_info)
4903 {
4904         struct rb_node *node;
4905         struct btrfs_delayed_ref_root *delayed_refs;
4906         struct btrfs_delayed_ref_node *ref;
4907         int ret = 0;
4908
4909         delayed_refs = &trans->delayed_refs;
4910
4911         spin_lock(&delayed_refs->lock);
4912         if (atomic_read(&delayed_refs->num_entries) == 0) {
4913                 spin_unlock(&delayed_refs->lock);
4914                 btrfs_debug(fs_info, "delayed_refs has NO entry");
4915                 return ret;
4916         }
4917
4918         while ((node = rb_first_cached(&delayed_refs->href_root)) != NULL) {
4919                 struct btrfs_delayed_ref_head *head;
4920                 struct rb_node *n;
4921                 bool pin_bytes = false;
4922
4923                 head = rb_entry(node, struct btrfs_delayed_ref_head,
4924                                 href_node);
4925                 if (btrfs_delayed_ref_lock(delayed_refs, head))
4926                         continue;
4927
4928                 spin_lock(&head->lock);
4929                 while ((n = rb_first_cached(&head->ref_tree)) != NULL) {
4930                         ref = rb_entry(n, struct btrfs_delayed_ref_node,
4931                                        ref_node);
4932                         ref->in_tree = 0;
4933                         rb_erase_cached(&ref->ref_node, &head->ref_tree);
4934                         RB_CLEAR_NODE(&ref->ref_node);
4935                         if (!list_empty(&ref->add_list))
4936                                 list_del(&ref->add_list);
4937                         atomic_dec(&delayed_refs->num_entries);
4938                         btrfs_put_delayed_ref(ref);
4939                 }
4940                 if (head->must_insert_reserved)
4941                         pin_bytes = true;
4942                 btrfs_free_delayed_extent_op(head->extent_op);
4943                 btrfs_delete_ref_head(delayed_refs, head);
4944                 spin_unlock(&head->lock);
4945                 spin_unlock(&delayed_refs->lock);
4946                 mutex_unlock(&head->mutex);
4947
4948                 if (pin_bytes) {
4949                         struct btrfs_block_group *cache;
4950
4951                         cache = btrfs_lookup_block_group(fs_info, head->bytenr);
4952                         BUG_ON(!cache);
4953
4954                         spin_lock(&cache->space_info->lock);
4955                         spin_lock(&cache->lock);
4956                         cache->pinned += head->num_bytes;
4957                         btrfs_space_info_update_bytes_pinned(fs_info,
4958                                 cache->space_info, head->num_bytes);
4959                         cache->reserved -= head->num_bytes;
4960                         cache->space_info->bytes_reserved -= head->num_bytes;
4961                         spin_unlock(&cache->lock);
4962                         spin_unlock(&cache->space_info->lock);
4963
4964                         btrfs_put_block_group(cache);
4965
4966                         btrfs_error_unpin_extent_range(fs_info, head->bytenr,
4967                                 head->bytenr + head->num_bytes - 1);
4968                 }
4969                 btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head);
4970                 btrfs_put_delayed_ref_head(head);
4971                 cond_resched();
4972                 spin_lock(&delayed_refs->lock);
4973         }
4974         btrfs_qgroup_destroy_extent_records(trans);
4975
4976         spin_unlock(&delayed_refs->lock);
4977
4978         return ret;
4979 }
4980
4981 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root)
4982 {
4983         struct btrfs_inode *btrfs_inode;
4984         struct list_head splice;
4985
4986         INIT_LIST_HEAD(&splice);
4987
4988         spin_lock(&root->delalloc_lock);
4989         list_splice_init(&root->delalloc_inodes, &splice);
4990
4991         while (!list_empty(&splice)) {
4992                 struct inode *inode = NULL;
4993                 btrfs_inode = list_first_entry(&splice, struct btrfs_inode,
4994                                                delalloc_inodes);
4995                 __btrfs_del_delalloc_inode(root, btrfs_inode);
4996                 spin_unlock(&root->delalloc_lock);
4997
4998                 /*
4999                  * Make sure we get a live inode and that it'll not disappear
5000                  * meanwhile.
5001                  */
5002                 inode = igrab(&btrfs_inode->vfs_inode);
5003                 if (inode) {
5004                         invalidate_inode_pages2(inode->i_mapping);
5005                         iput(inode);
5006                 }
5007                 spin_lock(&root->delalloc_lock);
5008         }
5009         spin_unlock(&root->delalloc_lock);
5010 }
5011
5012 static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info)
5013 {
5014         struct btrfs_root *root;
5015         struct list_head splice;
5016
5017         INIT_LIST_HEAD(&splice);
5018
5019         spin_lock(&fs_info->delalloc_root_lock);
5020         list_splice_init(&fs_info->delalloc_roots, &splice);
5021         while (!list_empty(&splice)) {
5022                 root = list_first_entry(&splice, struct btrfs_root,
5023                                          delalloc_root);
5024                 root = btrfs_grab_root(root);
5025                 BUG_ON(!root);
5026                 spin_unlock(&fs_info->delalloc_root_lock);
5027
5028                 btrfs_destroy_delalloc_inodes(root);
5029                 btrfs_put_root(root);
5030
5031                 spin_lock(&fs_info->delalloc_root_lock);
5032         }
5033         spin_unlock(&fs_info->delalloc_root_lock);
5034 }
5035
5036 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
5037                                         struct extent_io_tree *dirty_pages,
5038                                         int mark)
5039 {
5040         int ret;
5041         struct extent_buffer *eb;
5042         u64 start = 0;
5043         u64 end;
5044
5045         while (1) {
5046                 ret = find_first_extent_bit(dirty_pages, start, &start, &end,
5047                                             mark, NULL);
5048                 if (ret)
5049                         break;
5050
5051                 clear_extent_bits(dirty_pages, start, end, mark);
5052                 while (start <= end) {
5053                         eb = find_extent_buffer(fs_info, start);
5054                         start += fs_info->nodesize;
5055                         if (!eb)
5056                                 continue;
5057                         wait_on_extent_buffer_writeback(eb);
5058
5059                         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY,
5060                                                &eb->bflags))
5061                                 clear_extent_buffer_dirty(eb);
5062                         free_extent_buffer_stale(eb);
5063                 }
5064         }
5065
5066         return ret;
5067 }
5068
5069 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
5070                                        struct extent_io_tree *unpin)
5071 {
5072         u64 start;
5073         u64 end;
5074         int ret;
5075
5076         while (1) {
5077                 struct extent_state *cached_state = NULL;
5078
5079                 /*
5080                  * The btrfs_finish_extent_commit() may get the same range as
5081                  * ours between find_first_extent_bit and clear_extent_dirty.
5082                  * Hence, hold the unused_bg_unpin_mutex to avoid double unpin
5083                  * the same extent range.
5084                  */
5085                 mutex_lock(&fs_info->unused_bg_unpin_mutex);
5086                 ret = find_first_extent_bit(unpin, 0, &start, &end,
5087                                             EXTENT_DIRTY, &cached_state);
5088                 if (ret) {
5089                         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5090                         break;
5091                 }
5092
5093                 clear_extent_dirty(unpin, start, end, &cached_state);
5094                 free_extent_state(cached_state);
5095                 btrfs_error_unpin_extent_range(fs_info, start, end);
5096                 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5097                 cond_resched();
5098         }
5099
5100         return 0;
5101 }
5102
5103 static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache)
5104 {
5105         struct inode *inode;
5106
5107         inode = cache->io_ctl.inode;
5108         if (inode) {
5109                 invalidate_inode_pages2(inode->i_mapping);
5110                 BTRFS_I(inode)->generation = 0;
5111                 cache->io_ctl.inode = NULL;
5112                 iput(inode);
5113         }
5114         ASSERT(cache->io_ctl.pages == NULL);
5115         btrfs_put_block_group(cache);
5116 }
5117
5118 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
5119                              struct btrfs_fs_info *fs_info)
5120 {
5121         struct btrfs_block_group *cache;
5122
5123         spin_lock(&cur_trans->dirty_bgs_lock);
5124         while (!list_empty(&cur_trans->dirty_bgs)) {
5125                 cache = list_first_entry(&cur_trans->dirty_bgs,
5126                                          struct btrfs_block_group,
5127                                          dirty_list);
5128
5129                 if (!list_empty(&cache->io_list)) {
5130                         spin_unlock(&cur_trans->dirty_bgs_lock);
5131                         list_del_init(&cache->io_list);
5132                         btrfs_cleanup_bg_io(cache);
5133                         spin_lock(&cur_trans->dirty_bgs_lock);
5134                 }
5135
5136                 list_del_init(&cache->dirty_list);
5137                 spin_lock(&cache->lock);
5138                 cache->disk_cache_state = BTRFS_DC_ERROR;
5139                 spin_unlock(&cache->lock);
5140
5141                 spin_unlock(&cur_trans->dirty_bgs_lock);
5142                 btrfs_put_block_group(cache);
5143                 btrfs_delayed_refs_rsv_release(fs_info, 1);
5144                 spin_lock(&cur_trans->dirty_bgs_lock);
5145         }
5146         spin_unlock(&cur_trans->dirty_bgs_lock);
5147
5148         /*
5149          * Refer to the definition of io_bgs member for details why it's safe
5150          * to use it without any locking
5151          */
5152         while (!list_empty(&cur_trans->io_bgs)) {
5153                 cache = list_first_entry(&cur_trans->io_bgs,
5154                                          struct btrfs_block_group,
5155                                          io_list);
5156
5157                 list_del_init(&cache->io_list);
5158                 spin_lock(&cache->lock);
5159                 cache->disk_cache_state = BTRFS_DC_ERROR;
5160                 spin_unlock(&cache->lock);
5161                 btrfs_cleanup_bg_io(cache);
5162         }
5163 }
5164
5165 void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans,
5166                                    struct btrfs_fs_info *fs_info)
5167 {
5168         struct btrfs_device *dev, *tmp;
5169
5170         btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
5171         ASSERT(list_empty(&cur_trans->dirty_bgs));
5172         ASSERT(list_empty(&cur_trans->io_bgs));
5173
5174         list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list,
5175                                  post_commit_list) {
5176                 list_del_init(&dev->post_commit_list);
5177         }
5178
5179         btrfs_destroy_delayed_refs(cur_trans, fs_info);
5180
5181         cur_trans->state = TRANS_STATE_COMMIT_START;
5182         wake_up(&fs_info->transaction_blocked_wait);
5183
5184         cur_trans->state = TRANS_STATE_UNBLOCKED;
5185         wake_up(&fs_info->transaction_wait);
5186
5187         btrfs_destroy_delayed_inodes(fs_info);
5188
5189         btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages,
5190                                      EXTENT_DIRTY);
5191         btrfs_destroy_pinned_extent(fs_info, &cur_trans->pinned_extents);
5192
5193         btrfs_free_redirty_list(cur_trans);
5194
5195         cur_trans->state =TRANS_STATE_COMPLETED;
5196         wake_up(&cur_trans->commit_wait);
5197 }
5198
5199 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info)
5200 {
5201         struct btrfs_transaction *t;
5202
5203         mutex_lock(&fs_info->transaction_kthread_mutex);
5204
5205         spin_lock(&fs_info->trans_lock);
5206         while (!list_empty(&fs_info->trans_list)) {
5207                 t = list_first_entry(&fs_info->trans_list,
5208                                      struct btrfs_transaction, list);
5209                 if (t->state >= TRANS_STATE_COMMIT_START) {
5210                         refcount_inc(&t->use_count);
5211                         spin_unlock(&fs_info->trans_lock);
5212                         btrfs_wait_for_commit(fs_info, t->transid);
5213                         btrfs_put_transaction(t);
5214                         spin_lock(&fs_info->trans_lock);
5215                         continue;
5216                 }
5217                 if (t == fs_info->running_transaction) {
5218                         t->state = TRANS_STATE_COMMIT_DOING;
5219                         spin_unlock(&fs_info->trans_lock);
5220                         /*
5221                          * We wait for 0 num_writers since we don't hold a trans
5222                          * handle open currently for this transaction.
5223                          */
5224                         wait_event(t->writer_wait,
5225                                    atomic_read(&t->num_writers) == 0);
5226                 } else {
5227                         spin_unlock(&fs_info->trans_lock);
5228                 }
5229                 btrfs_cleanup_one_transaction(t, fs_info);
5230
5231                 spin_lock(&fs_info->trans_lock);
5232                 if (t == fs_info->running_transaction)
5233                         fs_info->running_transaction = NULL;
5234                 list_del_init(&t->list);
5235                 spin_unlock(&fs_info->trans_lock);
5236
5237                 btrfs_put_transaction(t);
5238                 trace_btrfs_transaction_commit(fs_info);
5239                 spin_lock(&fs_info->trans_lock);
5240         }
5241         spin_unlock(&fs_info->trans_lock);
5242         btrfs_destroy_all_ordered_extents(fs_info);
5243         btrfs_destroy_delayed_inodes(fs_info);
5244         btrfs_assert_delayed_root_empty(fs_info);
5245         btrfs_destroy_all_delalloc_inodes(fs_info);
5246         btrfs_drop_all_logs(fs_info);
5247         mutex_unlock(&fs_info->transaction_kthread_mutex);
5248
5249         return 0;
5250 }
5251
5252 int btrfs_init_root_free_objectid(struct btrfs_root *root)
5253 {
5254         struct btrfs_path *path;
5255         int ret;
5256         struct extent_buffer *l;
5257         struct btrfs_key search_key;
5258         struct btrfs_key found_key;
5259         int slot;
5260
5261         path = btrfs_alloc_path();
5262         if (!path)
5263                 return -ENOMEM;
5264
5265         search_key.objectid = BTRFS_LAST_FREE_OBJECTID;
5266         search_key.type = -1;
5267         search_key.offset = (u64)-1;
5268         ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0);
5269         if (ret < 0)
5270                 goto error;
5271         BUG_ON(ret == 0); /* Corruption */
5272         if (path->slots[0] > 0) {
5273                 slot = path->slots[0] - 1;
5274                 l = path->nodes[0];
5275                 btrfs_item_key_to_cpu(l, &found_key, slot);
5276                 root->free_objectid = max_t(u64, found_key.objectid + 1,
5277                                             BTRFS_FIRST_FREE_OBJECTID);
5278         } else {
5279                 root->free_objectid = BTRFS_FIRST_FREE_OBJECTID;
5280         }
5281         ret = 0;
5282 error:
5283         btrfs_free_path(path);
5284         return ret;
5285 }
5286
5287 int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid)
5288 {
5289         int ret;
5290         mutex_lock(&root->objectid_mutex);
5291
5292         if (unlikely(root->free_objectid >= BTRFS_LAST_FREE_OBJECTID)) {
5293                 btrfs_warn(root->fs_info,
5294                            "the objectid of root %llu reaches its highest value",
5295                            root->root_key.objectid);
5296                 ret = -ENOSPC;
5297                 goto out;
5298         }
5299
5300         *objectid = root->free_objectid++;
5301         ret = 0;
5302 out:
5303         mutex_unlock(&root->objectid_mutex);
5304         return ret;
5305 }