btrfs: scrub: avoid unnecessary csum tree search preparing stripes

One of the bottleneck of the new scrub code is the extra csum tree
search.

The old code would only do the csum tree search for each scrub bio,
which can be as large as 512KiB, thus they can afford to allocate a new
path each time.

But the new scrub code is doing csum tree search for each stripe, which
is only 64KiB, this means we'd better re-use the same csum path during
each search.

This patch would introduce a per-sctx path for csum tree search, as we
don't need to re-allocate the path every time we need to do a csum tree
search.

With this change we can further improve the queue depth and improve the
scrub read performance:

Before (with regression and cached extent tree path):

 Device         r/s      rkB/s   rrqm/s  %rrqm r_await rareq-sz aqu-sz  %util
 nvme0n1p3 15875.00 1013328.00    12.00   0.08    0.08    63.83   1.35 100.00

After (with both cached extent/csum tree path):

 nvme0n1p3 17759.00 1133280.00    10.00   0.06    0.08    63.81   1.50 100.00

Fixes: e02ee89baa66 ("btrfs: scrub: switch scrub_simple_mirror() to scrub_stripe infrastructure")
CC: stable@vger.kernel.org # 6.4+
Signed-off-by: Qu Wenruo <wqu@suse.com>
Reviewed-by: David Sterba <dsterba@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>

diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 696bf695d8eb0018f0074388eeab7eb28f4040de..1ce5dd1544995f16723769b5d7c66d5658cfe5fe 100644 (file)
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -597,29 +597,37 @@ fail:
  * Each bit represents a sector. Thus caller should ensure @csum_buf passed
  * in is large enough to contain all csums.
  */
-int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
-                             u8 *csum_buf, unsigned long *csum_bitmap,
-                             bool search_commit)
+int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path,
+                             u64 start, u64 end, u8 *csum_buf,
+                             unsigned long *csum_bitmap)
 {
        struct btrfs_fs_info *fs_info = root->fs_info;
        struct btrfs_key key;
-       struct btrfs_path *path;
        struct extent_buffer *leaf;
        struct btrfs_csum_item *item;
        const u64 orig_start = start;
+       bool free_path = false;
        int ret;
 
        ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
               IS_ALIGNED(end + 1, fs_info->sectorsize));
 
-       path = btrfs_alloc_path();
-       if (!path)
-               return -ENOMEM;
+       if (!path) {
+               path = btrfs_alloc_path();
+               if (!path)
+                       return -ENOMEM;
+               free_path = true;
+       }
 
-       if (search_commit) {
-               path->skip_locking = 1;
-               path->reada = READA_FORWARD;
-               path->search_commit_root = 1;
+       /* Check if we can reuse the previous path. */
+       if (path->nodes[0]) {
+               btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
+
+               if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
+                   key.type == BTRFS_EXTENT_CSUM_KEY &&
+                   key.offset <= start)
+                       goto search_forward;
+               btrfs_release_path(path);
        }
 
        key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
@@ -656,6 +664,7 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
                }
        }
 
+search_forward:
        while (start <= end) {
                u64 csum_end;
 
@@ -712,7 +721,8 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
        }
        ret = 0;
 fail:
-       btrfs_free_path(path);
+       if (free_path)
+               btrfs_free_path(path);
        return ret;
 }
 

diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h
index 4ec669b690080a85db3625b3d478b3e348db8693..04bd2d34efb14b535fb9fd69c993e030aa19c351 100644 (file)
--- a/fs/btrfs/file-item.h
+++ b/fs/btrfs/file-item.h
@@ -57,9 +57,9 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end,
 int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end,
                            struct list_head *list, int search_commit,
                            bool nowait);
-int btrfs_lookup_csums_bitmap(struct btrfs_root *root, u64 start, u64 end,
-                             u8 *csum_buf, unsigned long *csum_bitmap,
-                             bool search_commit);
+int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path,
+                             u64 start, u64 end, u8 *csum_buf,
+                             unsigned long *csum_bitmap);
 void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
                                     const struct btrfs_path *path,
                                     struct btrfs_file_extent_item *fi,

diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index f97fc62fbab2609a06580d8eea267a8a26886535..3e014b9370a3f22f9a6396fdc42841f3ca2d058d 100644 (file)
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -2105,8 +2105,8 @@ static void fill_data_csums(struct btrfs_raid_bio *rbio)
                goto error;
        }
 
-       ret = btrfs_lookup_csums_bitmap(csum_root, start, start + len - 1,
-                                       rbio->csum_buf, rbio->csum_bitmap, false);
+       ret = btrfs_lookup_csums_bitmap(csum_root, NULL, start, start + len - 1,
+                                       rbio->csum_buf, rbio->csum_bitmap);
        if (ret < 0)
                goto error;
        if (bitmap_empty(rbio->csum_bitmap, len >> fs_info->sectorsize_bits))

diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
index 6aabce141d65691cf640106293fe96559520d5fe..d04fd3c6e927bf5107fef9b0e19ad12f7a87f7bc 100644 (file)
--- a/fs/btrfs/scrub.c
+++ b/fs/btrfs/scrub.c
@@ -176,6 +176,7 @@ struct scrub_ctx {
        struct scrub_stripe     *raid56_data_stripes;
        struct btrfs_fs_info    *fs_info;
        struct btrfs_path       extent_path;
+       struct btrfs_path       csum_path;
        int                     first_free;
        int                     cur_stripe;
        atomic_t                cancel_req;
@@ -342,6 +343,8 @@ static noinline_for_stack struct scrub_ctx *scrub_setup_ctx(
        sctx->fs_info = fs_info;
        sctx->extent_path.search_commit_root = 1;
        sctx->extent_path.skip_locking = 1;
+       sctx->csum_path.search_commit_root = 1;
+       sctx->csum_path.skip_locking = 1;
        for (i = 0; i < SCRUB_STRIPES_PER_SCTX; i++) {
                int ret;
 
@@ -1471,6 +1474,7 @@ static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe)
  */
 static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg,
                                        struct btrfs_path *extent_path,
+                                       struct btrfs_path *csum_path,
                                        struct btrfs_device *dev, u64 physical,
                                        int mirror_num, u64 logical_start,
                                        u32 logical_len,
@@ -1562,9 +1566,9 @@ static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg,
                 */
                ASSERT(BITS_PER_LONG >= BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits);
 
-               ret = btrfs_lookup_csums_bitmap(csum_root, stripe->logical,
-                                               stripe_end, stripe->csums,
-                                               &csum_bitmap, true);
+               ret = btrfs_lookup_csums_bitmap(csum_root, csum_path,
+                                               stripe->logical, stripe_end,
+                                               stripe->csums, &csum_bitmap);
                if (ret < 0)
                        goto out;
                if (ret > 0)
@@ -1766,9 +1770,9 @@ static int queue_scrub_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *
 
        /* We can queue one stripe using the remaining slot. */
        scrub_reset_stripe(stripe);
-       ret = scrub_find_fill_first_stripe(bg, &sctx->extent_path, dev,
-                                          physical, mirror_num, logical,
-                                          length, stripe);
+       ret = scrub_find_fill_first_stripe(bg, &sctx->extent_path,
+                                          &sctx->csum_path, dev, physical,
+                                          mirror_num, logical, length, stripe);
        /* Either >0 as no more extents or <0 for error. */
        if (ret)
                return ret;
@@ -1787,6 +1791,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
        struct btrfs_raid_bio *rbio;
        struct btrfs_io_context *bioc = NULL;
        struct btrfs_path extent_path = { 0 };
+       struct btrfs_path csum_path = { 0 };
        struct bio *bio;
        struct scrub_stripe *stripe;
        bool all_empty = true;
@@ -1798,12 +1803,14 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
        ASSERT(sctx->raid56_data_stripes);
 
        /*
-        * For data stripe search, we cannot re-use the same extent path, as
-        * the data stripe bytenr may be smaller than previous extent.  Thus we
-        * have to use our own extent path.
+        * For data stripe search, we cannot re-use the same extent/csum paths,
+        * as the data stripe bytenr may be smaller than previous extent.  Thus
+        * we have to use our own extent/csum paths.
         */
        extent_path.search_commit_root = 1;
        extent_path.skip_locking = 1;
+       csum_path.search_commit_root = 1;
+       csum_path.skip_locking = 1;
 
        for (int i = 0; i < data_stripes; i++) {
                int stripe_index;
@@ -1819,7 +1826,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
 
                scrub_reset_stripe(stripe);
                set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state);
-               ret = scrub_find_fill_first_stripe(bg, &extent_path,
+               ret = scrub_find_fill_first_stripe(bg, &extent_path, &csum_path,
                                map->stripes[stripe_index].dev, physical, 1,
                                full_stripe_start + btrfs_stripe_nr_to_offset(i),
                                BTRFS_STRIPE_LEN, stripe);
@@ -1948,6 +1955,7 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx,
        btrfs_bio_counter_dec(fs_info);
 
        btrfs_release_path(&extent_path);
+       btrfs_release_path(&csum_path);
 out:
        return ret;
 }
@@ -2237,6 +2245,7 @@ out:
        if (!ret)
                ret = ret2;
        btrfs_release_path(&sctx->extent_path);
+       btrfs_release_path(&sctx->csum_path);
 
        if (sctx->raid56_data_stripes) {
                for (int i = 0; i < nr_data_stripes(map); i++)