Btrfs: add the ability to cache a pointer into the eb

[platform/kernel/linux-starfive.git] / fs / btrfs / ctree.c

diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index dede441..8af24da 100644 (file)
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -156,10 +156,23 @@ struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
 {
        struct extent_buffer *eb;
 
-       rcu_read_lock();
-       eb = rcu_dereference(root->node);
-       extent_buffer_get(eb);
-       rcu_read_unlock();
+       while (1) {
+               rcu_read_lock();
+               eb = rcu_dereference(root->node);
+
+               /*
+                * RCU really hurts here, we could free up the root node because
+                * it was cow'ed but we may not get the new root node yet so do
+                * the inc_not_zero dance and if it doesn't work then
+                * synchronize_rcu and try again.
+                */
+               if (atomic_inc_not_zero(&eb->refs)) {
+                       rcu_read_unlock();
+                       break;
+               }
+               rcu_read_unlock();
+               synchronize_rcu();
+       }
        return eb;
 }
 
@@ -240,7 +253,7 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 
        cow = btrfs_alloc_free_block(trans, root, buf->len, 0,
                                     new_root_objectid, &disk_key, level,
-                                    buf->start, 0);
+                                    buf->start, 0, 1);
        if (IS_ERR(cow))
                return PTR_ERR(cow);
 
@@ -261,9 +274,9 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 
        WARN_ON(btrfs_header_generation(buf) > trans->transid);
        if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
-               ret = btrfs_inc_ref(trans, root, cow, 1);
+               ret = btrfs_inc_ref(trans, root, cow, 1, 1);
        else
-               ret = btrfs_inc_ref(trans, root, cow, 0);
+               ret = btrfs_inc_ref(trans, root, cow, 0, 1);
 
        if (ret)
                return ret;
@@ -350,14 +363,14 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
                if ((owner == root->root_key.objectid ||
                     root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
                    !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
-                       ret = btrfs_inc_ref(trans, root, buf, 1);
+                       ret = btrfs_inc_ref(trans, root, buf, 1, 1);
                        BUG_ON(ret);
 
                        if (root->root_key.objectid ==
                            BTRFS_TREE_RELOC_OBJECTID) {
-                               ret = btrfs_dec_ref(trans, root, buf, 0);
+                               ret = btrfs_dec_ref(trans, root, buf, 0, 1);
                                BUG_ON(ret);
-                               ret = btrfs_inc_ref(trans, root, cow, 1);
+                               ret = btrfs_inc_ref(trans, root, cow, 1, 1);
                                BUG_ON(ret);
                        }
                        new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
@@ -365,9 +378,9 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
 
                        if (root->root_key.objectid ==
                            BTRFS_TREE_RELOC_OBJECTID)
-                               ret = btrfs_inc_ref(trans, root, cow, 1);
+                               ret = btrfs_inc_ref(trans, root, cow, 1, 1);
                        else
-                               ret = btrfs_inc_ref(trans, root, cow, 0);
+                               ret = btrfs_inc_ref(trans, root, cow, 0, 1);
                        BUG_ON(ret);
                }
                if (new_flags != 0) {
@@ -381,11 +394,11 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
                if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
                        if (root->root_key.objectid ==
                            BTRFS_TREE_RELOC_OBJECTID)
-                               ret = btrfs_inc_ref(trans, root, cow, 1);
+                               ret = btrfs_inc_ref(trans, root, cow, 1, 1);
                        else
-                               ret = btrfs_inc_ref(trans, root, cow, 0);
+                               ret = btrfs_inc_ref(trans, root, cow, 0, 1);
                        BUG_ON(ret);
-                       ret = btrfs_dec_ref(trans, root, buf, 1);
+                       ret = btrfs_dec_ref(trans, root, buf, 1, 1);
                        BUG_ON(ret);
                }
                clean_tree_block(trans, root, buf);
@@ -446,7 +459,7 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
 
        cow = btrfs_alloc_free_block(trans, root, buf->len, parent_start,
                                     root->root_key.objectid, &disk_key,
-                                    level, search_start, empty_size);
+                                    level, search_start, empty_size, 1);
        if (IS_ERR(cow))
                return PTR_ERR(cow);
 
@@ -484,7 +497,7 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
                rcu_assign_pointer(root->node, cow);
 
                btrfs_free_tree_block(trans, root, buf, parent_start,
-                                     last_ref);
+                                     last_ref, 1);
                free_extent_buffer(buf);
                add_root_to_dirty_list(root);
        } else {
@@ -500,11 +513,11 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
                                              trans->transid);
                btrfs_mark_buffer_dirty(parent);
                btrfs_free_tree_block(trans, root, buf, parent_start,
-                                     last_ref);
+                                     last_ref, 1);
        }
        if (unlock_orig)
                btrfs_tree_unlock(buf);
-       free_extent_buffer(buf);
+       free_extent_buffer_stale(buf);
        btrfs_mark_buffer_dirty(cow);
        *cow_ret = cow;
        return 0;
@@ -957,9 +970,9 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
                free_extent_buffer(mid);
 
                root_sub_used(root, mid->len);
-               btrfs_free_tree_block(trans, root, mid, 0, 1);
+               btrfs_free_tree_block(trans, root, mid, 0, 1, 0);
                /* once for the root ptr */
-               free_extent_buffer(mid);
+               free_extent_buffer_stale(mid);
                return 0;
        }
        if (btrfs_header_nritems(mid) >
@@ -1015,8 +1028,8 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
                        if (wret)
                                ret = wret;
                        root_sub_used(root, right->len);
-                       btrfs_free_tree_block(trans, root, right, 0, 1);
-                       free_extent_buffer(right);
+                       btrfs_free_tree_block(trans, root, right, 0, 1, 0);
+                       free_extent_buffer_stale(right);
                        right = NULL;
                } else {
                        struct btrfs_disk_key right_key;
@@ -1055,8 +1068,8 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
                if (wret)
                        ret = wret;
                root_sub_used(root, mid->len);
-               btrfs_free_tree_block(trans, root, mid, 0, 1);
-               free_extent_buffer(mid);
+               btrfs_free_tree_block(trans, root, mid, 0, 1, 0);
+               free_extent_buffer_stale(mid);
                mid = NULL;
        } else {
                /* update the parent key to reflect our changes */
@@ -2089,7 +2102,7 @@ static noinline int insert_new_root(struct btrfs_trans_handle *trans,
 
        c = btrfs_alloc_free_block(trans, root, root->nodesize, 0,
                                   root->root_key.objectid, &lower_key,
-                                  level, root->node->start, 0);
+                                  level, root->node->start, 0, 0);
        if (IS_ERR(c))
                return PTR_ERR(c);
 
@@ -2216,7 +2229,7 @@ static noinline int split_node(struct btrfs_trans_handle *trans,
 
        split = btrfs_alloc_free_block(trans, root, root->nodesize, 0,
                                        root->root_key.objectid,
-                                       &disk_key, level, c->start, 0);
+                                       &disk_key, level, c->start, 0, 0);
        if (IS_ERR(split))
                return PTR_ERR(split);
 
@@ -2320,6 +2333,7 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
 {
        struct extent_buffer *left = path->nodes[0];
        struct extent_buffer *upper = path->nodes[1];
+       struct btrfs_map_token token;
        struct btrfs_disk_key disk_key;
        int slot;
        u32 i;
@@ -2331,6 +2345,8 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
        u32 data_end;
        u32 this_item_size;
 
+       btrfs_init_map_token(&token);
+
        if (empty)
                nr = 0;
        else
@@ -2408,8 +2424,8 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
        push_space = BTRFS_LEAF_DATA_SIZE(root);
        for (i = 0; i < right_nritems; i++) {
                item = btrfs_item_nr(right, i);
-               push_space -= btrfs_item_size(right, item);
-               btrfs_set_item_offset(right, item, push_space);
+               push_space -= btrfs_token_item_size(right, item, &token);
+               btrfs_set_token_item_offset(right, item, push_space, &token);
        }
 
        left_nritems -= push_items;
@@ -2540,6 +2556,9 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
        int wret;
        u32 this_item_size;
        u32 old_left_item_size;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        if (empty)
                nr = min(right_nritems, max_slot);
@@ -2600,9 +2619,10 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
 
                item = btrfs_item_nr(left, i);
 
-               ioff = btrfs_item_offset(left, item);
-               btrfs_set_item_offset(left, item,
-                     ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size));
+               ioff = btrfs_token_item_offset(left, item, &token);
+               btrfs_set_token_item_offset(left, item,
+                     ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size),
+                     &token);
        }
        btrfs_set_header_nritems(left, old_left_nritems + push_items);
 
@@ -2632,8 +2652,9 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
        for (i = 0; i < right_nritems; i++) {
                item = btrfs_item_nr(right, i);
 
-               push_space = push_space - btrfs_item_size(right, item);
-               btrfs_set_item_offset(right, item, push_space);
+               push_space = push_space - btrfs_token_item_size(right,
+                                                               item, &token);
+               btrfs_set_token_item_offset(right, item, push_space, &token);
        }
 
        btrfs_mark_buffer_dirty(left);
@@ -2754,6 +2775,9 @@ static noinline int copy_for_split(struct btrfs_trans_handle *trans,
        int ret = 0;
        int wret;
        struct btrfs_disk_key disk_key;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        nritems = nritems - mid;
        btrfs_set_header_nritems(right, nritems);
@@ -2775,8 +2799,9 @@ static noinline int copy_for_split(struct btrfs_trans_handle *trans,
                struct btrfs_item *item = btrfs_item_nr(right, i);
                u32 ioff;
 
-               ioff = btrfs_item_offset(right, item);
-               btrfs_set_item_offset(right, item, ioff + rt_data_off);
+               ioff = btrfs_token_item_offset(right, item, &token);
+               btrfs_set_token_item_offset(right, item,
+                                           ioff + rt_data_off, &token);
        }
 
        btrfs_set_header_nritems(l, mid);
@@ -2970,7 +2995,7 @@ again:
 
        right = btrfs_alloc_free_block(trans, root, root->leafsize, 0,
                                        root->root_key.objectid,
-                                       &disk_key, 0, l->start, 0);
+                                       &disk_key, 0, l->start, 0, 0);
        if (IS_ERR(right))
                return PTR_ERR(right);
 
@@ -3271,6 +3296,9 @@ int btrfs_truncate_item(struct btrfs_trans_handle *trans,
        unsigned int old_size;
        unsigned int size_diff;
        int i;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
        slot = path->slots[0];
@@ -3297,8 +3325,9 @@ int btrfs_truncate_item(struct btrfs_trans_handle *trans,
                u32 ioff;
                item = btrfs_item_nr(leaf, i);
 
-               ioff = btrfs_item_offset(leaf, item);
-               btrfs_set_item_offset(leaf, item, ioff + size_diff);
+               ioff = btrfs_token_item_offset(leaf, item, &token);
+               btrfs_set_token_item_offset(leaf, item,
+                                           ioff + size_diff, &token);
        }
 
        /* shift the data */
@@ -3368,6 +3397,9 @@ int btrfs_extend_item(struct btrfs_trans_handle *trans,
        unsigned int old_data;
        unsigned int old_size;
        int i;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
 
@@ -3397,8 +3429,9 @@ int btrfs_extend_item(struct btrfs_trans_handle *trans,
                u32 ioff;
                item = btrfs_item_nr(leaf, i);
 
-               ioff = btrfs_item_offset(leaf, item);
-               btrfs_set_item_offset(leaf, item, ioff - data_size);
+               ioff = btrfs_token_item_offset(leaf, item, &token);
+               btrfs_set_token_item_offset(leaf, item,
+                                           ioff - data_size, &token);
        }
 
        /* shift the data */
@@ -3441,6 +3474,9 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
        unsigned int data_end;
        struct btrfs_disk_key disk_key;
        struct btrfs_key found_key;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        for (i = 0; i < nr; i++) {
                if (total_size + data_size[i] + sizeof(struct btrfs_item) >
@@ -3506,8 +3542,9 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
                        u32 ioff;
 
                        item = btrfs_item_nr(leaf, i);
-                       ioff = btrfs_item_offset(leaf, item);
-                       btrfs_set_item_offset(leaf, item, ioff - total_data);
+                       ioff = btrfs_token_item_offset(leaf, item, &token);
+                       btrfs_set_token_item_offset(leaf, item,
+                                                   ioff - total_data, &token);
                }
                /* shift the items */
                memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
@@ -3534,9 +3571,10 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
                btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
                btrfs_set_item_key(leaf, &disk_key, slot + i);
                item = btrfs_item_nr(leaf, slot + i);
-               btrfs_set_item_offset(leaf, item, data_end - data_size[i]);
+               btrfs_set_token_item_offset(leaf, item,
+                                           data_end - data_size[i], &token);
                data_end -= data_size[i];
-               btrfs_set_item_size(leaf, item, data_size[i]);
+               btrfs_set_token_item_size(leaf, item, data_size[i], &token);
        }
        btrfs_set_header_nritems(leaf, nritems + nr);
        btrfs_mark_buffer_dirty(leaf);
@@ -3575,6 +3613,9 @@ int setup_items_for_insert(struct btrfs_trans_handle *trans,
        int ret;
        struct extent_buffer *leaf;
        int slot;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
        slot = path->slots[0];
@@ -3606,8 +3647,9 @@ int setup_items_for_insert(struct btrfs_trans_handle *trans,
                        u32 ioff;
 
                        item = btrfs_item_nr(leaf, i);
-                       ioff = btrfs_item_offset(leaf, item);
-                       btrfs_set_item_offset(leaf, item, ioff - total_data);
+                       ioff = btrfs_token_item_offset(leaf, item, &token);
+                       btrfs_set_token_item_offset(leaf, item,
+                                                   ioff - total_data, &token);
                }
                /* shift the items */
                memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
@@ -3626,9 +3668,10 @@ int setup_items_for_insert(struct btrfs_trans_handle *trans,
                btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
                btrfs_set_item_key(leaf, &disk_key, slot + i);
                item = btrfs_item_nr(leaf, slot + i);
-               btrfs_set_item_offset(leaf, item, data_end - data_size[i]);
+               btrfs_set_token_item_offset(leaf, item,
+                                           data_end - data_size[i], &token);
                data_end -= data_size[i];
-               btrfs_set_item_size(leaf, item, data_size[i]);
+               btrfs_set_token_item_size(leaf, item, data_size[i], &token);
        }
 
        btrfs_set_header_nritems(leaf, nritems + nr);
@@ -3781,7 +3824,9 @@ static noinline int btrfs_del_leaf(struct btrfs_trans_handle *trans,
 
        root_sub_used(root, leaf->len);
 
-       btrfs_free_tree_block(trans, root, leaf, 0, 1);
+       extent_buffer_get(leaf);
+       btrfs_free_tree_block(trans, root, leaf, 0, 1, 0);
+       free_extent_buffer_stale(leaf);
        return 0;
 }
 /*
@@ -3799,6 +3844,9 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
        int wret;
        int i;
        u32 nritems;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
        last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
@@ -3820,8 +3868,9 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
                        u32 ioff;
 
                        item = btrfs_item_nr(leaf, i);
-                       ioff = btrfs_item_offset(leaf, item);
-                       btrfs_set_item_offset(leaf, item, ioff + dsize);
+                       ioff = btrfs_token_item_offset(leaf, item, &token);
+                       btrfs_set_token_item_offset(leaf, item,
+                                                   ioff + dsize, &token);
                }
 
                memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),