bpf: Refactor codes into bpf_local_storage_destroy

This patch first renames bpf_local_storage_unlink_nolock to
bpf_local_storage_destroy(). It better reflects that it is only
used when the storage's owner (sk/task/cgrp/inode) is being kfree().

All bpf_local_storage_destroy's caller is taking the spin lock and
then free the storage. This patch also moves these two steps into
the bpf_local_storage_destroy.

This is a preparation work for a later patch that uses
bpf_mem_cache_alloc/free in the bpf_local_storage.

Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230308065936.1550103-3-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h
index 502ad70..5908a95 100644 (file)
--- a/include/linux/bpf_local_storage.h
+++ b/include/linux/bpf_local_storage.h
@@ -128,7 +128,7 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
                         struct bpf_local_storage_map *smap,
                         bool cacheit_lockit);
 
-bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage);
+void bpf_local_storage_destroy(struct bpf_local_storage *local_storage);
 
 void bpf_local_storage_map_free(struct bpf_map *map,
                                struct bpf_local_storage_cache *cache,

diff --git a/kernel/bpf/bpf_cgrp_storage.c b/kernel/bpf/bpf_cgrp_storage.c
index 9ae07ae..492594d 100644 (file)
--- a/kernel/bpf/bpf_cgrp_storage.c
+++ b/kernel/bpf/bpf_cgrp_storage.c
@@ -46,8 +46,6 @@ static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner)
 void bpf_cgrp_storage_free(struct cgroup *cgroup)
 {
        struct bpf_local_storage *local_storage;
-       bool free_cgroup_storage = false;
-       unsigned long flags;
 
        rcu_read_lock();
        local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
@@ -57,14 +55,9 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
        }
 
        bpf_cgrp_storage_lock();
-       raw_spin_lock_irqsave(&local_storage->lock, flags);
-       free_cgroup_storage = bpf_local_storage_unlink_nolock(local_storage);
-       raw_spin_unlock_irqrestore(&local_storage->lock, flags);
+       bpf_local_storage_destroy(local_storage);
        bpf_cgrp_storage_unlock();
        rcu_read_unlock();
-
-       if (free_cgroup_storage)
-               kfree_rcu(local_storage, rcu);
 }
 
 static struct bpf_local_storage_data *

diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c
index 43e2619..2d25bcf 100644 (file)
--- a/kernel/bpf/bpf_inode_storage.c
+++ b/kernel/bpf/bpf_inode_storage.c
@@ -57,7 +57,6 @@ static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
 void bpf_inode_storage_free(struct inode *inode)
 {
        struct bpf_local_storage *local_storage;
-       bool free_inode_storage = false;
        struct bpf_storage_blob *bsb;
 
        bsb = bpf_inode(inode);
@@ -72,13 +71,8 @@ void bpf_inode_storage_free(struct inode *inode)
                return;
        }
 
-       raw_spin_lock_bh(&local_storage->lock);
-       free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
-       raw_spin_unlock_bh(&local_storage->lock);
+       bpf_local_storage_destroy(local_storage);
        rcu_read_unlock();
-
-       if (free_inode_storage)
-               kfree_rcu(local_storage, rcu);
 }
 
 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 1904a42..e19f9f5 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -652,11 +652,12 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
        return 0;
 }
 
-bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage)
+void bpf_local_storage_destroy(struct bpf_local_storage *local_storage)
 {
        struct bpf_local_storage_elem *selem;
        bool free_storage = false;
        struct hlist_node *n;
+       unsigned long flags;
 
        /* Neither the bpf_prog nor the bpf_map's syscall
         * could be modifying the local_storage->list now.
@@ -667,6 +668,7 @@ bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage)
         * when unlinking elem from the local_storage->list and
         * the map's bucket->list.
         */
+       raw_spin_lock_irqsave(&local_storage->lock, flags);
        hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
                /* Always unlink from map before unlinking from
                 * local_storage.
@@ -681,8 +683,10 @@ bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage)
                free_storage = bpf_selem_unlink_storage_nolock(
                        local_storage, selem, false, false);
        }
+       raw_spin_unlock_irqrestore(&local_storage->lock, flags);
 
-       return free_storage;
+       if (free_storage)
+               kfree_rcu(local_storage, rcu);
 }
 
 u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map)

diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index 20f9422..4dcef28 100644 (file)
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -72,8 +72,6 @@ task_storage_lookup(struct task_struct *task, struct bpf_map *map,
 void bpf_task_storage_free(struct task_struct *task)
 {
        struct bpf_local_storage *local_storage;
-       bool free_task_storage = false;
-       unsigned long flags;
 
        rcu_read_lock();
 
@@ -84,14 +82,9 @@ void bpf_task_storage_free(struct task_struct *task)
        }
 
        bpf_task_storage_lock();
-       raw_spin_lock_irqsave(&local_storage->lock, flags);
-       free_task_storage = bpf_local_storage_unlink_nolock(local_storage);
-       raw_spin_unlock_irqrestore(&local_storage->lock, flags);
+       bpf_local_storage_destroy(local_storage);
        bpf_task_storage_unlock();
        rcu_read_unlock();
-
-       if (free_task_storage)
-               kfree_rcu(local_storage, rcu);
 }
 
 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 7a36353..8f56438 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -49,7 +49,6 @@ static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
 void bpf_sk_storage_free(struct sock *sk)
 {
        struct bpf_local_storage *sk_storage;
-       bool free_sk_storage = false;
 
        rcu_read_lock();
        sk_storage = rcu_dereference(sk->sk_bpf_storage);
@@ -58,13 +57,8 @@ void bpf_sk_storage_free(struct sock *sk)
                return;
        }
 
-       raw_spin_lock_bh(&sk_storage->lock);
-       free_sk_storage = bpf_local_storage_unlink_nolock(sk_storage);
-       raw_spin_unlock_bh(&sk_storage->lock);
+       bpf_local_storage_destroy(sk_storage);
        rcu_read_unlock();
-
-       if (free_sk_storage)
-               kfree_rcu(sk_storage, rcu);
 }
 
 static void bpf_sk_storage_map_free(struct bpf_map *map)