bpf: Repurpose use_trace_rcu to reuse_now in bpf_local_storage

This patch re-purpose the use_trace_rcu to mean
if the freed memory can be reused immediately or not.
The use_trace_rcu is renamed to reuse_now. Other than
the boolean test is reversed, it should be a no-op.

The following explains the reason for the rename and how it will
be used in a later patch.

In a later patch, bpf_mem_cache_alloc/free will be used
in the bpf_local_storage. The bpf mem allocator will reuse
the freed memory immediately. Some of the free paths in
bpf_local_storage does not support memory to be reused immediately.
These paths are the "delete" elem cases from the bpf_*_storage_delete()
helper and the map_delete_elem() syscall. Note that "delete" elem
before the owner's (sk/task/cgrp/inode) lifetime ended is not
the common usage for the local storage.

The common free path, bpf_local_storage_destroy(), can reuse the
memory immediately. This common path means the storage stays with
its owner until the owner is destroyed.

The above mentioned "delete" elem paths that cannot
reuse immediately always has the 'use_trace_rcu ==  true'.
The cases that is safe for immediate reuse always have
'use_trace_rcu == false'. Instead of adding another arg
in a later patch, this patch re-purpose this arg
to reuse_now and have the test logic reversed.

In a later patch, 'reuse_now == true' will free to the
bpf_mem_cache_free() where the memory can be reused
immediately. 'reuse_now == false' will go through the
call_rcu_tasks_trace().

Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230308065936.1550103-7-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h
index 613b180..18a31ad 100644 (file)
--- a/include/linux/bpf_local_storage.h
+++ b/include/linux/bpf_local_storage.h
@@ -143,7 +143,7 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
                                   struct bpf_local_storage_elem *selem);
 
-void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
+void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
 
 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
                        struct bpf_local_storage_elem *selem);

diff --git a/kernel/bpf/bpf_cgrp_storage.c b/kernel/bpf/bpf_cgrp_storage.c
index 492594d..c975cac 100644 (file)
--- a/kernel/bpf/bpf_cgrp_storage.c
+++ b/kernel/bpf/bpf_cgrp_storage.c
@@ -121,7 +121,7 @@ static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map)
        if (!sdata)
                return -ENOENT;
 
-       bpf_selem_unlink(SELEM(sdata), true);
+       bpf_selem_unlink(SELEM(sdata), false);
        return 0;
 }
 

diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c
index 2d25bcf..ad2ab01 100644 (file)
--- a/kernel/bpf/bpf_inode_storage.c
+++ b/kernel/bpf/bpf_inode_storage.c
@@ -122,7 +122,7 @@ static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
        if (!sdata)
                return -ENOENT;
 
-       bpf_selem_unlink(SELEM(sdata), true);
+       bpf_selem_unlink(SELEM(sdata), false);
 
        return 0;
 }

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 5585dbf..70c34a9 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -147,7 +147,7 @@ static void bpf_selem_free_trace_rcu(struct rcu_head *rcu)
  */
 static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
                                            struct bpf_local_storage_elem *selem,
-                                           bool uncharge_mem, bool use_trace_rcu)
+                                           bool uncharge_mem, bool reuse_now)
 {
        struct bpf_local_storage_map *smap;
        bool free_local_storage;
@@ -201,7 +201,7 @@ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_stor
         * any special fields.
         */
        rec = smap->map.record;
-       if (use_trace_rcu) {
+       if (!reuse_now) {
                if (!IS_ERR_OR_NULL(rec))
                        call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_fields_trace_rcu);
                else
@@ -220,7 +220,7 @@ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_stor
 }
 
 static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,
-                                    bool use_trace_rcu)
+                                    bool reuse_now)
 {
        struct bpf_local_storage *local_storage;
        bool free_local_storage = false;
@@ -235,11 +235,11 @@ static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,
        raw_spin_lock_irqsave(&local_storage->lock, flags);
        if (likely(selem_linked_to_storage(selem)))
                free_local_storage = bpf_selem_unlink_storage_nolock(
-                       local_storage, selem, true, use_trace_rcu);
+                       local_storage, selem, true, reuse_now);
        raw_spin_unlock_irqrestore(&local_storage->lock, flags);
 
        if (free_local_storage) {
-               if (use_trace_rcu)
+               if (!reuse_now)
                        call_rcu_tasks_trace(&local_storage->rcu,
                                     bpf_local_storage_free_rcu);
                else
@@ -284,14 +284,14 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap,
        raw_spin_unlock_irqrestore(&b->lock, flags);
 }
 
-void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu)
+void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now)
 {
        /* Always unlink from map before unlinking from local_storage
         * because selem will be freed after successfully unlinked from
         * the local_storage.
         */
        bpf_selem_unlink_map(selem);
-       bpf_selem_unlink_storage(selem, use_trace_rcu);
+       bpf_selem_unlink_storage(selem, reuse_now);
 }
 
 /* If cacheit_lockit is false, this lookup function is lockless */
@@ -538,7 +538,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
        if (old_sdata) {
                bpf_selem_unlink_map(SELEM(old_sdata));
                bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
-                                               false, true);
+                                               false, false);
        }
 
 unlock:
@@ -651,7 +651,7 @@ void bpf_local_storage_destroy(struct bpf_local_storage *local_storage)
                 * of the loop will set the free_cgroup_storage to true.
                 */
                free_storage = bpf_selem_unlink_storage_nolock(
-                       local_storage, selem, false, false);
+                       local_storage, selem, false, true);
        }
        raw_spin_unlock_irqrestore(&local_storage->lock, flags);
 
@@ -745,7 +745,7 @@ void bpf_local_storage_map_free(struct bpf_map *map,
                                migrate_disable();
                                this_cpu_inc(*busy_counter);
                        }
-                       bpf_selem_unlink(selem, false);
+                       bpf_selem_unlink(selem, true);
                        if (busy_counter) {
                                this_cpu_dec(*busy_counter);
                                migrate_enable();
@@ -783,8 +783,8 @@ void bpf_local_storage_map_free(struct bpf_map *map,
                /* We cannot skip rcu_barrier() when rcu_trace_implies_rcu_gp()
                 * is true, because while call_rcu invocation is skipped in that
                 * case in bpf_selem_free_fields_trace_rcu (and all local
-                * storage maps pass use_trace_rcu = true), there can be
-                * call_rcu callbacks based on use_trace_rcu = false in the
+                * storage maps pass reuse_now = false), there can be
+                * call_rcu callbacks based on reuse_now = true in the
                 * while ((selem = ...)) loop above or when owner's free path
                 * calls bpf_local_storage_unlink_nolock.
                 */

diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index 4dcef28..c88cc04 100644 (file)
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -168,7 +168,7 @@ static int task_storage_delete(struct task_struct *task, struct bpf_map *map,
        if (!nobusy)
                return -EBUSY;
 
-       bpf_selem_unlink(SELEM(sdata), true);
+       bpf_selem_unlink(SELEM(sdata), false);
 
        return 0;
 }

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 8f56438..a5f185b 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -40,7 +40,7 @@ static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
        if (!sdata)
                return -ENOENT;
 
-       bpf_selem_unlink(SELEM(sdata), true);
+       bpf_selem_unlink(SELEM(sdata), false);
 
        return 0;
 }