bpf: Use pcpu_alloc_size() in bpf_mem_free{_rcu}()

[ Upstream commit 3f2189e4f77b7a3e979d143dc4ff586488c7e8a5 ]

For bpf_global_percpu_ma, the pointer passed to bpf_mem_free_rcu() is
allocated by kmalloc() and its size is fixed (16-bytes on x86-64). So
no matter which cache allocates the dynamic per-cpu area, on x86-64
cache[2] will always be used to free the per-cpu area.

Fix the unbalance by checking whether the bpf memory allocator is
per-cpu or not and use pcpu_alloc_size() instead of ksize() to
find the correct cache for per-cpu free.

Signed-off-by: Hou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231020133202.4043247-5-houtao@huaweicloud.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Stable-dep-of: 7ac5c53e0073 ("bpf: Use c->unit_size to select target cache during free")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h
index d644bbb..bb1223b 100644 (file)
--- a/include/linux/bpf_mem_alloc.h
+++ b/include/linux/bpf_mem_alloc.h
@@ -11,6 +11,7 @@ struct bpf_mem_caches;
 struct bpf_mem_alloc {
        struct bpf_mem_caches __percpu *caches;
        struct bpf_mem_cache __percpu *cache;
+       bool percpu;
        struct work_struct work;
 };
 

diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index 9657d59..5f93baf 100644 (file)
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -522,6 +522,8 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
        struct bpf_mem_cache *c, __percpu *pc;
        struct obj_cgroup *objcg = NULL;
 
+       ma->percpu = percpu;
+
        if (size) {
                pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
                if (!pc)
@@ -866,6 +868,17 @@ void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)
        return !ret ? NULL : ret + LLIST_NODE_SZ;
 }
 
+static notrace int bpf_mem_free_idx(void *ptr, bool percpu)
+{
+       size_t size;
+
+       if (percpu)
+               size = pcpu_alloc_size(*((void **)ptr));
+       else
+               size = ksize(ptr - LLIST_NODE_SZ);
+       return bpf_mem_cache_idx(size);
+}
+
 void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
 {
        int idx;
@@ -873,7 +886,7 @@ void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
        if (!ptr)
                return;
 
-       idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
+       idx = bpf_mem_free_idx(ptr, ma->percpu);
        if (idx < 0)
                return;
 
@@ -887,7 +900,7 @@ void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
        if (!ptr)
                return;
 
-       idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
+       idx = bpf_mem_free_idx(ptr, ma->percpu);
        if (idx < 0)
                return;