bpf: Re-enable unit_size checking for global per-cpu allocator

[platform/kernel/linux-starfive.git] / kernel / bpf / memalloc.c

diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index cf19415..9657d59 100644 (file)
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -491,21 +491,17 @@ static int check_obj_size(struct bpf_mem_cache *c, unsigned int idx)
        struct llist_node *first;
        unsigned int obj_size;
 
-       /* For per-cpu allocator, the size of free objects in free list doesn't
-        * match with unit_size and now there is no way to get the size of
-        * per-cpu pointer saved in free object, so just skip the checking.
-        */
-       if (c->percpu_size)
-               return 0;
-
        first = c->free_llist.first;
        if (!first)
                return 0;
 
-       obj_size = ksize(first);
+       if (c->percpu_size)
+               obj_size = pcpu_alloc_size(((void **)first)[1]);
+       else
+               obj_size = ksize(first);
        if (obj_size != c->unit_size) {
-               WARN_ONCE(1, "bpf_mem_cache[%u]: unexpected object size %u, expect %u\n",
-                         idx, obj_size, c->unit_size);
+               WARN_ONCE(1, "bpf_mem_cache[%u]: percpu %d, unexpected object size %u, expect %u\n",
+                         idx, c->percpu_size, obj_size, c->unit_size);
                return -EINVAL;
        }
        return 0;
@@ -958,6 +954,8 @@ void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags)
                memcg = get_memcg(c);
                old_memcg = set_active_memcg(memcg);
                ret = __alloc(c, NUMA_NO_NODE, GFP_KERNEL | __GFP_NOWARN | __GFP_ACCOUNT);
+               if (ret)
+                       *(struct bpf_mem_cache **)ret = c;
                set_active_memcg(old_memcg);
                mem_cgroup_put(memcg);
        }
@@ -965,37 +963,37 @@ void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags)
        return !ret ? NULL : ret + LLIST_NODE_SZ;
 }
 
-/* Most of the logic is taken from setup_kmalloc_cache_index_table() */
+/* The alignment of dynamic per-cpu area is 8, so c->unit_size and the
+ * actual size of dynamic per-cpu area will always be matched and there is
+ * no need to adjust size_index for per-cpu allocation. However for the
+ * simplicity of the implementation, use an unified size_index for both
+ * kmalloc and per-cpu allocation.
+ */
 static __init int bpf_mem_cache_adjust_size(void)
 {
-       unsigned int size, index;
+       unsigned int size;
 
-       /* Normally KMALLOC_MIN_SIZE is 8-bytes, but it can be
-        * up-to 256-bytes.
+       /* Adjusting the indexes in size_index() according to the object_size
+        * of underlying slab cache, so bpf_mem_alloc() will select a
+        * bpf_mem_cache with unit_size equal to the object_size of
+        * the underlying slab cache.
+        *
+        * The maximal value of KMALLOC_MIN_SIZE and __kmalloc_minalign() is
+        * 256-bytes, so only do adjustment for [8-bytes, 192-bytes].
         */
-       size = KMALLOC_MIN_SIZE;
-       if (size <= 192)
-               index = size_index[(size - 1) / 8];
-       else
-               index = fls(size - 1) - 1;
-       for (size = 8; size < KMALLOC_MIN_SIZE && size <= 192; size += 8)
-               size_index[(size - 1) / 8] = index;
+       for (size = 192; size >= 8; size -= 8) {
+               unsigned int kmalloc_size, index;
 
-       /* The minimal alignment is 64-bytes, so disable 96-bytes cache and
-        * use 128-bytes cache instead.
-        */
-       if (KMALLOC_MIN_SIZE >= 64) {
-               index = size_index[(128 - 1) / 8];
-               for (size = 64 + 8; size <= 96; size += 8)
-                       size_index[(size - 1) / 8] = index;
-       }
+               kmalloc_size = kmalloc_size_roundup(size);
+               if (kmalloc_size == size)
+                       continue;
 
-       /* The minimal alignment is 128-bytes, so disable 192-bytes cache and
-        * use 256-bytes cache instead.
-        */
-       if (KMALLOC_MIN_SIZE >= 128) {
-               index = fls(256 - 1) - 1;
-               for (size = 128 + 8; size <= 192; size += 8)
+               if (kmalloc_size <= 192)
+                       index = size_index[(kmalloc_size - 1) / 8];
+               else
+                       index = fls(kmalloc_size - 1) - 1;
+               /* Only overwrite if necessary */
+               if (size_index[(size - 1) / 8] != index)
                        size_index[(size - 1) / 8] = index;
        }