bpf: Convert percpu hash map to per-cpu bpf_mem_alloc.

Convert dynamic allocations in percpu hash map from alloc_percpu() to
bpf_mem_cache_alloc() from per-cpu bpf_mem_alloc. Since bpf_mem_alloc frees
objects after RCU gp the call_rcu() is removed. pcpu_init_value() now needs to
zero-fill per-cpu allocations, since dynamically allocated map elements are now
similar to full prealloc, since alloc_percpu() is not called inline and the
elements are reused in the freelist.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20220902211058.60789-12-alexei.starovoitov@gmail.com

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 70b02ff..a77b9c4 100644 (file)
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -94,6 +94,7 @@ struct bucket {
 struct bpf_htab {
        struct bpf_map map;
        struct bpf_mem_alloc ma;
+       struct bpf_mem_alloc pcpu_ma;
        struct bucket *buckets;
        void *elems;
        union {
@@ -121,14 +122,14 @@ struct htab_elem {
                struct {
                        void *padding;
                        union {
-                               struct bpf_htab *htab;
                                struct pcpu_freelist_node fnode;
                                struct htab_elem *batch_flink;
                        };
                };
        };
        union {
-               struct rcu_head rcu;
+               /* pointer to per-cpu pointer */
+               void *ptr_to_pptr;
                struct bpf_lru_node lru_node;
        };
        u32 hash;
@@ -448,8 +449,6 @@ static int htab_map_alloc_check(union bpf_attr *attr)
        bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED);
        int numa_node = bpf_map_attr_numa_node(attr);
 
-       BUILD_BUG_ON(offsetof(struct htab_elem, htab) !=
-                    offsetof(struct htab_elem, hash_node.pprev));
        BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
                     offsetof(struct htab_elem, hash_node.pprev));
 
@@ -610,6 +609,12 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
                err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false);
                if (err)
                        goto free_map_locked;
+               if (percpu) {
+                       err = bpf_mem_alloc_init(&htab->pcpu_ma,
+                                                round_up(htab->map.value_size, 8), true);
+                       if (err)
+                               goto free_map_locked;
+               }
        }
 
        return &htab->map;
@@ -620,6 +625,7 @@ free_map_locked:
        for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
                free_percpu(htab->map_locked[i]);
        bpf_map_area_free(htab->buckets);
+       bpf_mem_alloc_destroy(&htab->pcpu_ma);
        bpf_mem_alloc_destroy(&htab->ma);
 free_htab:
        lockdep_unregister_key(&htab->lockdep_key);
@@ -895,19 +901,11 @@ find_first_elem:
 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
 {
        if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
-               free_percpu(htab_elem_get_ptr(l, htab->map.key_size));
+               bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
        check_and_free_fields(htab, l);
        bpf_mem_cache_free(&htab->ma, l);
 }
 
-static void htab_elem_free_rcu(struct rcu_head *head)
-{
-       struct htab_elem *l = container_of(head, struct htab_elem, rcu);
-       struct bpf_htab *htab = l->htab;
-
-       htab_elem_free(htab, l);
-}
-
 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
 {
        struct bpf_map *map = &htab->map;
@@ -953,12 +951,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
                __pcpu_freelist_push(&htab->freelist, &l->fnode);
        } else {
                dec_elem_count(htab);
-               if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) {
-                       l->htab = htab;
-                       call_rcu(&l->rcu, htab_elem_free_rcu);
-               } else {
-                       htab_elem_free(htab, l);
-               }
+               htab_elem_free(htab, l);
        }
 }
 
@@ -983,13 +976,12 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
 static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
                            void *value, bool onallcpus)
 {
-       /* When using prealloc and not setting the initial value on all cpus,
-        * zero-fill element values for other cpus (just as what happens when
-        * not using prealloc). Otherwise, bpf program has no way to ensure
+       /* When not setting the initial value on all cpus, zero-fill element
+        * values for other cpus. Otherwise, bpf program has no way to ensure
         * known initial values for cpus other than current one
         * (onallcpus=false always when coming from bpf prog).
         */
-       if (htab_is_prealloc(htab) && !onallcpus) {
+       if (!onallcpus) {
                u32 size = round_up(htab->map.value_size, 8);
                int current_cpu = raw_smp_processor_id();
                int cpu;
@@ -1060,18 +1052,18 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 
        memcpy(l_new->key, key, key_size);
        if (percpu) {
-               size = round_up(size, 8);
                if (prealloc) {
                        pptr = htab_elem_get_ptr(l_new, key_size);
                } else {
                        /* alloc_percpu zero-fills */
-                       pptr = bpf_map_alloc_percpu(&htab->map, size, 8,
-                                                   GFP_NOWAIT | __GFP_NOWARN);
+                       pptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
                        if (!pptr) {
                                bpf_mem_cache_free(&htab->ma, l_new);
                                l_new = ERR_PTR(-ENOMEM);
                                goto dec_count;
                        }
+                       l_new->ptr_to_pptr = pptr;
+                       pptr = *(void **)pptr;
                }
 
                pcpu_init_value(htab, pptr, value, onallcpus);
@@ -1568,6 +1560,7 @@ static void htab_map_free(struct bpf_map *map)
        bpf_map_free_kptr_off_tab(map);
        free_percpu(htab->extra_elems);
        bpf_map_area_free(htab->buckets);
+       bpf_mem_alloc_destroy(&htab->pcpu_ma);
        bpf_mem_alloc_destroy(&htab->ma);
        if (htab->use_percpu_counter)
                percpu_counter_destroy(&htab->pcount);