bpf: Defer the free of inner map when necessary

[ Upstream commit 876673364161da50eed6b472d746ef88242b2368 ]

When updating or deleting an inner map in map array or map htab, the map
may still be accessed by non-sleepable program or sleepable program.
However bpf_map_fd_put_ptr() decreases the ref-counter of the inner map
directly through bpf_map_put(), if the ref-counter is the last one
(which is true for most cases), the inner map will be freed by
ops->map_free() in a kworker. But for now, most .map_free() callbacks
don't use synchronize_rcu() or its variants to wait for the elapse of a
RCU grace period, so after the invocation of ops->map_free completes,
the bpf program which is accessing the inner map may incur
use-after-free problem.

Fix the free of inner map by invoking bpf_map_free_deferred() after both
one RCU grace period and one tasks trace RCU grace period if the inner
map has been removed from the outer map before. The deferment is
accomplished by using call_rcu() or call_rcu_tasks_trace() when
releasing the last ref-counter of bpf map. The newly-added rcu_head
field in bpf_map shares the same storage space with work field to
reduce the size of bpf_map.

Fixes: bba1dc0b55ac ("bpf: Remove redundant synchronize_rcu.")
Fixes: 638e4b825d52 ("bpf: Allows per-cpu maps and map-in-map in sleepable programs")
Signed-off-by: Hou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231204140425.1480317-5-houtao@huaweicloud.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 4bed78e47347c9e7dff6fa1c3bc2b613177efa73..9b08d792fa95a9bb49647ce8506a7e6d7c7f21ea 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -275,7 +275,11 @@ struct bpf_map {
         */
        atomic64_t refcnt ____cacheline_aligned;
        atomic64_t usercnt;
-       struct work_struct work;
+       /* rcu is used before freeing and work is only used during freeing */
+       union {
+               struct work_struct work;
+               struct rcu_head rcu;
+       };
        struct mutex freeze_mutex;
        atomic64_t writecnt;
        /* 'Ownership' of program-containing map is claimed by the first program
@@ -291,6 +295,7 @@ struct bpf_map {
        } owner;
        bool bypass_spec_v1;
        bool frozen; /* write-once; write-protected by freeze_mutex */
+       bool free_after_mult_rcu_gp;
        s64 __percpu *elem_count;
 };
 

diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index 2dfeb5835e16564f8bdc0b8789ce11ac7e77896b..3248ff5d816172b945918ef03279c83f87dc709a 100644 (file)
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -129,10 +129,15 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map,
 
 void bpf_map_fd_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
 {
-       /* ptr->ops->map_free() has to go through one
-        * rcu grace period by itself.
+       struct bpf_map *inner_map = ptr;
+
+       /* The inner map may still be used by both non-sleepable and sleepable
+        * bpf program, so free it after one RCU grace period and one tasks
+        * trace RCU grace period.
         */
-       bpf_map_put(ptr);
+       if (need_defer)
+               WRITE_ONCE(inner_map->free_after_mult_rcu_gp, true);
+       bpf_map_put(inner_map);
 }
 
 u32 bpf_map_fd_sys_lookup_elem(void *ptr)

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index d77b2f8b93641b4445645ec2d9397849fe6a5af3..1625b9d5202c115a80fe304c122a0f0a8add59d8 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -717,6 +717,28 @@ static void bpf_map_put_uref(struct bpf_map *map)
        }
 }
 
+static void bpf_map_free_in_work(struct bpf_map *map)
+{
+       INIT_WORK(&map->work, bpf_map_free_deferred);
+       /* Avoid spawning kworkers, since they all might contend
+        * for the same mutex like slab_mutex.
+        */
+       queue_work(system_unbound_wq, &map->work);
+}
+
+static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
+{
+       bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
+}
+
+static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
+{
+       if (rcu_trace_implies_rcu_gp())
+               bpf_map_free_rcu_gp(rcu);
+       else
+               call_rcu(rcu, bpf_map_free_rcu_gp);
+}
+
 /* decrement map refcnt and schedule it for freeing via workqueue
  * (underlying map implementation ops->map_free() might sleep)
  */
@@ -726,11 +748,11 @@ void bpf_map_put(struct bpf_map *map)
                /* bpf_map_free_id() must be called first */
                bpf_map_free_id(map);
                btf_put(map->btf);
-               INIT_WORK(&map->work, bpf_map_free_deferred);
-               /* Avoid spawning kworkers, since they all might contend
-                * for the same mutex like slab_mutex.
-                */
-               queue_work(system_unbound_wq, &map->work);
+
+               if (READ_ONCE(map->free_after_mult_rcu_gp))
+                       call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
+               else
+                       bpf_map_free_in_work(map);
        }
 }
 EXPORT_SYMBOL_GPL(bpf_map_put);