bpf, x86: Generate trampolines from bpf_tramp_links

Replace struct bpf_tramp_progs with struct bpf_tramp_links to collect
struct bpf_tramp_link(s) for a trampoline.  struct bpf_tramp_link
extends bpf_link to act as a linked list node.

arch_prepare_bpf_trampoline() accepts a struct bpf_tramp_links to
collects all bpf_tramp_link(s) that a trampoline should call.

Change BPF trampoline and bpf_struct_ops to pass bpf_tramp_links
instead of bpf_tramp_progs.

Signed-off-by: Kui-Feng Lee <kuifeng@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20220510205923.3206889-2-kuifeng@fb.com

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 16b6efa..38eb431 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1762,10 +1762,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 }
 
 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
-                          struct bpf_prog *p, int stack_size, bool save_ret)
+                          struct bpf_tramp_link *l, int stack_size,
+                          bool save_ret)
 {
        u8 *prog = *pprog;
        u8 *jmp_insn;
+       struct bpf_prog *p = l->link.prog;
 
        /* arg1: mov rdi, progs[i] */
        emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
@@ -1850,14 +1852,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
 }
 
 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
-                     struct bpf_tramp_progs *tp, int stack_size,
+                     struct bpf_tramp_links *tl, int stack_size,
                      bool save_ret)
 {
        int i;
        u8 *prog = *pprog;
 
-       for (i = 0; i < tp->nr_progs; i++) {
-               if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size,
+       for (i = 0; i < tl->nr_links; i++) {
+               if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,
                                    save_ret))
                        return -EINVAL;
        }
@@ -1866,7 +1868,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
 }
 
 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
-                             struct bpf_tramp_progs *tp, int stack_size,
+                             struct bpf_tramp_links *tl, int stack_size,
                              u8 **branches)
 {
        u8 *prog = *pprog;
@@ -1877,8 +1879,8 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
         */
        emit_mov_imm32(&prog, false, BPF_REG_0, 0);
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
-       for (i = 0; i < tp->nr_progs; i++) {
-               if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))
+       for (i = 0; i < tl->nr_links; i++) {
+               if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, true))
                        return -EINVAL;
 
                /* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1980,14 +1982,14 @@ static bool is_valid_bpf_tramp_flags(unsigned int flags)
  */
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
                                const struct btf_func_model *m, u32 flags,
-                               struct bpf_tramp_progs *tprogs,
+                               struct bpf_tramp_links *tlinks,
                                void *orig_call)
 {
        int ret, i, nr_args = m->nr_args;
        int regs_off, ip_off, args_off, stack_size = nr_args * 8;
-       struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
-       struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
-       struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
+       struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
+       struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
+       struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
        u8 **branches = NULL;
        u8 *prog;
        bool save_ret;
@@ -2078,13 +2080,13 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                }
        }
 
-       if (fentry->nr_progs)
+       if (fentry->nr_links)
                if (invoke_bpf(m, &prog, fentry, regs_off,
                               flags & BPF_TRAMP_F_RET_FENTRY_RET))
                        return -EINVAL;
 
-       if (fmod_ret->nr_progs) {
-               branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),
+       if (fmod_ret->nr_links) {
+               branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),
                                   GFP_KERNEL);
                if (!branches)
                        return -ENOMEM;
@@ -2111,7 +2113,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                prog += X86_PATCH_SIZE;
        }
 
-       if (fmod_ret->nr_progs) {
+       if (fmod_ret->nr_links) {
                /* From Intel 64 and IA-32 Architectures Optimization
                 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
                 * Coding Rule 11: All branch targets should be 16-byte
@@ -2121,12 +2123,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                /* Update the branches saved in invoke_bpf_mod_ret with the
                 * aligned address of do_fexit.
                 */
-               for (i = 0; i < fmod_ret->nr_progs; i++)
+               for (i = 0; i < fmod_ret->nr_links; i++)
                        emit_cond_near_jump(&branches[i], prog, branches[i],
                                            X86_JNE);
        }
 
-       if (fexit->nr_progs)
+       if (fexit->nr_links)
                if (invoke_bpf(m, &prog, fexit, regs_off, false)) {
                        ret = -EINVAL;
                        goto cleanup;

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 551b719..75e0110 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -723,11 +723,11 @@ struct btf_func_model {
 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
  * bytes on x86.  Pick a number to fit into BPF_IMAGE_SIZE / 2
  */
-#define BPF_MAX_TRAMP_PROGS 38
+#define BPF_MAX_TRAMP_LINKS 38
 
-struct bpf_tramp_progs {
-       struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
-       int nr_progs;
+struct bpf_tramp_links {
+       struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
+       int nr_links;
 };
 
 /* Different use cases for BPF trampoline:
@@ -753,7 +753,7 @@ struct bpf_tramp_progs {
 struct bpf_tramp_image;
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
                                const struct btf_func_model *m, u32 flags,
-                               struct bpf_tramp_progs *tprogs,
+                               struct bpf_tramp_links *tlinks,
                                void *orig_call);
 /* these two functions are called from generated trampoline */
 u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
@@ -852,9 +852,10 @@ static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
 {
        return bpf_func(ctx, insnsi);
 }
+
 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
                                          struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -905,12 +906,12 @@ int bpf_jit_charge_modmem(u32 size);
 void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 #else
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
                                           struct bpf_trampoline *tr)
 {
        return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
                                             struct bpf_trampoline *tr)
 {
        return -ENOTSUPP;
@@ -1009,7 +1010,6 @@ struct bpf_prog_aux {
        bool tail_call_reachable;
        bool xdp_has_frags;
        bool use_bpf_prog_pack;
-       struct hlist_node tramp_hlist;
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;
        /* function name for valid attach_btf_id */
@@ -1096,6 +1096,18 @@ struct bpf_link_ops {
                              struct bpf_link_info *info);
 };
 
+struct bpf_tramp_link {
+       struct bpf_link link;
+       struct hlist_node tramp_hlist;
+};
+
+struct bpf_tracing_link {
+       struct bpf_tramp_link link;
+       enum bpf_attach_type attach_type;
+       struct bpf_trampoline *trampoline;
+       struct bpf_prog *tgt_prog;
+};
+
 struct bpf_link_primer {
        struct bpf_link *link;
        struct file *file;
@@ -1133,8 +1145,8 @@ bool bpf_struct_ops_get(const void *kdata);
 void bpf_struct_ops_put(const void *kdata);
 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
                                       void *value);
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-                                     struct bpf_prog *prog,
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
+                                     struct bpf_tramp_link *link,
                                      const struct btf_func_model *model,
                                      void *image, void *image_end);
 static inline bool bpf_try_module_get(const void *data, struct module *owner)

diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index 3e24ad0..2b9112b 100644 (file)
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -141,3 +141,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_XDP, xdp)
 BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf)
 #endif
 BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi)
+BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 95a3d1f..3d032ea 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1013,6 +1013,7 @@ enum bpf_link_type {
        BPF_LINK_TYPE_XDP = 6,
        BPF_LINK_TYPE_PERF_EVENT = 7,
        BPF_LINK_TYPE_KPROBE_MULTI = 8,
+       BPF_LINK_TYPE_STRUCT_OPS = 9,
 
        MAX_BPF_LINK_TYPE,
 };

diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index 3a0103a..d9a3c92 100644 (file)
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c
@@ -33,15 +33,15 @@ struct bpf_struct_ops_map {
        const struct bpf_struct_ops *st_ops;
        /* protect map_update */
        struct mutex lock;
-       /* progs has all the bpf_prog that is populated
+       /* link has all the bpf_links that is populated
         * to the func ptr of the kernel's struct
         * (in kvalue.data).
         */
-       struct bpf_prog **progs;
+       struct bpf_link **links;
        /* image is a page that has all the trampolines
         * that stores the func args before calling the bpf_prog.
         * A PAGE_SIZE "image" is enough to store all trampoline for
-        * "progs[]".
+        * "links[]".
         */
        void *image;
        /* uvalue->data stores the kernel struct
@@ -283,9 +283,9 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
        u32 i;
 
        for (i = 0; i < btf_type_vlen(t); i++) {
-               if (st_map->progs[i]) {
-                       bpf_prog_put(st_map->progs[i]);
-                       st_map->progs[i] = NULL;
+               if (st_map->links[i]) {
+                       bpf_link_put(st_map->links[i]);
+                       st_map->links[i] = NULL;
                }
        }
 }
@@ -316,18 +316,34 @@ static int check_zero_holes(const struct btf_type *t, void *data)
        return 0;
 }
 
-int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_progs *tprogs,
-                                     struct bpf_prog *prog,
+static void bpf_struct_ops_link_release(struct bpf_link *link)
+{
+}
+
+static void bpf_struct_ops_link_dealloc(struct bpf_link *link)
+{
+       struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link);
+
+       kfree(tlink);
+}
+
+const struct bpf_link_ops bpf_struct_ops_link_lops = {
+       .release = bpf_struct_ops_link_release,
+       .dealloc = bpf_struct_ops_link_dealloc,
+};
+
+int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
+                                     struct bpf_tramp_link *link,
                                      const struct btf_func_model *model,
                                      void *image, void *image_end)
 {
        u32 flags;
 
-       tprogs[BPF_TRAMP_FENTRY].progs[0] = prog;
-       tprogs[BPF_TRAMP_FENTRY].nr_progs = 1;
+       tlinks[BPF_TRAMP_FENTRY].links[0] = link;
+       tlinks[BPF_TRAMP_FENTRY].nr_links = 1;
        flags = model->ret_size > 0 ? BPF_TRAMP_F_RET_FENTRY_RET : 0;
        return arch_prepare_bpf_trampoline(NULL, image, image_end,
-                                          model, flags, tprogs, NULL);
+                                          model, flags, tlinks, NULL);
 }
 
 static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
@@ -338,7 +354,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
        struct bpf_struct_ops_value *uvalue, *kvalue;
        const struct btf_member *member;
        const struct btf_type *t = st_ops->type;
-       struct bpf_tramp_progs *tprogs = NULL;
+       struct bpf_tramp_links *tlinks = NULL;
        void *udata, *kdata;
        int prog_fd, err = 0;
        void *image, *image_end;
@@ -362,8 +378,8 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
        if (uvalue->state || refcount_read(&uvalue->refcnt))
                return -EINVAL;
 
-       tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
-       if (!tprogs)
+       tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+       if (!tlinks)
                return -ENOMEM;
 
        uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
@@ -386,6 +402,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
        for_each_member(i, t, member) {
                const struct btf_type *mtype, *ptype;
                struct bpf_prog *prog;
+               struct bpf_tramp_link *link;
                u32 moff;
 
                moff = __btf_member_bit_offset(t, member) / 8;
@@ -439,16 +456,26 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
                        err = PTR_ERR(prog);
                        goto reset_unlock;
                }
-               st_map->progs[i] = prog;
 
                if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
                    prog->aux->attach_btf_id != st_ops->type_id ||
                    prog->expected_attach_type != i) {
+                       bpf_prog_put(prog);
                        err = -EINVAL;
                        goto reset_unlock;
                }
 
-               err = bpf_struct_ops_prepare_trampoline(tprogs, prog,
+               link = kzalloc(sizeof(*link), GFP_USER);
+               if (!link) {
+                       bpf_prog_put(prog);
+                       err = -ENOMEM;
+                       goto reset_unlock;
+               }
+               bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
+                             &bpf_struct_ops_link_lops, prog);
+               st_map->links[i] = &link->link;
+
+               err = bpf_struct_ops_prepare_trampoline(tlinks, link,
                                                        &st_ops->func_models[i],
                                                        image, image_end);
                if (err < 0)
@@ -491,7 +518,7 @@ reset_unlock:
        memset(uvalue, 0, map->value_size);
        memset(kvalue, 0, map->value_size);
 unlock:
-       kfree(tprogs);
+       kfree(tlinks);
        mutex_unlock(&st_map->lock);
        return err;
 }
@@ -546,9 +573,9 @@ static void bpf_struct_ops_map_free(struct bpf_map *map)
 {
        struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
 
-       if (st_map->progs)
+       if (st_map->links)
                bpf_struct_ops_map_put_progs(st_map);
-       bpf_map_area_free(st_map->progs);
+       bpf_map_area_free(st_map->links);
        bpf_jit_free_exec(st_map->image);
        bpf_map_area_free(st_map->uvalue);
        bpf_map_area_free(st_map);
@@ -597,11 +624,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
        map = &st_map->map;
 
        st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
-       st_map->progs =
-               bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *),
+       st_map->links =
+               bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_links *),
                                   NUMA_NO_NODE);
        st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
-       if (!st_map->uvalue || !st_map->progs || !st_map->image) {
+       if (!st_map->uvalue || !st_map->links || !st_map->image) {
                bpf_struct_ops_map_free(map);
                return ERR_PTR(-ENOMEM);
        }

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 50164d3..2dc5827 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2864,19 +2864,12 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd)
 }
 EXPORT_SYMBOL(bpf_link_get_from_fd);
 
-struct bpf_tracing_link {
-       struct bpf_link link;
-       enum bpf_attach_type attach_type;
-       struct bpf_trampoline *trampoline;
-       struct bpf_prog *tgt_prog;
-};
-
 static void bpf_tracing_link_release(struct bpf_link *link)
 {
        struct bpf_tracing_link *tr_link =
-               container_of(link, struct bpf_tracing_link, link);
+               container_of(link, struct bpf_tracing_link, link.link);
 
-       WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
+       WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
                                                tr_link->trampoline));
 
        bpf_trampoline_put(tr_link->trampoline);
@@ -2889,7 +2882,7 @@ static void bpf_tracing_link_release(struct bpf_link *link)
 static void bpf_tracing_link_dealloc(struct bpf_link *link)
 {
        struct bpf_tracing_link *tr_link =
-               container_of(link, struct bpf_tracing_link, link);
+               container_of(link, struct bpf_tracing_link, link.link);
 
        kfree(tr_link);
 }
@@ -2898,7 +2891,7 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
                                         struct seq_file *seq)
 {
        struct bpf_tracing_link *tr_link =
-               container_of(link, struct bpf_tracing_link, link);
+               container_of(link, struct bpf_tracing_link, link.link);
 
        seq_printf(seq,
                   "attach_type:\t%d\n",
@@ -2909,7 +2902,7 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
                                           struct bpf_link_info *info)
 {
        struct bpf_tracing_link *tr_link =
-               container_of(link, struct bpf_tracing_link, link);
+               container_of(link, struct bpf_tracing_link, link.link);
 
        info->tracing.attach_type = tr_link->attach_type;
        bpf_trampoline_unpack_key(tr_link->trampoline->key,
@@ -2990,7 +2983,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
                err = -ENOMEM;
                goto out_put_prog;
        }
-       bpf_link_init(&link->link, BPF_LINK_TYPE_TRACING,
+       bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
                      &bpf_tracing_link_lops, prog);
        link->attach_type = prog->expected_attach_type;
 
@@ -3060,11 +3053,11 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
                tgt_prog = prog->aux->dst_prog;
        }
 
-       err = bpf_link_prime(&link->link, &link_primer);
+       err = bpf_link_prime(&link->link.link, &link_primer);
        if (err)
                goto out_unlock;
 
-       err = bpf_trampoline_link_prog(prog, tr);
+       err = bpf_trampoline_link_prog(&link->link, tr);
        if (err) {
                bpf_link_cleanup(&link_primer);
                link = NULL;

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index ada9775..d5e6bc5 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -168,30 +168,30 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
        return ret;
 }
 
-static struct bpf_tramp_progs *
+static struct bpf_tramp_links *
 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg)
 {
-       const struct bpf_prog_aux *aux;
-       struct bpf_tramp_progs *tprogs;
-       struct bpf_prog **progs;
+       struct bpf_tramp_link *link;
+       struct bpf_tramp_links *tlinks;
+       struct bpf_tramp_link **links;
        int kind;
 
        *total = 0;
-       tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
-       if (!tprogs)
+       tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+       if (!tlinks)
                return ERR_PTR(-ENOMEM);
 
        for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
-               tprogs[kind].nr_progs = tr->progs_cnt[kind];
+               tlinks[kind].nr_links = tr->progs_cnt[kind];
                *total += tr->progs_cnt[kind];
-               progs = tprogs[kind].progs;
+               links = tlinks[kind].links;
 
-               hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) {
-                       *ip_arg |= aux->prog->call_get_func_ip;
-                       *progs++ = aux->prog;
+               hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
+                       *ip_arg |= link->link.prog->call_get_func_ip;
+                       *links++ = link;
                }
        }
-       return tprogs;
+       return tlinks;
 }
 
 static void __bpf_tramp_image_put_deferred(struct work_struct *work)
@@ -330,14 +330,14 @@ out:
 static int bpf_trampoline_update(struct bpf_trampoline *tr)
 {
        struct bpf_tramp_image *im;
-       struct bpf_tramp_progs *tprogs;
+       struct bpf_tramp_links *tlinks;
        u32 flags = BPF_TRAMP_F_RESTORE_REGS;
        bool ip_arg = false;
        int err, total;
 
-       tprogs = bpf_trampoline_get_progs(tr, &total, &ip_arg);
-       if (IS_ERR(tprogs))
-               return PTR_ERR(tprogs);
+       tlinks = bpf_trampoline_get_progs(tr, &total, &ip_arg);
+       if (IS_ERR(tlinks))
+               return PTR_ERR(tlinks);
 
        if (total == 0) {
                err = unregister_fentry(tr, tr->cur_image->image);
@@ -353,15 +353,15 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
                goto out;
        }
 
-       if (tprogs[BPF_TRAMP_FEXIT].nr_progs ||
-           tprogs[BPF_TRAMP_MODIFY_RETURN].nr_progs)
+       if (tlinks[BPF_TRAMP_FEXIT].nr_links ||
+           tlinks[BPF_TRAMP_MODIFY_RETURN].nr_links)
                flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
 
        if (ip_arg)
                flags |= BPF_TRAMP_F_IP_ARG;
 
        err = arch_prepare_bpf_trampoline(im, im->image, im->image + PAGE_SIZE,
-                                         &tr->func.model, flags, tprogs,
+                                         &tr->func.model, flags, tlinks,
                                          tr->func.addr);
        if (err < 0)
                goto out;
@@ -381,7 +381,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr)
        tr->cur_image = im;
        tr->selector++;
 out:
-       kfree(tprogs);
+       kfree(tlinks);
        return err;
 }
 
@@ -407,13 +407,14 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
        }
 }
 
-int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
 {
        enum bpf_tramp_prog_type kind;
+       struct bpf_tramp_link *link_exiting;
        int err = 0;
        int cnt;
 
-       kind = bpf_attach_type_to_tramp(prog);
+       kind = bpf_attach_type_to_tramp(link->link.prog);
        mutex_lock(&tr->mutex);
        if (tr->extension_prog) {
                /* cannot attach fentry/fexit if extension prog is attached.
@@ -429,25 +430,33 @@ int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
                        err = -EBUSY;
                        goto out;
                }
-               tr->extension_prog = prog;
+               tr->extension_prog = link->link.prog;
                err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
-                                        prog->bpf_func);
+                                        link->link.prog->bpf_func);
                goto out;
        }
-       if (cnt >= BPF_MAX_TRAMP_PROGS) {
+       if (cnt >= BPF_MAX_TRAMP_LINKS) {
                err = -E2BIG;
                goto out;
        }
-       if (!hlist_unhashed(&prog->aux->tramp_hlist)) {
+       if (!hlist_unhashed(&link->tramp_hlist)) {
                /* prog already linked */
                err = -EBUSY;
                goto out;
        }
-       hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
+       hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) {
+               if (link_exiting->link.prog != link->link.prog)
+                       continue;
+               /* prog already linked */
+               err = -EBUSY;
+               goto out;
+       }
+
+       hlist_add_head(&link->tramp_hlist, &tr->progs_hlist[kind]);
        tr->progs_cnt[kind]++;
        err = bpf_trampoline_update(tr);
        if (err) {
-               hlist_del_init(&prog->aux->tramp_hlist);
+               hlist_del_init(&link->tramp_hlist);
                tr->progs_cnt[kind]--;
        }
 out:
@@ -456,12 +465,12 @@ out:
 }
 
 /* bpf_trampoline_unlink_prog() should never fail. */
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
 {
        enum bpf_tramp_prog_type kind;
        int err;
 
-       kind = bpf_attach_type_to_tramp(prog);
+       kind = bpf_attach_type_to_tramp(link->link.prog);
        mutex_lock(&tr->mutex);
        if (kind == BPF_TRAMP_REPLACE) {
                WARN_ON_ONCE(!tr->extension_prog);
@@ -470,7 +479,7 @@ int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
                tr->extension_prog = NULL;
                goto out;
        }
-       hlist_del_init(&prog->aux->tramp_hlist);
+       hlist_del_init(&link->tramp_hlist);
        tr->progs_cnt[kind]--;
        err = bpf_trampoline_update(tr);
 out:
@@ -635,7 +644,7 @@ void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr)
 int __weak
 arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
                            const struct btf_func_model *m, u32 flags,
-                           struct bpf_tramp_progs *tprogs,
+                           struct bpf_tramp_links *tlinks,
                            void *orig_call)
 {
        return -ENOTSUPP;

diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c
index d0e54e3..e78dadf 100644 (file)
--- a/net/bpf/bpf_dummy_struct_ops.c
+++ b/net/bpf/bpf_dummy_struct_ops.c
@@ -72,13 +72,16 @@ static int dummy_ops_call_op(void *image, struct bpf_dummy_ops_test_args *args)
                    args->args[3], args->args[4]);
 }
 
+extern const struct bpf_link_ops bpf_struct_ops_link_lops;
+
 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
                            union bpf_attr __user *uattr)
 {
        const struct bpf_struct_ops *st_ops = &bpf_bpf_dummy_ops;
        const struct btf_type *func_proto;
        struct bpf_dummy_ops_test_args *args;
-       struct bpf_tramp_progs *tprogs;
+       struct bpf_tramp_links *tlinks;
+       struct bpf_tramp_link *link = NULL;
        void *image = NULL;
        unsigned int op_idx;
        int prog_ret;
@@ -92,8 +95,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
        if (IS_ERR(args))
                return PTR_ERR(args);
 
-       tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
-       if (!tprogs) {
+       tlinks = kcalloc(BPF_TRAMP_MAX, sizeof(*tlinks), GFP_KERNEL);
+       if (!tlinks) {
                err = -ENOMEM;
                goto out;
        }
@@ -105,8 +108,17 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
        }
        set_vm_flush_reset_perms(image);
 
+       link = kzalloc(sizeof(*link), GFP_USER);
+       if (!link) {
+               err = -ENOMEM;
+               goto out;
+       }
+       /* prog doesn't take the ownership of the reference from caller */
+       bpf_prog_inc(prog);
+       bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog);
+
        op_idx = prog->expected_attach_type;
-       err = bpf_struct_ops_prepare_trampoline(tprogs, prog,
+       err = bpf_struct_ops_prepare_trampoline(tlinks, link,
                                                &st_ops->func_models[op_idx],
                                                image, image + PAGE_SIZE);
        if (err < 0)
@@ -124,7 +136,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 out:
        kfree(args);
        bpf_jit_free_exec(image);
-       kfree(tprogs);
+       if (link)
+               bpf_link_put(&link->link);
+       kfree(tlinks);
        return err;
 }
 

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 8fb0116..6353a78 100644 (file)
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -23,6 +23,7 @@ static const char * const link_type_name[] = {
        [BPF_LINK_TYPE_XDP]                     = "xdp",
        [BPF_LINK_TYPE_PERF_EVENT]              = "perf_event",
        [BPF_LINK_TYPE_KPROBE_MULTI]            = "kprobe_multi",
+       [BPF_LINK_TYPE_STRUCT_OPS]               = "struct_ops",
 };
 
 static struct hashmap *link_table;

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 95a3d1f..3d032ea 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1013,6 +1013,7 @@ enum bpf_link_type {
        BPF_LINK_TYPE_XDP = 6,
        BPF_LINK_TYPE_PERF_EVENT = 7,
        BPF_LINK_TYPE_KPROBE_MULTI = 8,
+       BPF_LINK_TYPE_STRUCT_OPS = 9,
 
        MAX_BPF_LINK_TYPE,
 };