libbpf: Support subprog address relocation

A new relocation RELO_SUBPROG_ADDR is added to capture
subprog addresses loaded with ld_imm64 insns. Such ld_imm64
insns are marked with BPF_PSEUDO_FUNC and will be passed to
kernel. For bpf_for_each_map_elem() case, kernel will
check that the to-be-used subprog address must be a static
function and replace it with proper actual jited func address.

Signed-off-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20210226204930.3885367-1-yhs@fb.com

diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 21a3eed..62d9ed5 100644 (file)
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -188,6 +188,7 @@ enum reloc_type {
        RELO_CALL,
        RELO_DATA,
        RELO_EXTERN,
+       RELO_SUBPROG_ADDR,
 };
 
 struct reloc_desc {
@@ -579,6 +580,11 @@ static bool is_ldimm64(struct bpf_insn *insn)
        return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
 }
 
+static bool insn_is_pseudo_func(struct bpf_insn *insn)
+{
+       return is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
+}
+
 static int
 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
                      const char *name, size_t sec_idx, const char *sec_name,
@@ -2979,6 +2985,23 @@ static bool sym_is_extern(const GElf_Sym *sym)
               GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
 }
 
+static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx)
+{
+       int bind = GELF_ST_BIND(sym->st_info);
+       int type = GELF_ST_TYPE(sym->st_info);
+
+       /* in .text section */
+       if (sym->st_shndx != text_shndx)
+               return false;
+
+       /* local function */
+       if (bind == STB_LOCAL && type == STT_SECTION)
+               return true;
+
+       /* global function */
+       return bind == STB_GLOBAL && type == STT_FUNC;
+}
+
 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
 {
        const struct btf_type *t;
@@ -3435,6 +3458,23 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
                return -LIBBPF_ERRNO__RELOC;
        }
 
+       /* loading subprog addresses */
+       if (sym_is_subprog(sym, obj->efile.text_shndx)) {
+               /* global_func: sym->st_value = offset in the section, insn->imm = 0.
+                * local_func: sym->st_value = 0, insn->imm = offset in the section.
+                */
+               if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
+                       pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
+                               prog->name, sym_name, (size_t)sym->st_value, insn->imm);
+                       return -LIBBPF_ERRNO__RELOC;
+               }
+
+               reloc_desc->type = RELO_SUBPROG_ADDR;
+               reloc_desc->insn_idx = insn_idx;
+               reloc_desc->sym_off = sym->st_value;
+               return 0;
+       }
+
        type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
        sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
 
@@ -6172,6 +6212,10 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
                        }
                        relo->processed = true;
                        break;
+               case RELO_SUBPROG_ADDR:
+                       insn[0].src_reg = BPF_PSEUDO_FUNC;
+                       /* will be handled as a follow up pass */
+                       break;
                case RELO_CALL:
                        /* will be handled as a follow up pass */
                        break;
@@ -6358,11 +6402,11 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
 
        for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
                insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
-               if (!insn_is_subprog_call(insn))
+               if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
                        continue;
 
                relo = find_prog_insn_relo(prog, insn_idx);
-               if (relo && relo->type != RELO_CALL) {
+               if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
                        pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
                                prog->name, insn_idx, relo->type);
                        return -LIBBPF_ERRNO__RELOC;
@@ -6374,8 +6418,22 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
                         * call always has imm = -1, but for static functions
                         * relocation is against STT_SECTION and insn->imm
                         * points to a start of a static function
+                        *
+                        * for subprog addr relocation, the relo->sym_off + insn->imm is
+                        * the byte offset in the corresponding section.
                         */
-                       sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
+                       if (relo->type == RELO_CALL)
+                               sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
+                       else
+                               sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
+               } else if (insn_is_pseudo_func(insn)) {
+                       /*
+                        * RELO_SUBPROG_ADDR relo is always emitted even if both
+                        * functions are in the same section, so it shouldn't reach here.
+                        */
+                       pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
+                               prog->name, insn_idx);
+                       return -LIBBPF_ERRNO__RELOC;
                } else {
                        /* if subprogram call is to a static function within
                         * the same ELF section, there won't be any relocation