bpf: Add support to inline bpf_get_func_ip helper on x86

Adding support to inline it on x86, because it's single
load instruction.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20220316122419.933957-6-jolsa@kernel.org

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index cf92f9c01556f337748f39aa1cb0ad085e18c5f7..0287176bfe9a53823d902090075e292c48351d69 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -13678,7 +13678,7 @@ patch_map_ops_generic:
                        continue;
                }
 
-               /* Implement bpf_get_func_ip inline. */
+               /* Implement tracing bpf_get_func_ip inline. */
                if (prog_type == BPF_PROG_TYPE_TRACING &&
                    insn->imm == BPF_FUNC_get_func_ip) {
                        /* Load IP address from ctx - 16 */
@@ -13693,6 +13693,25 @@ patch_map_ops_generic:
                        continue;
                }
 
+#ifdef CONFIG_X86
+               /* Implement kprobe_multi bpf_get_func_ip inline. */
+               if (prog_type == BPF_PROG_TYPE_KPROBE &&
+                   eatype == BPF_TRACE_KPROBE_MULTI &&
+                   insn->imm == BPF_FUNC_get_func_ip) {
+                       /* Load IP address from ctx (struct pt_regs) ip */
+                       insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1,
+                                                 offsetof(struct pt_regs, ip));
+
+                       new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
+                       if (!new_prog)
+                               return -ENOMEM;
+
+                       env->prog = prog = new_prog;
+                       insn      = new_prog->insnsi + i + delta;
+                       continue;
+               }
+#endif
+
 patch_call_imm:
                fn = env->ops->get_func_proto(insn->imm, env->prog);
                /* all functions that have prototype and verifier allowed

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 2507509322288de817600d6c8a23dea17045ab8b..0e7f8c9bc756d54e224df59058b25c872159c226 100644 (file)
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -1039,6 +1039,7 @@ static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = {
 
 BPF_CALL_1(bpf_get_func_ip_kprobe_multi, struct pt_regs *, regs)
 {
+       /* This helper call is inlined by verifier on x86. */
        return instruction_pointer(regs);
 }