x86/bpf: Use alternative RET encoding

commit d77cfe594ad50e0bf95d457e02ccd578791b2a15 upstream.

Use the return thunk in eBPF generated code, if needed.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index c904598..131f7ce 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -406,6 +406,21 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
        *pprog = prog;
 }
 
+static void emit_return(u8 **pprog, u8 *ip)
+{
+       u8 *prog = *pprog;
+
+       if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
+               emit_jump(&prog, &__x86_return_thunk, ip);
+       } else {
+               EMIT1(0xC3);            /* ret */
+               if (IS_ENABLED(CONFIG_SLS))
+                       EMIT1(0xCC);    /* int3 */
+       }
+
+       *pprog = prog;
+}
+
 /*
  * Generate the following code:
  *
@@ -1681,7 +1696,7 @@ emit_jmp:
                        ctx->cleanup_addr = proglen;
                        pop_callee_regs(&prog, callee_regs_used);
                        EMIT1(0xC9);         /* leave */
-                       EMIT1(0xC3);         /* ret */
+                       emit_return(&prog, image + addrs[i - 1] + (prog - temp));
                        break;
 
                default:
@@ -2127,7 +2142,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
        if (flags & BPF_TRAMP_F_SKIP_FRAME)
                /* skip our return address and return to parent */
                EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
-       EMIT1(0xC3); /* ret */
+       emit_return(&prog, prog);
        /* Make sure the trampoline generation logic doesn't overflow */
        if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) {
                ret = -EFAULT;