bpf, x64: Fix tailcall infinite loop

[platform/kernel/linux-rpi.git] / arch / x86 / net / bpf_jit_comp.c

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index a593004..52f36c4 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1018,6 +1018,10 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)
 
 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
 
+/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
+#define RESTORE_TAIL_CALL_CNT(stack)                           \
+       EMIT3_off32(0x48, 0x8B, 0x85, -round_up(stack, 8) - 8)
+
 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
                  int oldproglen, struct jit_context *ctx, bool jmp_padding)
 {
@@ -1623,9 +1627,7 @@ st:                       if (is_imm8(insn->off))
 
                        func = (u8 *) __bpf_call_base + imm32;
                        if (tail_call_reachable) {
-                               /* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
-                               EMIT3_off32(0x48, 0x8B, 0x85,
-                                           -round_up(bpf_prog->aux->stack_depth, 8) - 8);
+                               RESTORE_TAIL_CALL_CNT(bpf_prog->aux->stack_depth);
                                if (!imm32)
                                        return -EINVAL;
                                offs = 7 + x86_call_depth_emit_accounting(&prog, func);
@@ -2400,6 +2402,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
         *                     [ ...        ]
         *                     [ stack_arg2 ]
         * RBP - arg_stack_off [ stack_arg1 ]
+        * RSP                 [ tail_call_cnt ] BPF_TRAMP_F_TAIL_CALL_CTX
         */
 
        /* room for return value of orig_call or fentry prog */
@@ -2464,6 +2467,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
        else
                /* sub rsp, stack_size */
                EMIT4(0x48, 0x83, 0xEC, stack_size);
+       if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
+               EMIT1(0x50);            /* push rax */
        /* mov QWORD PTR [rbp - rbx_off], rbx */
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off);
 
@@ -2516,9 +2521,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                restore_regs(m, &prog, regs_off);
                save_args(m, &prog, arg_stack_off, true);
 
+               if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
+                       /* Before calling the original function, restore the
+                        * tail_call_cnt from stack to rax.
+                        */
+                       RESTORE_TAIL_CALL_CNT(stack_size);
+
                if (flags & BPF_TRAMP_F_ORIG_STACK) {
-                       emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8);
-                       EMIT2(0xff, 0xd0); /* call *rax */
+                       emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8);
+                       EMIT2(0xff, 0xd3); /* call *rbx */
                } else {
                        /* call original function */
                        if (emit_rsb_call(&prog, orig_call, prog)) {
@@ -2569,7 +2580,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
                        ret = -EINVAL;
                        goto cleanup;
                }
-       }
+       } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
+               /* Before running the original function, restore the
+                * tail_call_cnt from stack to rax.
+                */
+               RESTORE_TAIL_CALL_CNT(stack_size);
+
        /* restore return value of orig_call or fentry prog back into RAX */
        if (save_ret)
                emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);