x86: Undo return-thunk damage

Introduce X86_FEATURE_RETHUNK for those afflicted with needing this.

  [ bp: Do only INT3 padding - simpler. ]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Signed-off-by: Borislav Petkov <bp@suse.de>

diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 9b10c8c..9542c58 100644 (file)
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -76,6 +76,7 @@ extern int alternatives_patched;
 extern void alternative_instructions(void);
 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
 extern void apply_retpolines(s32 *start, s32 *end);
+extern void apply_returns(s32 *start, s32 *end);
 extern void apply_ibt_endbr(s32 *start, s32 *end);
 
 struct module;

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 4807302..295e690 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -300,6 +300,7 @@
 /* FREE!                               (11*32+11) */
 #define X86_FEATURE_RETPOLINE          (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_RETPOLINE_LFENCE   (11*32+13) /* "" Use LFENCE for Spectre variant 2 */
+#define X86_FEATURE_RETHUNK            (11*32+14) /* "" Use REturn THUNK */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
 #define X86_FEATURE_AVX_VNNI           (12*32+ 4) /* AVX VNNI instructions */

diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
index 4061f49..641c479 100644 (file)
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -54,7 +54,8 @@
 # define DISABLE_RETPOLINE     0
 #else
 # define DISABLE_RETPOLINE     ((1 << (X86_FEATURE_RETPOLINE & 31)) | \
-                                (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)))
+                                (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) | \
+                                (1 << (X86_FEATURE_RETHUNK & 31)))
 #endif
 
 #ifdef CONFIG_INTEL_IOMMU_SVM

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index e257f6c..76b7459 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -115,6 +115,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)
 }
 
 extern s32 __retpoline_sites[], __retpoline_sites_end[];
+extern s32 __return_sites[], __return_sites_end[];
 extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 extern s32 __smp_locks[], __smp_locks_end[];
@@ -507,9 +508,67 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
        }
 }
 
+/*
+ * Rewrite the compiler generated return thunk tail-calls.
+ *
+ * For example, convert:
+ *
+ *   JMP __x86_return_thunk
+ *
+ * into:
+ *
+ *   RET
+ */
+static int patch_return(void *addr, struct insn *insn, u8 *bytes)
+{
+       int i = 0;
+
+       if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
+               return -1;
+
+       bytes[i++] = RET_INSN_OPCODE;
+
+       for (; i < insn->length;)
+               bytes[i++] = INT3_INSN_OPCODE;
+
+       return i;
+}
+
+void __init_or_module noinline apply_returns(s32 *start, s32 *end)
+{
+       s32 *s;
+
+       for (s = start; s < end; s++) {
+               void *addr = (void *)s + *s;
+               struct insn insn;
+               int len, ret;
+               u8 bytes[16];
+               u8 op1;
+
+               ret = insn_decode_kernel(&insn, addr);
+               if (WARN_ON_ONCE(ret < 0))
+                       continue;
+
+               op1 = insn.opcode.bytes[0];
+               if (WARN_ON_ONCE(op1 != JMP32_INSN_OPCODE))
+                       continue;
+
+               DPRINTK("return thunk at: %pS (%px) len: %d to: %pS",
+                       addr, addr, insn.length,
+                       addr + insn.length + insn.immediate.value);
+
+               len = patch_return(addr, &insn, bytes);
+               if (len == insn.length) {
+                       DUMP_BYTES(((u8*)addr),  len, "%px: orig: ", addr);
+                       DUMP_BYTES(((u8*)bytes), len, "%px: repl: ", addr);
+                       text_poke_early(addr, bytes, len);
+               }
+       }
+}
 #else /* !CONFIG_RETPOLINE || !CONFIG_OBJTOOL */
 
 void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
+void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
 
 #endif /* CONFIG_RETPOLINE && CONFIG_OBJTOOL */
 
@@ -860,6 +919,7 @@ void __init alternative_instructions(void)
         * those can rewrite the retpoline thunks.
         */
        apply_retpolines(__retpoline_sites, __retpoline_sites_end);
+       apply_returns(__return_sites, __return_sites_end);
 
        /*
         * Then patch alternatives, such that those paravirt calls that are in

diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index b98ffcf..67828d9 100644 (file)
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -253,7 +253,7 @@ int module_finalize(const Elf_Ehdr *hdr,
 {
        const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
                *para = NULL, *orc = NULL, *orc_ip = NULL,
-               *retpolines = NULL, *ibt_endbr = NULL;
+               *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL;
        char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 
        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -271,6 +271,8 @@ int module_finalize(const Elf_Ehdr *hdr,
                        orc_ip = s;
                if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
                        retpolines = s;
+               if (!strcmp(".return_sites", secstrings + s->sh_name))
+                       returns = s;
                if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
                        ibt_endbr = s;
        }
@@ -287,6 +289,10 @@ int module_finalize(const Elf_Ehdr *hdr,
                void *rseg = (void *)retpolines->sh_addr;
                apply_retpolines(rseg, rseg + retpolines->sh_size);
        }
+       if (returns) {
+               void *rseg = (void *)returns->sh_addr;
+               apply_returns(rseg, rseg + returns->sh_size);
+       }
        if (alt) {
                /* patch .altinstructions */
                void *aseg = (void *)alt->sh_addr;

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 81aba71..ada7eb7 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -283,6 +283,13 @@ SECTIONS
                *(.retpoline_sites)
                __retpoline_sites_end = .;
        }
+
+       . = ALIGN(8);
+       .return_sites : AT(ADDR(.return_sites) - LOAD_OFFSET) {
+               __return_sites = .;
+               *(.return_sites)
+               __return_sites_end = .;
+       }
 #endif
 
 #ifdef CONFIG_X86_KERNEL_IBT