x86/retbleed: Add fine grained Kconfig knobs

[platform/kernel/linux-rpi.git] / arch / x86 / include / asm / nospec-branch.h

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index b3b2667..b73b870 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -122,13 +122,17 @@
   * monstrosity above, manually.
   */
 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
-#ifdef CONFIG_RETPOLINE
        ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr
        __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
 .Lskip_rsb_\@:
-#endif
 .endm
 
+#ifdef CONFIG_CPU_UNRET_ENTRY
+#define CALL_ZEN_UNTRAIN_RET   "call zen_untrain_ret"
+#else
+#define CALL_ZEN_UNTRAIN_RET   ""
+#endif
+
 /*
  * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
  * return thunk isn't mapped into the userspace tables (then again, AMD
@@ -141,10 +145,10 @@
  * where we have a stack but before any RET instruction.
  */
 .macro UNTRAIN_RET
-#ifdef CONFIG_RETPOLINE
+#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY)
        ANNOTATE_UNRET_END
        ALTERNATIVE_2 "",                                               \
-                     "call zen_untrain_ret", X86_FEATURE_UNRET,        \
+                     CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET,          \
                      "call entry_ibpb", X86_FEATURE_ENTRY_IBPB
 #endif
 .endm
@@ -277,6 +281,7 @@ static inline void indirect_branch_prediction_barrier(void)
 
 /* The Intel SPEC CTRL MSR base value cache */
 extern u64 x86_spec_ctrl_base;
+extern u64 x86_spec_ctrl_current;
 extern void write_spec_ctrl_current(u64 val, bool force);
 extern u64 spec_ctrl_current(void);
 
@@ -288,18 +293,16 @@ extern u64 spec_ctrl_current(void);
  */
 #define firmware_restrict_branch_speculation_start()                   \
 do {                                                                   \
-       u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS;                  \
-                                                                       \
        preempt_disable();                                              \
-       alternative_msr_write(MSR_IA32_SPEC_CTRL, val,                  \
+       alternative_msr_write(MSR_IA32_SPEC_CTRL,                       \
+                             spec_ctrl_current() | SPEC_CTRL_IBRS,     \
                              X86_FEATURE_USE_IBRS_FW);                 \
 } while (0)
 
 #define firmware_restrict_branch_speculation_end()                     \
 do {                                                                   \
-       u64 val = x86_spec_ctrl_base;                                   \
-                                                                       \
-       alternative_msr_write(MSR_IA32_SPEC_CTRL, val,                  \
+       alternative_msr_write(MSR_IA32_SPEC_CTRL,                       \
+                             spec_ctrl_current(),                      \
                              X86_FEATURE_USE_IBRS_FW);                 \
        preempt_enable();                                               \
 } while (0)