Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...

Pull x86 fixes from Thomas Gleixner:
 "Yet another pile of melted spectrum related changes:

   - sanitize the array_index_nospec protection mechanism: Remove the
     overengineered array_index_nospec_mask_check() magic and allow
     const-qualified types as index to avoid temporary storage in a
     non-const local variable.

   - make the microcode loader more robust by properly propagating error
     codes. Provide information about new feature bits after micro code
     was updated so administrators can act upon.

   - optimizations of the entry ASM code which reduce code footprint and
     make the code simpler and faster.

   - fix the {pmd,pud}_{set,clear}_flags() implementations to work
     properly on paravirt kernels by removing the address translation
     operations.

   - revert the harmful vmexit_fill_RSB() optimization

   - use IBRS around firmware calls

   - teach objtool about retpolines and add annotations for indirect
     jumps and calls.

   - explicitly disable jumplabel patching in __init code and handle
     patching failures properly instead of silently ignoring them.

   - remove indirect paravirt calls for writing the speculation control
     MSR as these calls are obviously proving the same attack vector
     which is tried to be mitigated.

   - a few small fixes which address build issues with recent compiler
     and assembler versions"

* 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (38 commits)
  KVM/VMX: Optimize vmx_vcpu_run() and svm_vcpu_run() by marking the RDMSR path as unlikely()
  KVM/x86: Remove indirect MSR op calls from SPEC_CTRL
  objtool, retpolines: Integrate objtool with retpoline support more closely
  x86/entry/64: Simplify ENCODE_FRAME_POINTER
  extable: Make init_kernel_text() global
  jump_label: Warn on failed jump_label patching attempt
  jump_label: Explicitly disable jump labels in __init code
  x86/entry/64: Open-code switch_to_thread_stack()
  x86/entry/64: Move ASM_CLAC to interrupt_entry()
  x86/entry/64: Remove 'interrupt' macro
  x86/entry/64: Move the switch_to_thread_stack() call to interrupt_entry()
  x86/entry/64: Move ENTER_IRQ_STACK from interrupt macro to interrupt_entry
  x86/entry/64: Move PUSH_AND_CLEAR_REGS from interrupt macro to helper function
  x86/speculation: Move firmware_restrict_branch_speculation_*() from C to CPP
  objtool: Add module specific retpoline rules
  objtool: Add retpoline validation
  objtool: Use existing global variables for options
  x86/mm/sme, objtool: Annotate indirect call in sme_encrypt_execute()
  x86/boot, objtool: Annotate indirect jump in secondary_startup_64()
  x86/paravirt, objtool: Annotate indirect calls
  ...

diff --cc Makefile
Simple merge

diff --cc arch/x86/Kconfig
Simple merge

diff --cc arch/x86/include/asm/nospec-branch.h
index 81a1be3265711acea06a2538b6b1fdd332b0ab19,b7063cfa19f97f763f3b45cd8050e41cdf69f0a9..d0dabeae05059883844ae2228af62c27885cdbfa
--- 1/arch/x86/include/asm/nospec-branch.h
--- 2/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@@ -163,55 -238,41 +238,78 @@@ static inline void vmexit_fill_RSB(void
  #endif
  }
  
+ #define alternative_msr_write(_msr, _val, _feature)           \
+       asm volatile(ALTERNATIVE("",                            \
+                                "movl %[msr], %%ecx\n\t"       \
+                                "movl %[val], %%eax\n\t"       \
+                                "movl $0, %%edx\n\t"           \
+                                "wrmsr",                       \
+                                _feature)                      \
+                    : : [msr] "i" (_msr), [val] "i" (_val)     \
+                    : "eax", "ecx", "edx", "memory")
+ 
  static inline void indirect_branch_prediction_barrier(void)
  {
-       asm volatile(ALTERNATIVE("",
-                                "movl %[msr], %%ecx\n\t"
-                                "movl %[val], %%eax\n\t"
-                                "movl $0, %%edx\n\t"
-                                "wrmsr",
-                                X86_FEATURE_USE_IBPB)
-                    : : [msr] "i" (MSR_IA32_PRED_CMD),
-                        [val] "i" (PRED_CMD_IBPB)
-                    : "eax", "ecx", "edx", "memory");
+       alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
+                             X86_FEATURE_USE_IBPB);
  }
  
+ /*
+  * With retpoline, we must use IBRS to restrict branch prediction
+  * before calling into firmware.
+  *
+  * (Implemented as CPP macros due to header hell.)
+  */
+ #define firmware_restrict_branch_speculation_start()                  \
+ do {                                                                  \
+       preempt_disable();                                              \
+       alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS,       \
+                             X86_FEATURE_USE_IBRS_FW);                 \
+ } while (0)
+ 
+ #define firmware_restrict_branch_speculation_end()                    \
+ do {                                                                  \
+       alternative_msr_write(MSR_IA32_SPEC_CTRL, 0,                    \
+                             X86_FEATURE_USE_IBRS_FW);                 \
+       preempt_enable();                                               \
+ } while (0)
+ 
  #endif /* __ASSEMBLY__ */
 +
 +/*
 + * Below is used in the eBPF JIT compiler and emits the byte sequence
 + * for the following assembly:
 + *
 + * With retpolines configured:
 + *
 + *    callq do_rop
 + *  spec_trap:
 + *    pause
 + *    lfence
 + *    jmp spec_trap
 + *  do_rop:
 + *    mov %rax,(%rsp)
 + *    retq
 + *
 + * Without retpolines configured:
 + *
 + *    jmp *%rax
 + */
 +#ifdef CONFIG_RETPOLINE
 +# define RETPOLINE_RAX_BPF_JIT_SIZE   17
 +# define RETPOLINE_RAX_BPF_JIT()                              \
 +      EMIT1_off32(0xE8, 7);    /* callq do_rop */             \
 +      /* spec_trap: */                                        \
 +      EMIT2(0xF3, 0x90);       /* pause */                    \
 +      EMIT3(0x0F, 0xAE, 0xE8); /* lfence */                   \
 +      EMIT2(0xEB, 0xF9);       /* jmp spec_trap */            \
 +      /* do_rop: */                                           \
 +      EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */    \
 +      EMIT1(0xC3);             /* retq */
 +#else
 +# define RETPOLINE_RAX_BPF_JIT_SIZE   2
 +# define RETPOLINE_RAX_BPF_JIT()                              \
 +      EMIT2(0xFF, 0xE0);       /* jmp *%rax */
 +#endif
 +
  #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */

diff --cc arch/x86/kvm/svm.c
Simple merge

diff --cc arch/x86/kvm/vmx.c
Simple merge

diff --cc include/linux/compiler-gcc.h
Simple merge