Merge tag 'x86_seves_for_v5.10' of git://git.kernel.org/pub/scm/linux/kernel/git...

Pull x86 SEV-ES support from Borislav Petkov:
 "SEV-ES enhances the current guest memory encryption support called SEV
  by also encrypting the guest register state, making the registers
  inaccessible to the hypervisor by en-/decrypting them on world
  switches. Thus, it adds additional protection to Linux guests against
  exfiltration, control flow and rollback attacks.

  With SEV-ES, the guest is in full control of what registers the
  hypervisor can access. This is provided by a guest-host exchange
  mechanism based on a new exception vector called VMM Communication
  Exception (#VC), a new instruction called VMGEXIT and a shared
  Guest-Host Communication Block which is a decrypted page shared
  between the guest and the hypervisor.

  Intercepts to the hypervisor become #VC exceptions in an SEV-ES guest
  so in order for that exception mechanism to work, the early x86 init
  code needed to be made able to handle exceptions, which, in itself,
  brings a bunch of very nice cleanups and improvements to the early
  boot code like an early page fault handler, allowing for on-demand
  building of the identity mapping. With that, !KASLR configurations do
  not use the EFI page table anymore but switch to a kernel-controlled
  one.

  The main part of this series adds the support for that new exchange
  mechanism. The goal has been to keep this as much as possibly separate
  from the core x86 code by concentrating the machinery in two
  SEV-ES-specific files:

    arch/x86/kernel/sev-es-shared.c
    arch/x86/kernel/sev-es.c

  Other interaction with core x86 code has been kept at minimum and
  behind static keys to minimize the performance impact on !SEV-ES
  setups.

  Work by Joerg Roedel and Thomas Lendacky and others"

* tag 'x86_seves_for_v5.10' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (73 commits)
  x86/sev-es: Use GHCB accessor for setting the MMIO scratch buffer
  x86/sev-es: Check required CPU features for SEV-ES
  x86/efi: Add GHCB mappings when SEV-ES is active
  x86/sev-es: Handle NMI State
  x86/sev-es: Support CPU offline/online
  x86/head/64: Don't call verify_cpu() on starting APs
  x86/smpboot: Load TSS and getcpu GDT entry before loading IDT
  x86/realmode: Setup AP jump table
  x86/realmode: Add SEV-ES specific trampoline entry point
  x86/vmware: Add VMware-specific handling for VMMCALL under SEV-ES
  x86/kvm: Add KVM-specific VMMCALL handling under SEV-ES
  x86/paravirt: Allow hypervisor-specific VMMCALL handling under SEV-ES
  x86/sev-es: Handle #DB Events
  x86/sev-es: Handle #AC Events
  x86/sev-es: Handle VMMCALL Events
  x86/sev-es: Handle MWAIT/MWAITX Events
  x86/sev-es: Handle MONITOR/MONITORX Events
  x86/sev-es: Handle INVD Events
  x86/sev-es: Handle RDPMC Events
  x86/sev-es: Handle RDTSC(P) Events
  ...

diff --cc arch/x86/Kconfig
Simple merge

diff --cc arch/x86/boot/compressed/Makefile
index 4fb989ef5665a7211e79cff26814c3370bace608,c01236ae1b7cf230d27a1121b2cb9ad5a6f609fd..ee249088cbfe2ff46f2b7e2c76252b1f9001bc9f
--- 1/arch/x86/boot/compressed/Makefile
--- 2/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@@ -29,10 -29,10 +29,10 @@@ targets := vmlinux vmlinux.bin vmlinux.
        vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
  
  KBUILD_CFLAGS := -m$(BITS) -O2
 -KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
 +KBUILD_CFLAGS += -fno-strict-aliasing -fPIE
  KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
  cflags-$(CONFIG_X86_32) := -march=i386
- cflags-$(CONFIG_X86_64) := -mcmodel=small
+ cflags-$(CONFIG_X86_64) := -mcmodel=small -mno-red-zone
  KBUILD_CFLAGS += $(cflags-y)
  KBUILD_CFLAGS += -mno-mmx -mno-sse
  KBUILD_CFLAGS += -ffreestanding
@@@ -43,10 -43,12 +43,15 @@@ KBUILD_CFLAGS += -Wno-pointer-sig
  KBUILD_CFLAGS += $(call cc-option,-fmacro-prefix-map=$(srctree)/=)
  KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
  KBUILD_CFLAGS += -D__DISABLE_EXPORTS
 +# Disable relocation relaxation in case the link is not PIE.
 +KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no)
 +KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h
  
+ # sev-es.c indirectly inludes inat-table.h which is generated during
+ # compilation and stored in $(objtree). Add the directory to the includes so
+ # that the compiler finds it even with out-of-tree builds (make O=/some/path).
+ CFLAGS_sev-es.o += -I$(objtree)/arch/x86/lib/
+ 
  KBUILD_AFLAGS  := $(KBUILD_CFLAGS) -D__ASSEMBLY__
  GCOV_PROFILE := n
  UBSAN_SANITIZE :=n

diff --cc arch/x86/boot/compressed/head_64.S
Simple merge

diff --cc arch/x86/entry/entry_64.S
Simple merge

diff --cc arch/x86/include/asm/cpufeatures.h
Simple merge

diff --cc arch/x86/include/asm/fpu/internal.h
index eb1ed3bd8d962565cad1598a0f6abf3ed01ae33f,ceeba9f6317222ba114bcd213fc6f55e7fb4e504..8d33ad80704f264ba4db632dde22a7e5f6caa816
--- 1/arch/x86/include/asm/fpu/internal.h
--- 2/arch/x86/include/asm/fpu/internal.h
+++ b/arch/x86/include/asm/fpu/internal.h
@@@ -583,42 -584,6 +584,13 @@@ static inline void switch_fpu_finish(st
                        pkru_val = pk->pkru;
        }
        __write_pkru(pkru_val);
 +
 +      /*
 +       * Expensive PASID MSR write will be avoided in update_pasid() because
 +       * TIF_NEED_FPU_LOAD was set. And the PASID state won't be updated
 +       * unless it's different from mm->pasid to reduce overhead.
 +       */
 +      update_pasid();
  }
  
- /*
-  * MXCSR and XCR definitions:
-  */
- 
- static inline void ldmxcsr(u32 mxcsr)
- {
-       asm volatile("ldmxcsr %0" :: "m" (mxcsr));
- }
- 
- extern unsigned int mxcsr_feature_mask;
- 
- #define XCR_XFEATURE_ENABLED_MASK     0x00000000
- 
- static inline u64 xgetbv(u32 index)
- {
-       u32 eax, edx;
- 
-       asm volatile("xgetbv" : "=a" (eax), "=d" (edx) : "c" (index));
-       return eax + ((u64)edx << 32);
- }
- 
- static inline void xsetbv(u32 index, u64 value)
- {
-       u32 eax = value;
-       u32 edx = value >> 32;
- 
-       asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
- }
- 
  #endif /* _ASM_X86_FPU_INTERNAL_H */

diff --cc arch/x86/include/asm/idtentry.h
Simple merge

diff --cc arch/x86/include/asm/msr-index.h
index c07a70ce7ffdfdef4c448c25ab2873a50d796bbf,249a4147c4b26df3ca1defe462b5b572095c60a8..972a34d935059aa49d01db8f20c3e598fffb22ac
--- 1/arch/x86/include/asm/msr-index.h
--- 2/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@@ -467,12 -464,14 +467,15 @@@
  #define MSR_AMD64_IBSOP_REG_MASK      ((1UL<<MSR_AMD64_IBSOP_REG_COUNT)-1)
  #define MSR_AMD64_IBSCTL              0xc001103a
  #define MSR_AMD64_IBSBRTARGET         0xc001103b
 +#define MSR_AMD64_ICIBSEXTDCTL                0xc001103c
  #define MSR_AMD64_IBSOPDATA4          0xc001103d
  #define MSR_AMD64_IBS_REG_COUNT_MAX   8 /* includes MSR_AMD64_IBSBRTARGET */
+ #define MSR_AMD64_SEV_ES_GHCB         0xc0010130
  #define MSR_AMD64_SEV                 0xc0010131
  #define MSR_AMD64_SEV_ENABLED_BIT     0
+ #define MSR_AMD64_SEV_ES_ENABLED_BIT  1
  #define MSR_AMD64_SEV_ENABLED         BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT)
+ #define MSR_AMD64_SEV_ES_ENABLED      BIT_ULL(MSR_AMD64_SEV_ES_ENABLED_BIT)
  
  #define MSR_AMD64_VIRT_SPEC_CTRL      0xc001011f
  

diff --cc arch/x86/include/asm/pgtable.h
Simple merge

diff --cc arch/x86/include/asm/processor.h
Simple merge

diff --cc arch/x86/include/asm/segment.h
Simple merge

diff --cc arch/x86/include/asm/traps.h
Simple merge

diff --cc arch/x86/include/asm/x86_init.h
index 397196fae24d36333bd2c6d4be43fe5041157cce,0304e2931cd3e3dc9046d298a33ae48f0f57fc60..dde5b3f1e7cd56367d0382bd1f591fd940ee70d5
--- 1/arch/x86/include/asm/x86_init.h
--- 2/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@@ -4,11 -4,12 +4,13 @@@
  
  #include <asm/bootparam.h>
  
+ struct ghcb;
  struct mpc_bus;
  struct mpc_cpu;
+ struct pt_regs;
  struct mpc_table;
  struct cpuinfo_x86;
 +struct irq_domain;
  
  /**
   * struct x86_init_mpparse - platform specific mpparse ops

diff --cc arch/x86/kernel/Makefile
Simple merge

diff --cc arch/x86/kernel/cpu/common.c
Simple merge

diff --cc arch/x86/kernel/cpu/scattered.c
index 2eb0a8c44b352216b62691b0be15c4a068257179,30f354989cf11e0318c81ab0600c4afb90b4553a..866c9a9bcdee7b1eb4c525aea2fb74ab5ec82882
--- 1/arch/x86/kernel/cpu/scattered.c
--- 2/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@@ -42,7 -41,7 +42,8 @@@ static const struct cpuid_bit cpuid_bit
        { X86_FEATURE_MBA,              CPUID_EBX,  6, 0x80000008, 0 },
        { X86_FEATURE_SME,              CPUID_EAX,  0, 0x8000001f, 0 },
        { X86_FEATURE_SEV,              CPUID_EAX,  1, 0x8000001f, 0 },
+       { X86_FEATURE_SEV_ES,           CPUID_EAX,  3, 0x8000001f, 0 },
 +      { X86_FEATURE_SME_COHERENT,     CPUID_EAX, 10, 0x8000001f, 0 },
        { 0, 0, 0, 0, 0 }
  };
  

diff --cc arch/x86/kernel/dumpstack.c
Simple merge

diff --cc arch/x86/kernel/idt.c
Simple merge

diff --cc arch/x86/kernel/kvm.c
Simple merge

diff --cc arch/x86/kernel/nmi.c
Simple merge

diff --cc arch/x86/kernel/traps.c
Simple merge

diff --cc arch/x86/kvm/svm/nested.c
index e90bc436f5849b3bbe4727a5cd6e59d5b687c8d1,28036629abf8d7a4e655245081de4e5ffe51c3dc..598a769f19617a81baf6af473b1da6f79c41e847
--- 1/arch/x86/kvm/svm/nested.c
--- 2/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@@ -1128,18 -1139,21 +1141,24 @@@ static int svm_set_nested_state(struct 
         * contains saved L1 state.
         */
        copy_vmcb_control_area(&hsave->control, &svm->vmcb->control);
-       hsave->save = save;
+       hsave->save = *save;
  
        svm->nested.vmcb = kvm_state->hdr.svm.vmcb_pa;
-       load_nested_vmcb_control(svm, &ctl);
+       load_nested_vmcb_control(svm, ctl);
        nested_prepare_vmcb_control(svm);
  
 +      if (!nested_svm_vmrun_msrpm(svm))
 +              return -EINVAL;
 +
  out_set_gif:
        svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
-       return 0;
+ 
+       ret = 0;
+ out_free:
+       kfree(save);
+       kfree(ctl);
+ 
+       return ret;
  }
  
  struct kvm_x86_nested_ops svm_nested_ops = {

diff --cc arch/x86/kvm/svm/svm.c
Simple merge

diff --cc arch/x86/mm/extable.c
Simple merge