arch/x86/kvm/vmx/ops.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __KVM_X86_VMX_INSN_H
   3 #define __KVM_X86_VMX_INSN_H
   4
   5 #include <linux/nospec.h>
   6
   7 #include <asm/kvm_host.h>
   8 #include <asm/vmx.h>
   9
  10 #include "evmcs.h"
  11 #include "vmcs.h"
  12
  13 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  14
  15 __attribute__((regparm(0))) void vmread_error_trampoline(unsigned long field,
  16                                                          bool fault);
  17 void vmwrite_error(unsigned long field, unsigned long value);
  18 void vmclear_error(struct vmcs *vmcs, u64 phys_addr);
  19 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr);
  20 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva);
  21 void invept_error(unsigned long ext, u64 eptp, gpa_t gpa);
  22
  23 static __always_inline void vmcs_check16(unsigned long field)
  24 {
  25         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
  26                          "16-bit accessor invalid for 64-bit field");
  27         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  28                          "16-bit accessor invalid for 64-bit high field");
  29         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  30                          "16-bit accessor invalid for 32-bit high field");
  31         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  32                          "16-bit accessor invalid for natural width field");
  33 }
  34
  35 static __always_inline void vmcs_check32(unsigned long field)
  36 {
  37         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  38                          "32-bit accessor invalid for 16-bit field");
  39         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  40                          "32-bit accessor invalid for natural width field");
  41 }
  42
  43 static __always_inline void vmcs_check64(unsigned long field)
  44 {
  45         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  46                          "64-bit accessor invalid for 16-bit field");
  47         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  48                          "64-bit accessor invalid for 64-bit high field");
  49         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  50                          "64-bit accessor invalid for 32-bit field");
  51         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  52                          "64-bit accessor invalid for natural width field");
  53 }
  54
  55 static __always_inline void vmcs_checkl(unsigned long field)
  56 {
  57         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  58                          "Natural width accessor invalid for 16-bit field");
  59         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
  60                          "Natural width accessor invalid for 64-bit field");
  61         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  62                          "Natural width accessor invalid for 64-bit high field");
  63         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  64                          "Natural width accessor invalid for 32-bit field");
  65 }
  66
  67 static __always_inline unsigned long __vmcs_readl(unsigned long field)
  68 {
  69         unsigned long value;
  70
  71         asm volatile("1: vmread %2, %1\n\t"
  72                      ".byte 0x3e\n\t" /* branch taken hint */
  73                      "ja 3f\n\t"
  74
  75                      /*
  76                       * VMREAD failed.  Push '0' for @fault, push the failing
  77                       * @field, and bounce through the trampoline to preserve
  78                       * volatile registers.
  79                       */
  80                      "push $0\n\t"
  81                      "push %2\n\t"
  82                      "2:call vmread_error_trampoline\n\t"
  83
  84                      /*
  85                       * Unwind the stack.  Note, the trampoline zeros out the
  86                       * memory for @fault so that the result is '0' on error.
  87                       */
  88                      "pop %2\n\t"
  89                      "pop %1\n\t"
  90                      "3:\n\t"
  91
  92                      /* VMREAD faulted.  As above, except push '1' for @fault. */
  93                      ".pushsection .fixup, \"ax\"\n\t"
  94                      "4: push $1\n\t"
  95                      "push %2\n\t"
  96                      "jmp 2b\n\t"
  97                      ".popsection\n\t"
  98                      _ASM_EXTABLE(1b, 4b)
  99                      : ASM_CALL_CONSTRAINT, "=r"(value) : "r"(field) : "cc");
 100         return value;
 101 }
 102
 103 static __always_inline u16 vmcs_read16(unsigned long field)
 104 {
 105         vmcs_check16(field);
 106         if (static_branch_unlikely(&enable_evmcs))
 107                 return evmcs_read16(field);
 108         return __vmcs_readl(field);
 109 }
 110
 111 static __always_inline u32 vmcs_read32(unsigned long field)
 112 {
 113         vmcs_check32(field);
 114         if (static_branch_unlikely(&enable_evmcs))
 115                 return evmcs_read32(field);
 116         return __vmcs_readl(field);
 117 }
 118
 119 static __always_inline u64 vmcs_read64(unsigned long field)
 120 {
 121         vmcs_check64(field);
 122         if (static_branch_unlikely(&enable_evmcs))
 123                 return evmcs_read64(field);
 124 #ifdef CONFIG_X86_64
 125         return __vmcs_readl(field);
 126 #else
 127         return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
 128 #endif
 129 }
 130
 131 static __always_inline unsigned long vmcs_readl(unsigned long field)
 132 {
 133         vmcs_checkl(field);
 134         if (static_branch_unlikely(&enable_evmcs))
 135                 return evmcs_read64(field);
 136         return __vmcs_readl(field);
 137 }
 138
 139 #define vmx_asm1(insn, op1, error_args...)                              \
 140 do {                                                                    \
 141         asm_volatile_goto("1: " __stringify(insn) " %0\n\t"             \
 142                           ".byte 0x2e\n\t" /* branch not taken hint */  \
 143                           "jna %l[error]\n\t"                           \
 144                           _ASM_EXTABLE(1b, %l[fault])                   \
 145                           : : op1 : "cc" : error, fault);               \
 146         return;                                                         \
 147 error:                                                                  \
 148         insn##_error(error_args);                                       \
 149         return;                                                         \
 150 fault:                                                                  \
 151         kvm_spurious_fault();                                           \
 152 } while (0)
 153
 154 #define vmx_asm2(insn, op1, op2, error_args...)                         \
 155 do {                                                                    \
 156         asm_volatile_goto("1: "  __stringify(insn) " %1, %0\n\t"        \
 157                           ".byte 0x2e\n\t" /* branch not taken hint */  \
 158                           "jna %l[error]\n\t"                           \
 159                           _ASM_EXTABLE(1b, %l[fault])                   \
 160                           : : op1, op2 : "cc" : error, fault);          \
 161         return;                                                         \
 162 error:                                                                  \
 163         insn##_error(error_args);                                       \
 164         return;                                                         \
 165 fault:                                                                  \
 166         kvm_spurious_fault();                                           \
 167 } while (0)
 168
 169 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
 170 {
 171         vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value);
 172 }
 173
 174 static __always_inline void vmcs_write16(unsigned long field, u16 value)
 175 {
 176         vmcs_check16(field);
 177         if (static_branch_unlikely(&enable_evmcs))
 178                 return evmcs_write16(field, value);
 179
 180         __vmcs_writel(field, value);
 181 }
 182
 183 static __always_inline void vmcs_write32(unsigned long field, u32 value)
 184 {
 185         vmcs_check32(field);
 186         if (static_branch_unlikely(&enable_evmcs))
 187                 return evmcs_write32(field, value);
 188
 189         __vmcs_writel(field, value);
 190 }
 191
 192 static __always_inline void vmcs_write64(unsigned long field, u64 value)
 193 {
 194         vmcs_check64(field);
 195         if (static_branch_unlikely(&enable_evmcs))
 196                 return evmcs_write64(field, value);
 197
 198         __vmcs_writel(field, value);
 199 #ifndef CONFIG_X86_64
 200         __vmcs_writel(field+1, value >> 32);
 201 #endif
 202 }
 203
 204 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
 205 {
 206         vmcs_checkl(field);
 207         if (static_branch_unlikely(&enable_evmcs))
 208                 return evmcs_write64(field, value);
 209
 210         __vmcs_writel(field, value);
 211 }
 212
 213 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
 214 {
 215         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
 216                          "vmcs_clear_bits does not support 64-bit fields");
 217         if (static_branch_unlikely(&enable_evmcs))
 218                 return evmcs_write32(field, evmcs_read32(field) & ~mask);
 219
 220         __vmcs_writel(field, __vmcs_readl(field) & ~mask);
 221 }
 222
 223 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
 224 {
 225         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
 226                          "vmcs_set_bits does not support 64-bit fields");
 227         if (static_branch_unlikely(&enable_evmcs))
 228                 return evmcs_write32(field, evmcs_read32(field) | mask);
 229
 230         __vmcs_writel(field, __vmcs_readl(field) | mask);
 231 }
 232
 233 static inline void vmcs_clear(struct vmcs *vmcs)
 234 {
 235         u64 phys_addr = __pa(vmcs);
 236
 237         vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr);
 238 }
 239
 240 static inline void vmcs_load(struct vmcs *vmcs)
 241 {
 242         u64 phys_addr = __pa(vmcs);
 243
 244         if (static_branch_unlikely(&enable_evmcs))
 245                 return evmcs_load(phys_addr);
 246
 247         vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr);
 248 }
 249
 250 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva)
 251 {
 252         struct {
 253                 u64 vpid : 16;
 254                 u64 rsvd : 48;
 255                 u64 gva;
 256         } operand = { vpid, 0, gva };
 257
 258         vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva);
 259 }
 260
 261 static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa)
 262 {
 263         struct {
 264                 u64 eptp, gpa;
 265         } operand = {eptp, gpa};
 266
 267         vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa);
 268 }
 269
 270 static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr)
 271 {
 272         if (vpid == 0)
 273                 return true;
 274
 275         if (cpu_has_vmx_invvpid_individual_addr()) {
 276                 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr);
 277                 return true;
 278         }
 279
 280         return false;
 281 }
 282
 283 static inline void vpid_sync_vcpu_single(int vpid)
 284 {
 285         if (vpid == 0)
 286                 return;
 287
 288         if (cpu_has_vmx_invvpid_single())
 289                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
 290 }
 291
 292 static inline void vpid_sync_vcpu_global(void)
 293 {
 294         if (cpu_has_vmx_invvpid_global())
 295                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
 296 }
 297
 298 static inline void vpid_sync_context(int vpid)
 299 {
 300         if (cpu_has_vmx_invvpid_single())
 301                 vpid_sync_vcpu_single(vpid);
 302         else
 303                 vpid_sync_vcpu_global();
 304 }
 305
 306 static inline void ept_sync_global(void)
 307 {
 308         __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
 309 }
 310
 311 static inline void ept_sync_context(u64 eptp)
 312 {
 313         if (cpu_has_vmx_invept_context())
 314                 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
 315         else
 316                 ept_sync_global();
 317 }
 318
 319 #endif /* __KVM_X86_VMX_INSN_H */