arch/x86/kvm/vmx/ops.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __KVM_X86_VMX_INSN_H
   3 #define __KVM_X86_VMX_INSN_H
   4
   5 #include <linux/nospec.h>
   6
   7 #include <asm/kvm_host.h>
   8 #include <asm/vmx.h>
   9
  10 #include "evmcs.h"
  11 #include "vmcs.h"
  12
  13 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  14
  15 asmlinkage void vmread_error(unsigned long field, bool fault);
  16 void vmwrite_error(unsigned long field, unsigned long value);
  17 void vmclear_error(struct vmcs *vmcs, u64 phys_addr);
  18 void vmptrld_error(struct vmcs *vmcs, u64 phys_addr);
  19 void invvpid_error(unsigned long ext, u16 vpid, gva_t gva);
  20 void invept_error(unsigned long ext, u64 eptp, gpa_t gpa);
  21
  22 static __always_inline void vmcs_check16(unsigned long field)
  23 {
  24         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
  25                          "16-bit accessor invalid for 64-bit field");
  26         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  27                          "16-bit accessor invalid for 64-bit high field");
  28         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  29                          "16-bit accessor invalid for 32-bit high field");
  30         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  31                          "16-bit accessor invalid for natural width field");
  32 }
  33
  34 static __always_inline void vmcs_check32(unsigned long field)
  35 {
  36         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  37                          "32-bit accessor invalid for 16-bit field");
  38         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  39                          "32-bit accessor invalid for natural width field");
  40 }
  41
  42 static __always_inline void vmcs_check64(unsigned long field)
  43 {
  44         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  45                          "64-bit accessor invalid for 16-bit field");
  46         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  47                          "64-bit accessor invalid for 64-bit high field");
  48         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  49                          "64-bit accessor invalid for 32-bit field");
  50         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
  51                          "64-bit accessor invalid for natural width field");
  52 }
  53
  54 static __always_inline void vmcs_checkl(unsigned long field)
  55 {
  56         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
  57                          "Natural width accessor invalid for 16-bit field");
  58         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
  59                          "Natural width accessor invalid for 64-bit field");
  60         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
  61                          "Natural width accessor invalid for 64-bit high field");
  62         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
  63                          "Natural width accessor invalid for 32-bit field");
  64 }
  65
  66 static __always_inline unsigned long __vmcs_readl(unsigned long field)
  67 {
  68         unsigned long value;
  69
  70         asm volatile("1: vmread %2, %1\n\t"
  71                      ".byte 0x3e\n\t" /* branch taken hint */
  72                      "ja 3f\n\t"
  73                      "mov %2, %%" _ASM_ARG1 "\n\t"
  74                      "xor %%" _ASM_ARG2 ", %%" _ASM_ARG2 "\n\t"
  75                      "2: call vmread_error\n\t"
  76                      "xor %k1, %k1\n\t"
  77                      "3:\n\t"
  78
  79                      ".pushsection .fixup, \"ax\"\n\t"
  80                      "4: mov %2, %%" _ASM_ARG1 "\n\t"
  81                      "mov $1, %%" _ASM_ARG2 "\n\t"
  82                      "jmp 2b\n\t"
  83                      ".popsection\n\t"
  84                      _ASM_EXTABLE(1b, 4b)
  85                      : ASM_CALL_CONSTRAINT, "=r"(value) : "r"(field) : "cc");
  86         return value;
  87 }
  88
  89 static __always_inline u16 vmcs_read16(unsigned long field)
  90 {
  91         vmcs_check16(field);
  92         if (static_branch_unlikely(&enable_evmcs))
  93                 return evmcs_read16(field);
  94         return __vmcs_readl(field);
  95 }
  96
  97 static __always_inline u32 vmcs_read32(unsigned long field)
  98 {
  99         vmcs_check32(field);
 100         if (static_branch_unlikely(&enable_evmcs))
 101                 return evmcs_read32(field);
 102         return __vmcs_readl(field);
 103 }
 104
 105 static __always_inline u64 vmcs_read64(unsigned long field)
 106 {
 107         vmcs_check64(field);
 108         if (static_branch_unlikely(&enable_evmcs))
 109                 return evmcs_read64(field);
 110 #ifdef CONFIG_X86_64
 111         return __vmcs_readl(field);
 112 #else
 113         return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
 114 #endif
 115 }
 116
 117 static __always_inline unsigned long vmcs_readl(unsigned long field)
 118 {
 119         vmcs_checkl(field);
 120         if (static_branch_unlikely(&enable_evmcs))
 121                 return evmcs_read64(field);
 122         return __vmcs_readl(field);
 123 }
 124
 125 #define vmx_asm1(insn, op1, error_args...)                              \
 126 do {                                                                    \
 127         asm_volatile_goto("1: " __stringify(insn) " %0\n\t"             \
 128                           ".byte 0x2e\n\t" /* branch not taken hint */  \
 129                           "jna %l[error]\n\t"                           \
 130                           _ASM_EXTABLE(1b, %l[fault])                   \
 131                           : : op1 : "cc" : error, fault);               \
 132         return;                                                         \
 133 error:                                                                  \
 134         insn##_error(error_args);                                       \
 135         return;                                                         \
 136 fault:                                                                  \
 137         kvm_spurious_fault();                                           \
 138 } while (0)
 139
 140 #define vmx_asm2(insn, op1, op2, error_args...)                         \
 141 do {                                                                    \
 142         asm_volatile_goto("1: "  __stringify(insn) " %1, %0\n\t"        \
 143                           ".byte 0x2e\n\t" /* branch not taken hint */  \
 144                           "jna %l[error]\n\t"                           \
 145                           _ASM_EXTABLE(1b, %l[fault])                   \
 146                           : : op1, op2 : "cc" : error, fault);          \
 147         return;                                                         \
 148 error:                                                                  \
 149         insn##_error(error_args);                                       \
 150         return;                                                         \
 151 fault:                                                                  \
 152         kvm_spurious_fault();                                           \
 153 } while (0)
 154
 155 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
 156 {
 157         vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value);
 158 }
 159
 160 static __always_inline void vmcs_write16(unsigned long field, u16 value)
 161 {
 162         vmcs_check16(field);
 163         if (static_branch_unlikely(&enable_evmcs))
 164                 return evmcs_write16(field, value);
 165
 166         __vmcs_writel(field, value);
 167 }
 168
 169 static __always_inline void vmcs_write32(unsigned long field, u32 value)
 170 {
 171         vmcs_check32(field);
 172         if (static_branch_unlikely(&enable_evmcs))
 173                 return evmcs_write32(field, value);
 174
 175         __vmcs_writel(field, value);
 176 }
 177
 178 static __always_inline void vmcs_write64(unsigned long field, u64 value)
 179 {
 180         vmcs_check64(field);
 181         if (static_branch_unlikely(&enable_evmcs))
 182                 return evmcs_write64(field, value);
 183
 184         __vmcs_writel(field, value);
 185 #ifndef CONFIG_X86_64
 186         __vmcs_writel(field+1, value >> 32);
 187 #endif
 188 }
 189
 190 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
 191 {
 192         vmcs_checkl(field);
 193         if (static_branch_unlikely(&enable_evmcs))
 194                 return evmcs_write64(field, value);
 195
 196         __vmcs_writel(field, value);
 197 }
 198
 199 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
 200 {
 201         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
 202                          "vmcs_clear_bits does not support 64-bit fields");
 203         if (static_branch_unlikely(&enable_evmcs))
 204                 return evmcs_write32(field, evmcs_read32(field) & ~mask);
 205
 206         __vmcs_writel(field, __vmcs_readl(field) & ~mask);
 207 }
 208
 209 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
 210 {
 211         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
 212                          "vmcs_set_bits does not support 64-bit fields");
 213         if (static_branch_unlikely(&enable_evmcs))
 214                 return evmcs_write32(field, evmcs_read32(field) | mask);
 215
 216         __vmcs_writel(field, __vmcs_readl(field) | mask);
 217 }
 218
 219 static inline void vmcs_clear(struct vmcs *vmcs)
 220 {
 221         u64 phys_addr = __pa(vmcs);
 222
 223         vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr);
 224 }
 225
 226 static inline void vmcs_load(struct vmcs *vmcs)
 227 {
 228         u64 phys_addr = __pa(vmcs);
 229
 230         if (static_branch_unlikely(&enable_evmcs))
 231                 return evmcs_load(phys_addr);
 232
 233         vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr);
 234 }
 235
 236 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva)
 237 {
 238         struct {
 239                 u64 vpid : 16;
 240                 u64 rsvd : 48;
 241                 u64 gva;
 242         } operand = { vpid, 0, gva };
 243
 244         vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva);
 245 }
 246
 247 static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa)
 248 {
 249         struct {
 250                 u64 eptp, gpa;
 251         } operand = {eptp, gpa};
 252
 253         vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa);
 254 }
 255
 256 static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr)
 257 {
 258         if (vpid == 0)
 259                 return true;
 260
 261         if (cpu_has_vmx_invvpid_individual_addr()) {
 262                 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr);
 263                 return true;
 264         }
 265
 266         return false;
 267 }
 268
 269 static inline void vpid_sync_vcpu_single(int vpid)
 270 {
 271         if (vpid == 0)
 272                 return;
 273
 274         if (cpu_has_vmx_invvpid_single())
 275                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
 276 }
 277
 278 static inline void vpid_sync_vcpu_global(void)
 279 {
 280         if (cpu_has_vmx_invvpid_global())
 281                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
 282 }
 283
 284 static inline void vpid_sync_context(int vpid)
 285 {
 286         if (cpu_has_vmx_invvpid_single())
 287                 vpid_sync_vcpu_single(vpid);
 288         else
 289                 vpid_sync_vcpu_global();
 290 }
 291
 292 static inline void ept_sync_global(void)
 293 {
 294         __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
 295 }
 296
 297 static inline void ept_sync_context(u64 eptp)
 298 {
 299         if (cpu_has_vmx_invept_context())
 300                 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
 301         else
 302                 ept_sync_global();
 303 }
 304
 305 #endif /* __KVM_X86_VMX_INSN_H */