arch/x86/kvm/svm/svm.c

   1 #define pr_fmt(fmt) "SVM: " fmt
   2
   3 #include <linux/kvm_host.h>
   4
   5 #include "irq.h"
   6 #include "mmu.h"
   7 #include "kvm_cache_regs.h"
   8 #include "x86.h"
   9 #include "cpuid.h"
  10 #include "pmu.h"
  11
  12 #include <linux/module.h>
  13 #include <linux/mod_devicetable.h>
  14 #include <linux/kernel.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/highmem.h>
  17 #include <linux/amd-iommu.h>
  18 #include <linux/sched.h>
  19 #include <linux/trace_events.h>
  20 #include <linux/slab.h>
  21 #include <linux/hashtable.h>
  22 #include <linux/objtool.h>
  23 #include <linux/psp-sev.h>
  24 #include <linux/file.h>
  25 #include <linux/pagemap.h>
  26 #include <linux/swap.h>
  27 #include <linux/rwsem.h>
  28 #include <linux/cc_platform.h>
  29
  30 #include <asm/apic.h>
  31 #include <asm/perf_event.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/desc.h>
  34 #include <asm/debugreg.h>
  35 #include <asm/kvm_para.h>
  36 #include <asm/irq_remapping.h>
  37 #include <asm/spec-ctrl.h>
  38 #include <asm/cpu_device_id.h>
  39 #include <asm/traps.h>
  40 #include <asm/fpu/api.h>
  41
  42 #include <asm/virtext.h>
  43 #include "trace.h"
  44
  45 #include "svm.h"
  46 #include "svm_ops.h"
  47
  48 #include "kvm_onhyperv.h"
  49 #include "svm_onhyperv.h"
  50
  51 MODULE_AUTHOR("Qumranet");
  52 MODULE_LICENSE("GPL");
  53
  54 #ifdef MODULE
  55 static const struct x86_cpu_id svm_cpu_id[] = {
  56         X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
  57         {}
  58 };
  59 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  60 #endif
  61
  62 #define SEG_TYPE_LDT 2
  63 #define SEG_TYPE_BUSY_TSS16 3
  64
  65 #define SVM_FEATURE_LBRV           (1 <<  1)
  66 #define SVM_FEATURE_SVML           (1 <<  2)
  67 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  68 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  69 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  70 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  71 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  72
  73 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  74
  75 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  76 #define TSC_RATIO_MIN           0x0000000000000001ULL
  77 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  78
  79 static bool erratum_383_found __read_mostly;
  80
  81 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
  82
  83 /*
  84  * Set osvw_len to higher value when updated Revision Guides
  85  * are published and we know what the new status bits are
  86  */
  87 static uint64_t osvw_len = 4, osvw_status;
  88
  89 static DEFINE_PER_CPU(u64, current_tsc_ratio);
  90 #define TSC_RATIO_DEFAULT       0x0100000000ULL
  91
  92 static const struct svm_direct_access_msrs {
  93         u32 index;   /* Index of the MSR */
  94         bool always; /* True if intercept is initially cleared */
  95 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
  96         { .index = MSR_STAR,                            .always = true  },
  97         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
  98         { .index = MSR_IA32_SYSENTER_EIP,               .always = false },
  99         { .index = MSR_IA32_SYSENTER_ESP,               .always = false },
 100 #ifdef CONFIG_X86_64
 101         { .index = MSR_GS_BASE,                         .always = true  },
 102         { .index = MSR_FS_BASE,                         .always = true  },
 103         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 104         { .index = MSR_LSTAR,                           .always = true  },
 105         { .index = MSR_CSTAR,                           .always = true  },
 106         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 107 #endif
 108         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
 109         { .index = MSR_IA32_PRED_CMD,                   .always = false },
 110         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 111         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 112         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 113         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 114         { .index = MSR_EFER,                            .always = false },
 115         { .index = MSR_IA32_CR_PAT,                     .always = false },
 116         { .index = MSR_AMD64_SEV_ES_GHCB,               .always = true  },
 117         { .index = MSR_INVALID,                         .always = false },
 118 };
 119
 120 /*
 121  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 122  * pause_filter_count: On processors that support Pause filtering(indicated
 123  *      by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
 124  *      count value. On VMRUN this value is loaded into an internal counter.
 125  *      Each time a pause instruction is executed, this counter is decremented
 126  *      until it reaches zero at which time a #VMEXIT is generated if pause
 127  *      intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
 128  *      Intercept Filtering for more details.
 129  *      This also indicate if ple logic enabled.
 130  *
 131  * pause_filter_thresh: In addition, some processor families support advanced
 132  *      pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
 133  *      the amount of time a guest is allowed to execute in a pause loop.
 134  *      In this mode, a 16-bit pause filter threshold field is added in the
 135  *      VMCB. The threshold value is a cycle count that is used to reset the
 136  *      pause counter. As with simple pause filtering, VMRUN loads the pause
 137  *      count value from VMCB into an internal counter. Then, on each pause
 138  *      instruction the hardware checks the elapsed number of cycles since
 139  *      the most recent pause instruction against the pause filter threshold.
 140  *      If the elapsed cycle count is greater than the pause filter threshold,
 141  *      then the internal pause count is reloaded from the VMCB and execution
 142  *      continues. If the elapsed cycle count is less than the pause filter
 143  *      threshold, then the internal pause count is decremented. If the count
 144  *      value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 145  *      triggered. If advanced pause filtering is supported and pause filter
 146  *      threshold field is set to zero, the filter will operate in the simpler,
 147  *      count only mode.
 148  */
 149
 150 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 151 module_param(pause_filter_thresh, ushort, 0444);
 152
 153 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 154 module_param(pause_filter_count, ushort, 0444);
 155
 156 /* Default doubles per-vcpu window every exit. */
 157 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 158 module_param(pause_filter_count_grow, ushort, 0444);
 159
 160 /* Default resets per-vcpu window every exit to pause_filter_count. */
 161 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 162 module_param(pause_filter_count_shrink, ushort, 0444);
 163
 164 /* Default is to compute the maximum so we can never overflow. */
 165 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 166 module_param(pause_filter_count_max, ushort, 0444);
 167
 168 /*
 169  * Use nested page tables by default.  Note, NPT may get forced off by
 170  * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
 171  */
 172 bool npt_enabled = true;
 173 module_param_named(npt, npt_enabled, bool, 0444);
 174
 175 /* allow nested virtualization in KVM/SVM */
 176 static int nested = true;
 177 module_param(nested, int, S_IRUGO);
 178
 179 /* enable/disable Next RIP Save */
 180 static int nrips = true;
 181 module_param(nrips, int, 0444);
 182
 183 /* enable/disable Virtual VMLOAD VMSAVE */
 184 static int vls = true;
 185 module_param(vls, int, 0444);
 186
 187 /* enable/disable Virtual GIF */
 188 static int vgif = true;
 189 module_param(vgif, int, 0444);
 190
 191 /* enable/disable LBR virtualization */
 192 static int lbrv = true;
 193 module_param(lbrv, int, 0444);
 194
 195 static int tsc_scaling = true;
 196 module_param(tsc_scaling, int, 0444);
 197
 198 /*
 199  * enable / disable AVIC.  Because the defaults differ for APICv
 200  * support between VMX and SVM we cannot use module_param_named.
 201  */
 202 static bool avic;
 203 module_param(avic, bool, 0444);
 204
 205 bool __read_mostly dump_invalid_vmcb;
 206 module_param(dump_invalid_vmcb, bool, 0644);
 207
 208
 209 bool intercept_smi = true;
 210 module_param(intercept_smi, bool, 0444);
 211
 212
 213 static bool svm_gp_erratum_intercept = true;
 214
 215 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 216
 217 static unsigned long iopm_base;
 218
 219 struct kvm_ldttss_desc {
 220         u16 limit0;
 221         u16 base0;
 222         unsigned base1:8, type:5, dpl:2, p:1;
 223         unsigned limit1:4, zero0:3, g:1, base2:8;
 224         u32 base3;
 225         u32 zero1;
 226 } __attribute__((packed));
 227
 228 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 229
 230 /*
 231  * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
 232  * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
 233  *
 234  * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
 235  * defer the restoration of TSC_AUX until the CPU returns to userspace.
 236  */
 237 static int tsc_aux_uret_slot __read_mostly = -1;
 238
 239 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 240
 241 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 242 #define MSRS_RANGE_SIZE 2048
 243 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 244
 245 u32 svm_msrpm_offset(u32 msr)
 246 {
 247         u32 offset;
 248         int i;
 249
 250         for (i = 0; i < NUM_MSR_MAPS; i++) {
 251                 if (msr < msrpm_ranges[i] ||
 252                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 253                         continue;
 254
 255                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 256                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 257
 258                 /* Now we have the u8 offset - but need the u32 offset */
 259                 return offset / 4;
 260         }
 261
 262         /* MSR not in any range */
 263         return MSR_INVALID;
 264 }
 265
 266 #define MAX_INST_SIZE 15
 267
 268 static int get_max_npt_level(void)
 269 {
 270 #ifdef CONFIG_X86_64
 271         return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
 272 #else
 273         return PT32E_ROOT_LEVEL;
 274 #endif
 275 }
 276
 277 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 278 {
 279         struct vcpu_svm *svm = to_svm(vcpu);
 280         u64 old_efer = vcpu->arch.efer;
 281         vcpu->arch.efer = efer;
 282
 283         if (!npt_enabled) {
 284                 /* Shadow paging assumes NX to be available.  */
 285                 efer |= EFER_NX;
 286
 287                 if (!(efer & EFER_LMA))
 288                         efer &= ~EFER_LME;
 289         }
 290
 291         if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
 292                 if (!(efer & EFER_SVME)) {
 293                         svm_leave_nested(svm);
 294                         svm_set_gif(svm, true);
 295                         /* #GP intercept is still needed for vmware backdoor */
 296                         if (!enable_vmware_backdoor)
 297                                 clr_exception_intercept(svm, GP_VECTOR);
 298
 299                         /*
 300                          * Free the nested guest state, unless we are in SMM.
 301                          * In this case we will return to the nested guest
 302                          * as soon as we leave SMM.
 303                          */
 304                         if (!is_smm(vcpu))
 305                                 svm_free_nested(svm);
 306
 307                 } else {
 308                         int ret = svm_allocate_nested(svm);
 309
 310                         if (ret) {
 311                                 vcpu->arch.efer = old_efer;
 312                                 return ret;
 313                         }
 314
 315                         if (svm_gp_erratum_intercept)
 316                                 set_exception_intercept(svm, GP_VECTOR);
 317                 }
 318         }
 319
 320         svm->vmcb->save.efer = efer | EFER_SVME;
 321         vmcb_mark_dirty(svm->vmcb, VMCB_CR);
 322         return 0;
 323 }
 324
 325 static int is_external_interrupt(u32 info)
 326 {
 327         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 328         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 329 }
 330
 331 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 332 {
 333         struct vcpu_svm *svm = to_svm(vcpu);
 334         u32 ret = 0;
 335
 336         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 337                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 338         return ret;
 339 }
 340
 341 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 342 {
 343         struct vcpu_svm *svm = to_svm(vcpu);
 344
 345         if (mask == 0)
 346                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 347         else
 348                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 349
 350 }
 351
 352 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
 353 {
 354         struct vcpu_svm *svm = to_svm(vcpu);
 355
 356         /*
 357          * SEV-ES does not expose the next RIP. The RIP update is controlled by
 358          * the type of exit and the #VC handler in the guest.
 359          */
 360         if (sev_es_guest(vcpu->kvm))
 361                 goto done;
 362
 363         if (nrips && svm->vmcb->control.next_rip != 0) {
 364                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 365                 svm->next_rip = svm->vmcb->control.next_rip;
 366         }
 367
 368         if (!svm->next_rip) {
 369                 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
 370                         return 0;
 371         } else {
 372                 kvm_rip_write(vcpu, svm->next_rip);
 373         }
 374
 375 done:
 376         svm_set_interrupt_shadow(vcpu, 0);
 377
 378         return 1;
 379 }
 380
 381 static void svm_queue_exception(struct kvm_vcpu *vcpu)
 382 {
 383         struct vcpu_svm *svm = to_svm(vcpu);
 384         unsigned nr = vcpu->arch.exception.nr;
 385         bool has_error_code = vcpu->arch.exception.has_error_code;
 386         u32 error_code = vcpu->arch.exception.error_code;
 387
 388         kvm_deliver_exception_payload(vcpu);
 389
 390         if (nr == BP_VECTOR && !nrips) {
 391                 unsigned long rip, old_rip = kvm_rip_read(vcpu);
 392
 393                 /*
 394                  * For guest debugging where we have to reinject #BP if some
 395                  * INT3 is guest-owned:
 396                  * Emulate nRIP by moving RIP forward. Will fail if injection
 397                  * raises a fault that is not intercepted. Still better than
 398                  * failing in all cases.
 399                  */
 400                 (void)skip_emulated_instruction(vcpu);
 401                 rip = kvm_rip_read(vcpu);
 402                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 403                 svm->int3_injected = rip - old_rip;
 404         }
 405
 406         svm->vmcb->control.event_inj = nr
 407                 | SVM_EVTINJ_VALID
 408                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 409                 | SVM_EVTINJ_TYPE_EXEPT;
 410         svm->vmcb->control.event_inj_err = error_code;
 411 }
 412
 413 static void svm_init_erratum_383(void)
 414 {
 415         u32 low, high;
 416         int err;
 417         u64 val;
 418
 419         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 420                 return;
 421
 422         /* Use _safe variants to not break nested virtualization */
 423         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 424         if (err)
 425                 return;
 426
 427         val |= (1ULL << 47);
 428
 429         low  = lower_32_bits(val);
 430         high = upper_32_bits(val);
 431
 432         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 433
 434         erratum_383_found = true;
 435 }
 436
 437 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 438 {
 439         /*
 440          * Guests should see errata 400 and 415 as fixed (assuming that
 441          * HLT and IO instructions are intercepted).
 442          */
 443         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 444         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 445
 446         /*
 447          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 448          * all osvw.status bits inside that length, including bit 0 (which is
 449          * reserved for erratum 298), are valid. However, if host processor's
 450          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 451          * be conservative here and therefore we tell the guest that erratum 298
 452          * is present (because we really don't know).
 453          */
 454         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 455                 vcpu->arch.osvw.status |= 1;
 456 }
 457
 458 static int has_svm(void)
 459 {
 460         const char *msg;
 461
 462         if (!cpu_has_svm(&msg)) {
 463                 printk(KERN_INFO "has_svm: %s\n", msg);
 464                 return 0;
 465         }
 466
 467         if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
 468                 pr_info("KVM is unsupported when running as an SEV guest\n");
 469                 return 0;
 470         }
 471
 472         return 1;
 473 }
 474
 475 static void svm_hardware_disable(void)
 476 {
 477         /* Make sure we clean up behind us */
 478         if (tsc_scaling)
 479                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 480
 481         cpu_svm_disable();
 482
 483         amd_pmu_disable_virt();
 484 }
 485
 486 static int svm_hardware_enable(void)
 487 {
 488
 489         struct svm_cpu_data *sd;
 490         uint64_t efer;
 491         struct desc_struct *gdt;
 492         int me = raw_smp_processor_id();
 493
 494         rdmsrl(MSR_EFER, efer);
 495         if (efer & EFER_SVME)
 496                 return -EBUSY;
 497
 498         if (!has_svm()) {
 499                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 500                 return -EINVAL;
 501         }
 502         sd = per_cpu(svm_data, me);
 503         if (!sd) {
 504                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 505                 return -EINVAL;
 506         }
 507
 508         sd->asid_generation = 1;
 509         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 510         sd->next_asid = sd->max_asid + 1;
 511         sd->min_asid = max_sev_asid + 1;
 512
 513         gdt = get_current_gdt_rw();
 514         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 515
 516         wrmsrl(MSR_EFER, efer | EFER_SVME);
 517
 518         wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area));
 519
 520         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 521                 /*
 522                  * Set the default value, even if we don't use TSC scaling
 523                  * to avoid having stale value in the msr
 524                  */
 525                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 526                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 527         }
 528
 529
 530         /*
 531          * Get OSVW bits.
 532          *
 533          * Note that it is possible to have a system with mixed processor
 534          * revisions and therefore different OSVW bits. If bits are not the same
 535          * on different processors then choose the worst case (i.e. if erratum
 536          * is present on one processor and not on another then assume that the
 537          * erratum is present everywhere).
 538          */
 539         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 540                 uint64_t len, status = 0;
 541                 int err;
 542
 543                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 544                 if (!err)
 545                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 546                                                       &err);
 547
 548                 if (err)
 549                         osvw_status = osvw_len = 0;
 550                 else {
 551                         if (len < osvw_len)
 552                                 osvw_len = len;
 553                         osvw_status |= status;
 554                         osvw_status &= (1ULL << osvw_len) - 1;
 555                 }
 556         } else
 557                 osvw_status = osvw_len = 0;
 558
 559         svm_init_erratum_383();
 560
 561         amd_pmu_enable_virt();
 562
 563         return 0;
 564 }
 565
 566 static void svm_cpu_uninit(int cpu)
 567 {
 568         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
 569
 570         if (!sd)
 571                 return;
 572
 573         per_cpu(svm_data, cpu) = NULL;
 574         kfree(sd->sev_vmcbs);
 575         __free_page(sd->save_area);
 576         kfree(sd);
 577 }
 578
 579 static int svm_cpu_init(int cpu)
 580 {
 581         struct svm_cpu_data *sd;
 582         int ret = -ENOMEM;
 583
 584         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 585         if (!sd)
 586                 return ret;
 587         sd->cpu = cpu;
 588         sd->save_area = alloc_page(GFP_KERNEL);
 589         if (!sd->save_area)
 590                 goto free_cpu_data;
 591
 592         clear_page(page_address(sd->save_area));
 593
 594         ret = sev_cpu_init(sd);
 595         if (ret)
 596                 goto free_save_area;
 597
 598         per_cpu(svm_data, cpu) = sd;
 599
 600         return 0;
 601
 602 free_save_area:
 603         __free_page(sd->save_area);
 604 free_cpu_data:
 605         kfree(sd);
 606         return ret;
 607
 608 }
 609
 610 static int direct_access_msr_slot(u32 msr)
 611 {
 612         u32 i;
 613
 614         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 615                 if (direct_access_msrs[i].index == msr)
 616                         return i;
 617
 618         return -ENOENT;
 619 }
 620
 621 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
 622                                      int write)
 623 {
 624         struct vcpu_svm *svm = to_svm(vcpu);
 625         int slot = direct_access_msr_slot(msr);
 626
 627         if (slot == -ENOENT)
 628                 return;
 629
 630         /* Set the shadow bitmaps to the desired intercept states */
 631         if (read)
 632                 set_bit(slot, svm->shadow_msr_intercept.read);
 633         else
 634                 clear_bit(slot, svm->shadow_msr_intercept.read);
 635
 636         if (write)
 637                 set_bit(slot, svm->shadow_msr_intercept.write);
 638         else
 639                 clear_bit(slot, svm->shadow_msr_intercept.write);
 640 }
 641
 642 static bool valid_msr_intercept(u32 index)
 643 {
 644         return direct_access_msr_slot(index) != -ENOENT;
 645 }
 646
 647 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
 648 {
 649         u8 bit_write;
 650         unsigned long tmp;
 651         u32 offset;
 652         u32 *msrpm;
 653
 654         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
 655                                       to_svm(vcpu)->msrpm;
 656
 657         offset    = svm_msrpm_offset(msr);
 658         bit_write = 2 * (msr & 0x0f) + 1;
 659         tmp       = msrpm[offset];
 660
 661         BUG_ON(offset == MSR_INVALID);
 662
 663         return !!test_bit(bit_write,  &tmp);
 664 }
 665
 666 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
 667                                         u32 msr, int read, int write)
 668 {
 669         u8 bit_read, bit_write;
 670         unsigned long tmp;
 671         u32 offset;
 672
 673         /*
 674          * If this warning triggers extend the direct_access_msrs list at the
 675          * beginning of the file
 676          */
 677         WARN_ON(!valid_msr_intercept(msr));
 678
 679         /* Enforce non allowed MSRs to trap */
 680         if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
 681                 read = 0;
 682
 683         if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
 684                 write = 0;
 685
 686         offset    = svm_msrpm_offset(msr);
 687         bit_read  = 2 * (msr & 0x0f);
 688         bit_write = 2 * (msr & 0x0f) + 1;
 689         tmp       = msrpm[offset];
 690
 691         BUG_ON(offset == MSR_INVALID);
 692
 693         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 694         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 695
 696         msrpm[offset] = tmp;
 697
 698         svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
 699
 700 }
 701
 702 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
 703                           int read, int write)
 704 {
 705         set_shadow_msr_intercept(vcpu, msr, read, write);
 706         set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
 707 }
 708
 709 u32 *svm_vcpu_alloc_msrpm(void)
 710 {
 711         unsigned int order = get_order(MSRPM_SIZE);
 712         struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
 713         u32 *msrpm;
 714
 715         if (!pages)
 716                 return NULL;
 717
 718         msrpm = page_address(pages);
 719         memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
 720
 721         return msrpm;
 722 }
 723
 724 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
 725 {
 726         int i;
 727
 728         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 729                 if (!direct_access_msrs[i].always)
 730                         continue;
 731                 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
 732         }
 733 }
 734
 735
 736 void svm_vcpu_free_msrpm(u32 *msrpm)
 737 {
 738         __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
 739 }
 740
 741 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
 742 {
 743         struct vcpu_svm *svm = to_svm(vcpu);
 744         u32 i;
 745
 746         /*
 747          * Set intercept permissions for all direct access MSRs again. They
 748          * will automatically get filtered through the MSR filter, so we are
 749          * back in sync after this.
 750          */
 751         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 752                 u32 msr = direct_access_msrs[i].index;
 753                 u32 read = test_bit(i, svm->shadow_msr_intercept.read);
 754                 u32 write = test_bit(i, svm->shadow_msr_intercept.write);
 755
 756                 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
 757         }
 758 }
 759
 760 static void add_msr_offset(u32 offset)
 761 {
 762         int i;
 763
 764         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 765
 766                 /* Offset already in list? */
 767                 if (msrpm_offsets[i] == offset)
 768                         return;
 769
 770                 /* Slot used by another offset? */
 771                 if (msrpm_offsets[i] != MSR_INVALID)
 772                         continue;
 773
 774                 /* Add offset to list */
 775                 msrpm_offsets[i] = offset;
 776
 777                 return;
 778         }
 779
 780         /*
 781          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 782          * increase MSRPM_OFFSETS in this case.
 783          */
 784         BUG();
 785 }
 786
 787 static void init_msrpm_offsets(void)
 788 {
 789         int i;
 790
 791         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 792
 793         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 794                 u32 offset;
 795
 796                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 797                 BUG_ON(offset == MSR_INVALID);
 798
 799                 add_msr_offset(offset);
 800         }
 801 }
 802
 803 static void svm_enable_lbrv(struct kvm_vcpu *vcpu)
 804 {
 805         struct vcpu_svm *svm = to_svm(vcpu);
 806
 807         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
 808         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 809         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 810         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 811         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 812 }
 813
 814 static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
 815 {
 816         struct vcpu_svm *svm = to_svm(vcpu);
 817
 818         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
 819         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 820         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 821         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 822         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 823 }
 824
 825 void disable_nmi_singlestep(struct vcpu_svm *svm)
 826 {
 827         svm->nmi_singlestep = false;
 828
 829         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
 830                 /* Clear our flags if they were not set by the guest */
 831                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
 832                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
 833                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
 834                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
 835         }
 836 }
 837
 838 static void grow_ple_window(struct kvm_vcpu *vcpu)
 839 {
 840         struct vcpu_svm *svm = to_svm(vcpu);
 841         struct vmcb_control_area *control = &svm->vmcb->control;
 842         int old = control->pause_filter_count;
 843
 844         control->pause_filter_count = __grow_ple_window(old,
 845                                                         pause_filter_count,
 846                                                         pause_filter_count_grow,
 847                                                         pause_filter_count_max);
 848
 849         if (control->pause_filter_count != old) {
 850                 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 851                 trace_kvm_ple_window_update(vcpu->vcpu_id,
 852                                             control->pause_filter_count, old);
 853         }
 854 }
 855
 856 static void shrink_ple_window(struct kvm_vcpu *vcpu)
 857 {
 858         struct vcpu_svm *svm = to_svm(vcpu);
 859         struct vmcb_control_area *control = &svm->vmcb->control;
 860         int old = control->pause_filter_count;
 861
 862         control->pause_filter_count =
 863                                 __shrink_ple_window(old,
 864                                                     pause_filter_count,
 865                                                     pause_filter_count_shrink,
 866                                                     pause_filter_count);
 867         if (control->pause_filter_count != old) {
 868                 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 869                 trace_kvm_ple_window_update(vcpu->vcpu_id,
 870                                             control->pause_filter_count, old);
 871         }
 872 }
 873
 874 /*
 875  * The default MMIO mask is a single bit (excluding the present bit),
 876  * which could conflict with the memory encryption bit. Check for
 877  * memory encryption support and override the default MMIO mask if
 878  * memory encryption is enabled.
 879  */
 880 static __init void svm_adjust_mmio_mask(void)
 881 {
 882         unsigned int enc_bit, mask_bit;
 883         u64 msr, mask;
 884
 885         /* If there is no memory encryption support, use existing mask */
 886         if (cpuid_eax(0x80000000) < 0x8000001f)
 887                 return;
 888
 889         /* If memory encryption is not enabled, use existing mask */
 890         rdmsrl(MSR_AMD64_SYSCFG, msr);
 891         if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
 892                 return;
 893
 894         enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
 895         mask_bit = boot_cpu_data.x86_phys_bits;
 896
 897         /* Increment the mask bit if it is the same as the encryption bit */
 898         if (enc_bit == mask_bit)
 899                 mask_bit++;
 900
 901         /*
 902          * If the mask bit location is below 52, then some bits above the
 903          * physical addressing limit will always be reserved, so use the
 904          * rsvd_bits() function to generate the mask. This mask, along with
 905          * the present bit, will be used to generate a page fault with
 906          * PFER.RSV = 1.
 907          *
 908          * If the mask bit location is 52 (or above), then clear the mask.
 909          */
 910         mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
 911
 912         kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
 913 }
 914
 915 static void svm_hardware_teardown(void)
 916 {
 917         int cpu;
 918
 919         sev_hardware_teardown();
 920
 921         for_each_possible_cpu(cpu)
 922                 svm_cpu_uninit(cpu);
 923
 924         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT),
 925         get_order(IOPM_SIZE));
 926         iopm_base = 0;
 927 }
 928
 929 static __init void svm_set_cpu_caps(void)
 930 {
 931         kvm_set_cpu_caps();
 932
 933         supported_xss = 0;
 934
 935         /* CPUID 0x80000001 and 0x8000000A (SVM features) */
 936         if (nested) {
 937                 kvm_cpu_cap_set(X86_FEATURE_SVM);
 938
 939                 if (nrips)
 940                         kvm_cpu_cap_set(X86_FEATURE_NRIPS);
 941
 942                 if (npt_enabled)
 943                         kvm_cpu_cap_set(X86_FEATURE_NPT);
 944
 945                 if (tsc_scaling)
 946                         kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
 947
 948                 /* Nested VM can receive #VMEXIT instead of triggering #GP */
 949                 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
 950         }
 951
 952         /* CPUID 0x80000008 */
 953         if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
 954             boot_cpu_has(X86_FEATURE_AMD_SSBD))
 955                 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
 956
 957         /* CPUID 0x8000001F (SME/SEV features) */
 958         sev_set_cpu_caps();
 959 }
 960
 961 static __init int svm_hardware_setup(void)
 962 {
 963         int cpu;
 964         struct page *iopm_pages;
 965         void *iopm_va;
 966         int r;
 967         unsigned int order = get_order(IOPM_SIZE);
 968
 969         /*
 970          * NX is required for shadow paging and for NPT if the NX huge pages
 971          * mitigation is enabled.
 972          */
 973         if (!boot_cpu_has(X86_FEATURE_NX)) {
 974                 pr_err_ratelimited("NX (Execute Disable) not supported\n");
 975                 return -EOPNOTSUPP;
 976         }
 977         kvm_enable_efer_bits(EFER_NX);
 978
 979         iopm_pages = alloc_pages(GFP_KERNEL, order);
 980
 981         if (!iopm_pages)
 982                 return -ENOMEM;
 983
 984         iopm_va = page_address(iopm_pages);
 985         memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
 986         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 987
 988         init_msrpm_offsets();
 989
 990         supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
 991
 992         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 993                 kvm_enable_efer_bits(EFER_FFXSR);
 994
 995         if (tsc_scaling) {
 996                 if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 997                         tsc_scaling = false;
 998                 } else {
 999                         pr_info("TSC scaling supported\n");
1000                         kvm_has_tsc_control = true;
1001                         kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1002                         kvm_tsc_scaling_ratio_frac_bits = 32;
1003                 }
1004         }
1005
1006         tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
1007
1008         /* Check for pause filtering support */
1009         if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1010                 pause_filter_count = 0;
1011                 pause_filter_thresh = 0;
1012         } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1013                 pause_filter_thresh = 0;
1014         }
1015
1016         if (nested) {
1017                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1018                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1019         }
1020
1021         /*
1022          * KVM's MMU doesn't support using 2-level paging for itself, and thus
1023          * NPT isn't supported if the host is using 2-level paging since host
1024          * CR4 is unchanged on VMRUN.
1025          */
1026         if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
1027                 npt_enabled = false;
1028
1029         if (!boot_cpu_has(X86_FEATURE_NPT))
1030                 npt_enabled = false;
1031
1032         /* Force VM NPT level equal to the host's max NPT level */
1033         kvm_configure_mmu(npt_enabled, get_max_npt_level(),
1034                           get_max_npt_level(), PG_LEVEL_1G);
1035         pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
1036
1037         /* Note, SEV setup consumes npt_enabled. */
1038         sev_hardware_setup();
1039
1040         svm_hv_hardware_setup();
1041
1042         svm_adjust_mmio_mask();
1043
1044         for_each_possible_cpu(cpu) {
1045                 r = svm_cpu_init(cpu);
1046                 if (r)
1047                         goto err;
1048         }
1049
1050         if (nrips) {
1051                 if (!boot_cpu_has(X86_FEATURE_NRIPS))
1052                         nrips = false;
1053         }
1054
1055         enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC);
1056
1057         if (enable_apicv) {
1058                 pr_info("AVIC enabled\n");
1059
1060                 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1061         }
1062
1063         if (vls) {
1064                 if (!npt_enabled ||
1065                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1066                     !IS_ENABLED(CONFIG_X86_64)) {
1067                         vls = false;
1068                 } else {
1069                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1070                 }
1071         }
1072
1073         if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
1074                 svm_gp_erratum_intercept = false;
1075
1076         if (vgif) {
1077                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1078                         vgif = false;
1079                 else
1080                         pr_info("Virtual GIF supported\n");
1081         }
1082
1083         if (lbrv) {
1084                 if (!boot_cpu_has(X86_FEATURE_LBRV))
1085                         lbrv = false;
1086                 else
1087                         pr_info("LBR virtualization supported\n");
1088         }
1089
1090         svm_set_cpu_caps();
1091
1092         /*
1093          * It seems that on AMD processors PTE's accessed bit is
1094          * being set by the CPU hardware before the NPF vmexit.
1095          * This is not expected behaviour and our tests fail because
1096          * of it.
1097          * A workaround here is to disable support for
1098          * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
1099          * In this case userspace can know if there is support using
1100          * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
1101          * it
1102          * If future AMD CPU models change the behaviour described above,
1103          * this variable can be changed accordingly
1104          */
1105         allow_smaller_maxphyaddr = !npt_enabled;
1106
1107         return 0;
1108
1109 err:
1110         svm_hardware_teardown();
1111         return r;
1112 }
1113
1114 static void init_seg(struct vmcb_seg *seg)
1115 {
1116         seg->selector = 0;
1117         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1118                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1119         seg->limit = 0xffff;
1120         seg->base = 0;
1121 }
1122
1123 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1124 {
1125         seg->selector = 0;
1126         seg->attrib = SVM_SELECTOR_P_MASK | type;
1127         seg->limit = 0xffff;
1128         seg->base = 0;
1129 }
1130
1131 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1132 {
1133         struct vcpu_svm *svm = to_svm(vcpu);
1134
1135         return svm->nested.ctl.tsc_offset;
1136 }
1137
1138 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1139 {
1140         struct vcpu_svm *svm = to_svm(vcpu);
1141
1142         return svm->tsc_ratio_msr;
1143 }
1144
1145 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1146 {
1147         struct vcpu_svm *svm = to_svm(vcpu);
1148
1149         svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1150         svm->vmcb->control.tsc_offset = offset;
1151         vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1152 }
1153
1154 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier)
1155 {
1156         wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
1157 }
1158
1159 /* Evaluate instruction intercepts that depend on guest CPUID features. */
1160 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1161                                               struct vcpu_svm *svm)
1162 {
1163         /*
1164          * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1165          * roots, or if INVPCID is disabled in the guest to inject #UD.
1166          */
1167         if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1168                 if (!npt_enabled ||
1169                     !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1170                         svm_set_intercept(svm, INTERCEPT_INVPCID);
1171                 else
1172                         svm_clr_intercept(svm, INTERCEPT_INVPCID);
1173         }
1174
1175         if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1176                 if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1177                         svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1178                 else
1179                         svm_set_intercept(svm, INTERCEPT_RDTSCP);
1180         }
1181 }
1182
1183 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1184 {
1185         struct vcpu_svm *svm = to_svm(vcpu);
1186
1187         if (guest_cpuid_is_intel(vcpu)) {
1188                 /*
1189                  * We must intercept SYSENTER_EIP and SYSENTER_ESP
1190                  * accesses because the processor only stores 32 bits.
1191                  * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1192                  */
1193                 svm_set_intercept(svm, INTERCEPT_VMLOAD);
1194                 svm_set_intercept(svm, INTERCEPT_VMSAVE);
1195                 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1196
1197                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1198                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1199         } else {
1200                 /*
1201                  * If hardware supports Virtual VMLOAD VMSAVE then enable it
1202                  * in VMCB and clear intercepts to avoid #VMEXIT.
1203                  */
1204                 if (vls) {
1205                         svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1206                         svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1207                         svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1208                 }
1209                 /* No need to intercept these MSRs */
1210                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1211                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1212         }
1213 }
1214
1215 static void init_vmcb(struct kvm_vcpu *vcpu)
1216 {
1217         struct vcpu_svm *svm = to_svm(vcpu);
1218         struct vmcb_control_area *control = &svm->vmcb->control;
1219         struct vmcb_save_area *save = &svm->vmcb->save;
1220
1221         svm_set_intercept(svm, INTERCEPT_CR0_READ);
1222         svm_set_intercept(svm, INTERCEPT_CR3_READ);
1223         svm_set_intercept(svm, INTERCEPT_CR4_READ);
1224         svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1225         svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1226         svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1227         if (!kvm_vcpu_apicv_active(vcpu))
1228                 svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1229
1230         set_dr_intercepts(svm);
1231
1232         set_exception_intercept(svm, PF_VECTOR);
1233         set_exception_intercept(svm, UD_VECTOR);
1234         set_exception_intercept(svm, MC_VECTOR);
1235         set_exception_intercept(svm, AC_VECTOR);
1236         set_exception_intercept(svm, DB_VECTOR);
1237         /*
1238          * Guest access to VMware backdoor ports could legitimately
1239          * trigger #GP because of TSS I/O permission bitmap.
1240          * We intercept those #GP and allow access to them anyway
1241          * as VMware does.
1242          */
1243         if (enable_vmware_backdoor)
1244                 set_exception_intercept(svm, GP_VECTOR);
1245
1246         svm_set_intercept(svm, INTERCEPT_INTR);
1247         svm_set_intercept(svm, INTERCEPT_NMI);
1248
1249         if (intercept_smi)
1250                 svm_set_intercept(svm, INTERCEPT_SMI);
1251
1252         svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1253         svm_set_intercept(svm, INTERCEPT_RDPMC);
1254         svm_set_intercept(svm, INTERCEPT_CPUID);
1255         svm_set_intercept(svm, INTERCEPT_INVD);
1256         svm_set_intercept(svm, INTERCEPT_INVLPG);
1257         svm_set_intercept(svm, INTERCEPT_INVLPGA);
1258         svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1259         svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1260         svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1261         svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1262         svm_set_intercept(svm, INTERCEPT_VMRUN);
1263         svm_set_intercept(svm, INTERCEPT_VMMCALL);
1264         svm_set_intercept(svm, INTERCEPT_VMLOAD);
1265         svm_set_intercept(svm, INTERCEPT_VMSAVE);
1266         svm_set_intercept(svm, INTERCEPT_STGI);
1267         svm_set_intercept(svm, INTERCEPT_CLGI);
1268         svm_set_intercept(svm, INTERCEPT_SKINIT);
1269         svm_set_intercept(svm, INTERCEPT_WBINVD);
1270         svm_set_intercept(svm, INTERCEPT_XSETBV);
1271         svm_set_intercept(svm, INTERCEPT_RDPRU);
1272         svm_set_intercept(svm, INTERCEPT_RSM);
1273
1274         if (!kvm_mwait_in_guest(vcpu->kvm)) {
1275                 svm_set_intercept(svm, INTERCEPT_MONITOR);
1276                 svm_set_intercept(svm, INTERCEPT_MWAIT);
1277         }
1278
1279         if (!kvm_hlt_in_guest(vcpu->kvm))
1280                 svm_set_intercept(svm, INTERCEPT_HLT);
1281
1282         control->iopm_base_pa = __sme_set(iopm_base);
1283         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1284         control->int_ctl = V_INTR_MASKING_MASK;
1285
1286         init_seg(&save->es);
1287         init_seg(&save->ss);
1288         init_seg(&save->ds);
1289         init_seg(&save->fs);
1290         init_seg(&save->gs);
1291
1292         save->cs.selector = 0xf000;
1293         save->cs.base = 0xffff0000;
1294         /* Executable/Readable Code Segment */
1295         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1296                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1297         save->cs.limit = 0xffff;
1298
1299         save->gdtr.base = 0;
1300         save->gdtr.limit = 0xffff;
1301         save->idtr.base = 0;
1302         save->idtr.limit = 0xffff;
1303
1304         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1305         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1306
1307         if (npt_enabled) {
1308                 /* Setup VMCB for Nested Paging */
1309                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1310                 svm_clr_intercept(svm, INTERCEPT_INVLPG);
1311                 clr_exception_intercept(svm, PF_VECTOR);
1312                 svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1313                 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1314                 save->g_pat = vcpu->arch.pat;
1315                 save->cr3 = 0;
1316         }
1317         svm->current_vmcb->asid_generation = 0;
1318         svm->asid = 0;
1319
1320         svm->nested.vmcb12_gpa = INVALID_GPA;
1321         svm->nested.last_vmcb12_gpa = INVALID_GPA;
1322
1323         if (!kvm_pause_in_guest(vcpu->kvm)) {
1324                 control->pause_filter_count = pause_filter_count;
1325                 if (pause_filter_thresh)
1326                         control->pause_filter_thresh = pause_filter_thresh;
1327                 svm_set_intercept(svm, INTERCEPT_PAUSE);
1328         } else {
1329                 svm_clr_intercept(svm, INTERCEPT_PAUSE);
1330         }
1331
1332         svm_recalc_instruction_intercepts(vcpu, svm);
1333
1334         /*
1335          * If the host supports V_SPEC_CTRL then disable the interception
1336          * of MSR_IA32_SPEC_CTRL.
1337          */
1338         if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1339                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1340
1341         if (kvm_vcpu_apicv_active(vcpu))
1342                 avic_init_vmcb(svm);
1343
1344         if (vgif) {
1345                 svm_clr_intercept(svm, INTERCEPT_STGI);
1346                 svm_clr_intercept(svm, INTERCEPT_CLGI);
1347                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1348         }
1349
1350         if (sev_guest(vcpu->kvm)) {
1351                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1352                 clr_exception_intercept(svm, UD_VECTOR);
1353
1354                 if (sev_es_guest(vcpu->kvm)) {
1355                         /* Perform SEV-ES specific VMCB updates */
1356                         sev_es_init_vmcb(svm);
1357                 }
1358         }
1359
1360         svm_hv_init_vmcb(svm->vmcb);
1361         init_vmcb_after_set_cpuid(vcpu);
1362
1363         vmcb_mark_all_dirty(svm->vmcb);
1364
1365         enable_gif(svm);
1366 }
1367
1368 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1369 {
1370         struct vcpu_svm *svm = to_svm(vcpu);
1371
1372         svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1373
1374         svm_init_osvw(vcpu);
1375         vcpu->arch.microcode_version = 0x01000065;
1376         svm->tsc_ratio_msr = kvm_default_tsc_scaling_ratio;
1377
1378         if (sev_es_guest(vcpu->kvm))
1379                 sev_es_vcpu_reset(svm);
1380 }
1381
1382 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1383 {
1384         struct vcpu_svm *svm = to_svm(vcpu);
1385
1386         svm->spec_ctrl = 0;
1387         svm->virt_spec_ctrl = 0;
1388
1389         init_vmcb(vcpu);
1390
1391         if (!init_event)
1392                 __svm_vcpu_reset(vcpu);
1393 }
1394
1395 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1396 {
1397         svm->current_vmcb = target_vmcb;
1398         svm->vmcb = target_vmcb->ptr;
1399 }
1400
1401 static int svm_create_vcpu(struct kvm_vcpu *vcpu)
1402 {
1403         struct vcpu_svm *svm;
1404         struct page *vmcb01_page;
1405         struct page *vmsa_page = NULL;
1406         int err;
1407
1408         BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1409         svm = to_svm(vcpu);
1410
1411         err = -ENOMEM;
1412         vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1413         if (!vmcb01_page)
1414                 goto out;
1415
1416         if (sev_es_guest(vcpu->kvm)) {
1417                 /*
1418                  * SEV-ES guests require a separate VMSA page used to contain
1419                  * the encrypted register state of the guest.
1420                  */
1421                 vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1422                 if (!vmsa_page)
1423                         goto error_free_vmcb_page;
1424
1425                 /*
1426                  * SEV-ES guests maintain an encrypted version of their FPU
1427                  * state which is restored and saved on VMRUN and VMEXIT.
1428                  * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't
1429                  * do xsave/xrstor on it.
1430                  */
1431                 fpstate_set_confidential(&vcpu->arch.guest_fpu);
1432         }
1433
1434         err = avic_init_vcpu(svm);
1435         if (err)
1436                 goto error_free_vmsa_page;
1437
1438         /* We initialize this flag to true to make sure that the is_running
1439          * bit would be set the first time the vcpu is loaded.
1440          */
1441         if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm))
1442                 svm->avic_is_running = true;
1443
1444         svm->msrpm = svm_vcpu_alloc_msrpm();
1445         if (!svm->msrpm) {
1446                 err = -ENOMEM;
1447                 goto error_free_vmsa_page;
1448         }
1449
1450         svm->vmcb01.ptr = page_address(vmcb01_page);
1451         svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1452         svm_switch_vmcb(svm, &svm->vmcb01);
1453
1454         if (vmsa_page)
1455                 svm->sev_es.vmsa = page_address(vmsa_page);
1456
1457         svm->guest_state_loaded = false;
1458
1459         return 0;
1460
1461 error_free_vmsa_page:
1462         if (vmsa_page)
1463                 __free_page(vmsa_page);
1464 error_free_vmcb_page:
1465         __free_page(vmcb01_page);
1466 out:
1467         return err;
1468 }
1469
1470 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1471 {
1472         int i;
1473
1474         for_each_online_cpu(i)
1475                 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
1476 }
1477
1478 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1479 {
1480         struct vcpu_svm *svm = to_svm(vcpu);
1481
1482         /*
1483          * The vmcb page can be recycled, causing a false negative in
1484          * svm_vcpu_load(). So, ensure that no logical CPU has this
1485          * vmcb page recorded as its current vmcb.
1486          */
1487         svm_clear_current_vmcb(svm->vmcb);
1488
1489         svm_free_nested(svm);
1490
1491         sev_free_vcpu(vcpu);
1492
1493         __free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT));
1494         __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1495 }
1496
1497 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
1498 {
1499         struct vcpu_svm *svm = to_svm(vcpu);
1500         struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
1501
1502         if (sev_es_guest(vcpu->kvm))
1503                 sev_es_unmap_ghcb(svm);
1504
1505         if (svm->guest_state_loaded)
1506                 return;
1507
1508         /*
1509          * Save additional host state that will be restored on VMEXIT (sev-es)
1510          * or subsequent vmload of host save area.
1511          */
1512         if (sev_es_guest(vcpu->kvm)) {
1513                 sev_es_prepare_guest_switch(svm, vcpu->cpu);
1514         } else {
1515                 vmsave(__sme_page_pa(sd->save_area));
1516         }
1517
1518         if (tsc_scaling) {
1519                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1520                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1521                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
1522                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1523                 }
1524         }
1525
1526         if (likely(tsc_aux_uret_slot >= 0))
1527                 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1528
1529         svm->guest_state_loaded = true;
1530 }
1531
1532 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1533 {
1534         to_svm(vcpu)->guest_state_loaded = false;
1535 }
1536
1537 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1538 {
1539         struct vcpu_svm *svm = to_svm(vcpu);
1540         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1541
1542         if (sd->current_vmcb != svm->vmcb) {
1543                 sd->current_vmcb = svm->vmcb;
1544                 indirect_branch_prediction_barrier();
1545         }
1546         if (kvm_vcpu_apicv_active(vcpu))
1547                 avic_vcpu_load(vcpu, cpu);
1548 }
1549
1550 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1551 {
1552         if (kvm_vcpu_apicv_active(vcpu))
1553                 avic_vcpu_put(vcpu);
1554
1555         svm_prepare_host_switch(vcpu);
1556
1557         ++vcpu->stat.host_state_reload;
1558 }
1559
1560 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1561 {
1562         struct vcpu_svm *svm = to_svm(vcpu);
1563         unsigned long rflags = svm->vmcb->save.rflags;
1564
1565         if (svm->nmi_singlestep) {
1566                 /* Hide our flags if they were not set by the guest */
1567                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1568                         rflags &= ~X86_EFLAGS_TF;
1569                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1570                         rflags &= ~X86_EFLAGS_RF;
1571         }
1572         return rflags;
1573 }
1574
1575 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1576 {
1577         if (to_svm(vcpu)->nmi_singlestep)
1578                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1579
1580        /*
1581         * Any change of EFLAGS.VM is accompanied by a reload of SS
1582         * (caused by either a task switch or an inter-privilege IRET),
1583         * so we do not need to update the CPL here.
1584         */
1585         to_svm(vcpu)->vmcb->save.rflags = rflags;
1586 }
1587
1588 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1589 {
1590         switch (reg) {
1591         case VCPU_EXREG_PDPTR:
1592                 BUG_ON(!npt_enabled);
1593                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1594                 break;
1595         default:
1596                 KVM_BUG_ON(1, vcpu->kvm);
1597         }
1598 }
1599
1600 static void svm_set_vintr(struct vcpu_svm *svm)
1601 {
1602         struct vmcb_control_area *control;
1603
1604         /*
1605          * The following fields are ignored when AVIC is enabled
1606          */
1607         WARN_ON(kvm_apicv_activated(svm->vcpu.kvm));
1608
1609         svm_set_intercept(svm, INTERCEPT_VINTR);
1610
1611         /*
1612          * This is just a dummy VINTR to actually cause a vmexit to happen.
1613          * Actual injection of virtual interrupts happens through EVENTINJ.
1614          */
1615         control = &svm->vmcb->control;
1616         control->int_vector = 0x0;
1617         control->int_ctl &= ~V_INTR_PRIO_MASK;
1618         control->int_ctl |= V_IRQ_MASK |
1619                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1620         vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1621 }
1622
1623 static void svm_clear_vintr(struct vcpu_svm *svm)
1624 {
1625         svm_clr_intercept(svm, INTERCEPT_VINTR);
1626
1627         /* Drop int_ctl fields related to VINTR injection.  */
1628         svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1629         if (is_guest_mode(&svm->vcpu)) {
1630                 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1631
1632                 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1633                         (svm->nested.ctl.int_ctl & V_TPR_MASK));
1634
1635                 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1636                         V_IRQ_INJECTION_BITS_MASK;
1637
1638                 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1639         }
1640
1641         vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1642 }
1643
1644 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1645 {
1646         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1647         struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1648
1649         switch (seg) {
1650         case VCPU_SREG_CS: return &save->cs;
1651         case VCPU_SREG_DS: return &save->ds;
1652         case VCPU_SREG_ES: return &save->es;
1653         case VCPU_SREG_FS: return &save01->fs;
1654         case VCPU_SREG_GS: return &save01->gs;
1655         case VCPU_SREG_SS: return &save->ss;
1656         case VCPU_SREG_TR: return &save01->tr;
1657         case VCPU_SREG_LDTR: return &save01->ldtr;
1658         }
1659         BUG();
1660         return NULL;
1661 }
1662
1663 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1664 {
1665         struct vmcb_seg *s = svm_seg(vcpu, seg);
1666
1667         return s->base;
1668 }
1669
1670 static void svm_get_segment(struct kvm_vcpu *vcpu,
1671                             struct kvm_segment *var, int seg)
1672 {
1673         struct vmcb_seg *s = svm_seg(vcpu, seg);
1674
1675         var->base = s->base;
1676         var->limit = s->limit;
1677         var->selector = s->selector;
1678         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1679         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1680         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1681         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1682         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1683         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1684         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1685
1686         /*
1687          * AMD CPUs circa 2014 track the G bit for all segments except CS.
1688          * However, the SVM spec states that the G bit is not observed by the
1689          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1690          * So let's synthesize a legal G bit for all segments, this helps
1691          * running KVM nested. It also helps cross-vendor migration, because
1692          * Intel's vmentry has a check on the 'G' bit.
1693          */
1694         var->g = s->limit > 0xfffff;
1695
1696         /*
1697          * AMD's VMCB does not have an explicit unusable field, so emulate it
1698          * for cross vendor migration purposes by "not present"
1699          */
1700         var->unusable = !var->present;
1701
1702         switch (seg) {
1703         case VCPU_SREG_TR:
1704                 /*
1705                  * Work around a bug where the busy flag in the tr selector
1706                  * isn't exposed
1707                  */
1708                 var->type |= 0x2;
1709                 break;
1710         case VCPU_SREG_DS:
1711         case VCPU_SREG_ES:
1712         case VCPU_SREG_FS:
1713         case VCPU_SREG_GS:
1714                 /*
1715                  * The accessed bit must always be set in the segment
1716                  * descriptor cache, although it can be cleared in the
1717                  * descriptor, the cached bit always remains at 1. Since
1718                  * Intel has a check on this, set it here to support
1719                  * cross-vendor migration.
1720                  */
1721                 if (!var->unusable)
1722                         var->type |= 0x1;
1723                 break;
1724         case VCPU_SREG_SS:
1725                 /*
1726                  * On AMD CPUs sometimes the DB bit in the segment
1727                  * descriptor is left as 1, although the whole segment has
1728                  * been made unusable. Clear it here to pass an Intel VMX
1729                  * entry check when cross vendor migrating.
1730                  */
1731                 if (var->unusable)
1732                         var->db = 0;
1733                 /* This is symmetric with svm_set_segment() */
1734                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1735                 break;
1736         }
1737 }
1738
1739 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1740 {
1741         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1742
1743         return save->cpl;
1744 }
1745
1746 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1747 {
1748         struct vcpu_svm *svm = to_svm(vcpu);
1749
1750         dt->size = svm->vmcb->save.idtr.limit;
1751         dt->address = svm->vmcb->save.idtr.base;
1752 }
1753
1754 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1755 {
1756         struct vcpu_svm *svm = to_svm(vcpu);
1757
1758         svm->vmcb->save.idtr.limit = dt->size;
1759         svm->vmcb->save.idtr.base = dt->address ;
1760         vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1761 }
1762
1763 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1764 {
1765         struct vcpu_svm *svm = to_svm(vcpu);
1766
1767         dt->size = svm->vmcb->save.gdtr.limit;
1768         dt->address = svm->vmcb->save.gdtr.base;
1769 }
1770
1771 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1772 {
1773         struct vcpu_svm *svm = to_svm(vcpu);
1774
1775         svm->vmcb->save.gdtr.limit = dt->size;
1776         svm->vmcb->save.gdtr.base = dt->address ;
1777         vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1778 }
1779
1780 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1781 {
1782         struct vcpu_svm *svm = to_svm(vcpu);
1783         u64 hcr0 = cr0;
1784
1785 #ifdef CONFIG_X86_64
1786         if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) {
1787                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1788                         vcpu->arch.efer |= EFER_LMA;
1789                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1790                 }
1791
1792                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1793                         vcpu->arch.efer &= ~EFER_LMA;
1794                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1795                 }
1796         }
1797 #endif
1798         vcpu->arch.cr0 = cr0;
1799
1800         if (!npt_enabled)
1801                 hcr0 |= X86_CR0_PG | X86_CR0_WP;
1802
1803         /*
1804          * re-enable caching here because the QEMU bios
1805          * does not do it - this results in some delay at
1806          * reboot
1807          */
1808         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1809                 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1810
1811         svm->vmcb->save.cr0 = hcr0;
1812         vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1813
1814         /*
1815          * SEV-ES guests must always keep the CR intercepts cleared. CR
1816          * tracking is done using the CR write traps.
1817          */
1818         if (sev_es_guest(vcpu->kvm))
1819                 return;
1820
1821         if (hcr0 == cr0) {
1822                 /* Selective CR0 write remains on.  */
1823                 svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1824                 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1825         } else {
1826                 svm_set_intercept(svm, INTERCEPT_CR0_READ);
1827                 svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1828         }
1829 }
1830
1831 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1832 {
1833         return true;
1834 }
1835
1836 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1837 {
1838         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1839         unsigned long old_cr4 = vcpu->arch.cr4;
1840
1841         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1842                 svm_flush_tlb(vcpu);
1843
1844         vcpu->arch.cr4 = cr4;
1845         if (!npt_enabled)
1846                 cr4 |= X86_CR4_PAE;
1847         cr4 |= host_cr4_mce;
1848         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1849         vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1850
1851         if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1852                 kvm_update_cpuid_runtime(vcpu);
1853 }
1854
1855 static void svm_set_segment(struct kvm_vcpu *vcpu,
1856                             struct kvm_segment *var, int seg)
1857 {
1858         struct vcpu_svm *svm = to_svm(vcpu);
1859         struct vmcb_seg *s = svm_seg(vcpu, seg);
1860
1861         s->base = var->base;
1862         s->limit = var->limit;
1863         s->selector = var->selector;
1864         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1865         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1866         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1867         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1868         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1869         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1870         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1871         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1872
1873         /*
1874          * This is always accurate, except if SYSRET returned to a segment
1875          * with SS.DPL != 3.  Intel does not have this quirk, and always
1876          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1877          * would entail passing the CPL to userspace and back.
1878          */
1879         if (seg == VCPU_SREG_SS)
1880                 /* This is symmetric with svm_get_segment() */
1881                 svm->vmcb->save.cpl = (var->dpl & 3);
1882
1883         vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1884 }
1885
1886 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1887 {
1888         struct vcpu_svm *svm = to_svm(vcpu);
1889
1890         clr_exception_intercept(svm, BP_VECTOR);
1891
1892         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1893                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1894                         set_exception_intercept(svm, BP_VECTOR);
1895         }
1896 }
1897
1898 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1899 {
1900         if (sd->next_asid > sd->max_asid) {
1901                 ++sd->asid_generation;
1902                 sd->next_asid = sd->min_asid;
1903                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1904                 vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1905         }
1906
1907         svm->current_vmcb->asid_generation = sd->asid_generation;
1908         svm->asid = sd->next_asid++;
1909 }
1910
1911 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
1912 {
1913         struct vmcb *vmcb = svm->vmcb;
1914
1915         if (svm->vcpu.arch.guest_state_protected)
1916                 return;
1917
1918         if (unlikely(value != vmcb->save.dr6)) {
1919                 vmcb->save.dr6 = value;
1920                 vmcb_mark_dirty(vmcb, VMCB_DR);
1921         }
1922 }
1923
1924 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1925 {
1926         struct vcpu_svm *svm = to_svm(vcpu);
1927
1928         if (vcpu->arch.guest_state_protected)
1929                 return;
1930
1931         get_debugreg(vcpu->arch.db[0], 0);
1932         get_debugreg(vcpu->arch.db[1], 1);
1933         get_debugreg(vcpu->arch.db[2], 2);
1934         get_debugreg(vcpu->arch.db[3], 3);
1935         /*
1936          * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
1937          * because db_interception might need it.  We can do it before vmentry.
1938          */
1939         vcpu->arch.dr6 = svm->vmcb->save.dr6;
1940         vcpu->arch.dr7 = svm->vmcb->save.dr7;
1941         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1942         set_dr_intercepts(svm);
1943 }
1944
1945 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1946 {
1947         struct vcpu_svm *svm = to_svm(vcpu);
1948
1949         if (vcpu->arch.guest_state_protected)
1950                 return;
1951
1952         svm->vmcb->save.dr7 = value;
1953         vmcb_mark_dirty(svm->vmcb, VMCB_DR);
1954 }
1955
1956 static int pf_interception(struct kvm_vcpu *vcpu)
1957 {
1958         struct vcpu_svm *svm = to_svm(vcpu);
1959
1960         u64 fault_address = svm->vmcb->control.exit_info_2;
1961         u64 error_code = svm->vmcb->control.exit_info_1;
1962
1963         return kvm_handle_page_fault(vcpu, error_code, fault_address,
1964                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1965                         svm->vmcb->control.insn_bytes : NULL,
1966                         svm->vmcb->control.insn_len);
1967 }
1968
1969 static int npf_interception(struct kvm_vcpu *vcpu)
1970 {
1971         struct vcpu_svm *svm = to_svm(vcpu);
1972
1973         u64 fault_address = svm->vmcb->control.exit_info_2;
1974         u64 error_code = svm->vmcb->control.exit_info_1;
1975
1976         trace_kvm_page_fault(fault_address, error_code);
1977         return kvm_mmu_page_fault(vcpu, fault_address, error_code,
1978                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1979                         svm->vmcb->control.insn_bytes : NULL,
1980                         svm->vmcb->control.insn_len);
1981 }
1982
1983 static int db_interception(struct kvm_vcpu *vcpu)
1984 {
1985         struct kvm_run *kvm_run = vcpu->run;
1986         struct vcpu_svm *svm = to_svm(vcpu);
1987
1988         if (!(vcpu->guest_debug &
1989               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1990                 !svm->nmi_singlestep) {
1991                 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
1992                 kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
1993                 return 1;
1994         }
1995
1996         if (svm->nmi_singlestep) {
1997                 disable_nmi_singlestep(svm);
1998                 /* Make sure we check for pending NMIs upon entry */
1999                 kvm_make_request(KVM_REQ_EVENT, vcpu);
2000         }
2001
2002         if (vcpu->guest_debug &
2003             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2004                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2005                 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
2006                 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
2007                 kvm_run->debug.arch.pc =
2008                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2009                 kvm_run->debug.arch.exception = DB_VECTOR;
2010                 return 0;
2011         }
2012
2013         return 1;
2014 }
2015
2016 static int bp_interception(struct kvm_vcpu *vcpu)
2017 {
2018         struct vcpu_svm *svm = to_svm(vcpu);
2019         struct kvm_run *kvm_run = vcpu->run;
2020
2021         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2022         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2023         kvm_run->debug.arch.exception = BP_VECTOR;
2024         return 0;
2025 }
2026
2027 static int ud_interception(struct kvm_vcpu *vcpu)
2028 {
2029         return handle_ud(vcpu);
2030 }
2031
2032 static int ac_interception(struct kvm_vcpu *vcpu)
2033 {
2034         kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
2035         return 1;
2036 }
2037
2038 static bool is_erratum_383(void)
2039 {
2040         int err, i;
2041         u64 value;
2042
2043         if (!erratum_383_found)
2044                 return false;
2045
2046         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2047         if (err)
2048                 return false;
2049
2050         /* Bit 62 may or may not be set for this mce */
2051         value &= ~(1ULL << 62);
2052
2053         if (value != 0xb600000000010015ULL)
2054                 return false;
2055
2056         /* Clear MCi_STATUS registers */
2057         for (i = 0; i < 6; ++i)
2058                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2059
2060         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2061         if (!err) {
2062                 u32 low, high;
2063
2064                 value &= ~(1ULL << 2);
2065                 low    = lower_32_bits(value);
2066                 high   = upper_32_bits(value);
2067
2068                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2069         }
2070
2071         /* Flush tlb to evict multi-match entries */
2072         __flush_tlb_all();
2073
2074         return true;
2075 }
2076
2077 static void svm_handle_mce(struct kvm_vcpu *vcpu)
2078 {
2079         if (is_erratum_383()) {
2080                 /*
2081                  * Erratum 383 triggered. Guest state is corrupt so kill the
2082                  * guest.
2083                  */
2084                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2085
2086                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2087
2088                 return;
2089         }
2090
2091         /*
2092          * On an #MC intercept the MCE handler is not called automatically in
2093          * the host. So do it by hand here.
2094          */
2095         kvm_machine_check();
2096 }
2097
2098 static int mc_interception(struct kvm_vcpu *vcpu)
2099 {
2100         return 1;
2101 }
2102
2103 static int shutdown_interception(struct kvm_vcpu *vcpu)
2104 {
2105         struct kvm_run *kvm_run = vcpu->run;
2106         struct vcpu_svm *svm = to_svm(vcpu);
2107
2108         /*
2109          * The VM save area has already been encrypted so it
2110          * cannot be reinitialized - just terminate.
2111          */
2112         if (sev_es_guest(vcpu->kvm))
2113                 return -EINVAL;
2114
2115         /*
2116          * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2117          * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2118          * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2119          * userspace.  At a platform view, INIT is acceptable behavior as
2120          * there exist bare metal platforms that automatically INIT the CPU
2121          * in response to shutdown.
2122          */
2123         clear_page(svm->vmcb);
2124         kvm_vcpu_reset(vcpu, true);
2125
2126         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2127         return 0;
2128 }
2129
2130 static int io_interception(struct kvm_vcpu *vcpu)
2131 {
2132         struct vcpu_svm *svm = to_svm(vcpu);
2133         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2134         int size, in, string;
2135         unsigned port;
2136
2137         ++vcpu->stat.io_exits;
2138         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2139         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2140         port = io_info >> 16;
2141         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2142
2143         if (string) {
2144                 if (sev_es_guest(vcpu->kvm))
2145                         return sev_es_string_io(svm, size, port, in);
2146                 else
2147                         return kvm_emulate_instruction(vcpu, 0);
2148         }
2149
2150         svm->next_rip = svm->vmcb->control.exit_info_2;
2151
2152         return kvm_fast_pio(vcpu, size, port, in);
2153 }
2154
2155 static int nmi_interception(struct kvm_vcpu *vcpu)
2156 {
2157         return 1;
2158 }
2159
2160 static int smi_interception(struct kvm_vcpu *vcpu)
2161 {
2162         return 1;
2163 }
2164
2165 static int intr_interception(struct kvm_vcpu *vcpu)
2166 {
2167         ++vcpu->stat.irq_exits;
2168         return 1;
2169 }
2170
2171 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2172 {
2173         struct vcpu_svm *svm = to_svm(vcpu);
2174         struct vmcb *vmcb12;
2175         struct kvm_host_map map;
2176         int ret;
2177
2178         if (nested_svm_check_permissions(vcpu))
2179                 return 1;
2180
2181         ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2182         if (ret) {
2183                 if (ret == -EINVAL)
2184                         kvm_inject_gp(vcpu, 0);
2185                 return 1;
2186         }
2187
2188         vmcb12 = map.hva;
2189
2190         ret = kvm_skip_emulated_instruction(vcpu);
2191
2192         if (vmload) {
2193                 svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2194                 svm->sysenter_eip_hi = 0;
2195                 svm->sysenter_esp_hi = 0;
2196         } else {
2197                 svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2198         }
2199
2200         kvm_vcpu_unmap(vcpu, &map, true);
2201
2202         return ret;
2203 }
2204
2205 static int vmload_interception(struct kvm_vcpu *vcpu)
2206 {
2207         return vmload_vmsave_interception(vcpu, true);
2208 }
2209
2210 static int vmsave_interception(struct kvm_vcpu *vcpu)
2211 {
2212         return vmload_vmsave_interception(vcpu, false);
2213 }
2214
2215 static int vmrun_interception(struct kvm_vcpu *vcpu)
2216 {
2217         if (nested_svm_check_permissions(vcpu))
2218                 return 1;
2219
2220         return nested_svm_vmrun(vcpu);
2221 }
2222
2223 enum {
2224         NONE_SVM_INSTR,
2225         SVM_INSTR_VMRUN,
2226         SVM_INSTR_VMLOAD,
2227         SVM_INSTR_VMSAVE,
2228 };
2229
2230 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2231 static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2232 {
2233         struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2234
2235         if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2236                 return NONE_SVM_INSTR;
2237
2238         switch (ctxt->modrm) {
2239         case 0xd8: /* VMRUN */
2240                 return SVM_INSTR_VMRUN;
2241         case 0xda: /* VMLOAD */
2242                 return SVM_INSTR_VMLOAD;
2243         case 0xdb: /* VMSAVE */
2244                 return SVM_INSTR_VMSAVE;
2245         default:
2246                 break;
2247         }
2248
2249         return NONE_SVM_INSTR;
2250 }
2251
2252 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2253 {
2254         const int guest_mode_exit_codes[] = {
2255                 [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2256                 [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2257                 [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2258         };
2259         int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2260                 [SVM_INSTR_VMRUN] = vmrun_interception,
2261                 [SVM_INSTR_VMLOAD] = vmload_interception,
2262                 [SVM_INSTR_VMSAVE] = vmsave_interception,
2263         };
2264         struct vcpu_svm *svm = to_svm(vcpu);
2265         int ret;
2266
2267         if (is_guest_mode(vcpu)) {
2268                 /* Returns '1' or -errno on failure, '0' on success. */
2269                 ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2270                 if (ret)
2271                         return ret;
2272                 return 1;
2273         }
2274         return svm_instr_handlers[opcode](vcpu);
2275 }
2276
2277 /*
2278  * #GP handling code. Note that #GP can be triggered under the following two
2279  * cases:
2280  *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2281  *      some AMD CPUs when EAX of these instructions are in the reserved memory
2282  *      regions (e.g. SMM memory on host).
2283  *   2) VMware backdoor
2284  */
2285 static int gp_interception(struct kvm_vcpu *vcpu)
2286 {
2287         struct vcpu_svm *svm = to_svm(vcpu);
2288         u32 error_code = svm->vmcb->control.exit_info_1;
2289         int opcode;
2290
2291         /* Both #GP cases have zero error_code */
2292         if (error_code)
2293                 goto reinject;
2294
2295         /* All SVM instructions expect page aligned RAX */
2296         if (svm->vmcb->save.rax & ~PAGE_MASK)
2297                 goto reinject;
2298
2299         /* Decode the instruction for usage later */
2300         if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2301                 goto reinject;
2302
2303         opcode = svm_instr_opcode(vcpu);
2304
2305         if (opcode == NONE_SVM_INSTR) {
2306                 if (!enable_vmware_backdoor)
2307                         goto reinject;
2308
2309                 /*
2310                  * VMware backdoor emulation on #GP interception only handles
2311                  * IN{S}, OUT{S}, and RDPMC.
2312                  */
2313                 if (!is_guest_mode(vcpu))
2314                         return kvm_emulate_instruction(vcpu,
2315                                 EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2316         } else
2317                 return emulate_svm_instr(vcpu, opcode);
2318
2319 reinject:
2320         kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2321         return 1;
2322 }
2323
2324 void svm_set_gif(struct vcpu_svm *svm, bool value)
2325 {
2326         if (value) {
2327                 /*
2328                  * If VGIF is enabled, the STGI intercept is only added to
2329                  * detect the opening of the SMI/NMI window; remove it now.
2330                  * Likewise, clear the VINTR intercept, we will set it
2331                  * again while processing KVM_REQ_EVENT if needed.
2332                  */
2333                 if (vgif_enabled(svm))
2334                         svm_clr_intercept(svm, INTERCEPT_STGI);
2335                 if (svm_is_intercept(svm, INTERCEPT_VINTR))
2336                         svm_clear_vintr(svm);
2337
2338                 enable_gif(svm);
2339                 if (svm->vcpu.arch.smi_pending ||
2340                     svm->vcpu.arch.nmi_pending ||
2341                     kvm_cpu_has_injectable_intr(&svm->vcpu))
2342                         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2343         } else {
2344                 disable_gif(svm);
2345
2346                 /*
2347                  * After a CLGI no interrupts should come.  But if vGIF is
2348                  * in use, we still rely on the VINTR intercept (rather than
2349                  * STGI) to detect an open interrupt window.
2350                 */
2351                 if (!vgif_enabled(svm))
2352                         svm_clear_vintr(svm);
2353         }
2354 }
2355
2356 static int stgi_interception(struct kvm_vcpu *vcpu)
2357 {
2358         int ret;
2359
2360         if (nested_svm_check_permissions(vcpu))
2361                 return 1;
2362
2363         ret = kvm_skip_emulated_instruction(vcpu);
2364         svm_set_gif(to_svm(vcpu), true);
2365         return ret;
2366 }
2367
2368 static int clgi_interception(struct kvm_vcpu *vcpu)
2369 {
2370         int ret;
2371
2372         if (nested_svm_check_permissions(vcpu))
2373                 return 1;
2374
2375         ret = kvm_skip_emulated_instruction(vcpu);
2376         svm_set_gif(to_svm(vcpu), false);
2377         return ret;
2378 }
2379
2380 static int invlpga_interception(struct kvm_vcpu *vcpu)
2381 {
2382         gva_t gva = kvm_rax_read(vcpu);
2383         u32 asid = kvm_rcx_read(vcpu);
2384
2385         /* FIXME: Handle an address size prefix. */
2386         if (!is_long_mode(vcpu))
2387                 gva = (u32)gva;
2388
2389         trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2390
2391         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2392         kvm_mmu_invlpg(vcpu, gva);
2393
2394         return kvm_skip_emulated_instruction(vcpu);
2395 }
2396
2397 static int skinit_interception(struct kvm_vcpu *vcpu)
2398 {
2399         trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2400
2401         kvm_queue_exception(vcpu, UD_VECTOR);
2402         return 1;
2403 }
2404
2405 static int task_switch_interception(struct kvm_vcpu *vcpu)
2406 {
2407         struct vcpu_svm *svm = to_svm(vcpu);
2408         u16 tss_selector;
2409         int reason;
2410         int int_type = svm->vmcb->control.exit_int_info &
2411                 SVM_EXITINTINFO_TYPE_MASK;
2412         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2413         uint32_t type =
2414                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2415         uint32_t idt_v =
2416                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2417         bool has_error_code = false;
2418         u32 error_code = 0;
2419
2420         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2421
2422         if (svm->vmcb->control.exit_info_2 &
2423             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2424                 reason = TASK_SWITCH_IRET;
2425         else if (svm->vmcb->control.exit_info_2 &
2426                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2427                 reason = TASK_SWITCH_JMP;
2428         else if (idt_v)
2429                 reason = TASK_SWITCH_GATE;
2430         else
2431                 reason = TASK_SWITCH_CALL;
2432
2433         if (reason == TASK_SWITCH_GATE) {
2434                 switch (type) {
2435                 case SVM_EXITINTINFO_TYPE_NMI:
2436                         vcpu->arch.nmi_injected = false;
2437                         break;
2438                 case SVM_EXITINTINFO_TYPE_EXEPT:
2439                         if (svm->vmcb->control.exit_info_2 &
2440                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2441                                 has_error_code = true;
2442                                 error_code =
2443                                         (u32)svm->vmcb->control.exit_info_2;
2444                         }
2445                         kvm_clear_exception_queue(vcpu);
2446                         break;
2447                 case SVM_EXITINTINFO_TYPE_INTR:
2448                         kvm_clear_interrupt_queue(vcpu);
2449                         break;
2450                 default:
2451                         break;
2452                 }
2453         }
2454
2455         if (reason != TASK_SWITCH_GATE ||
2456             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2457             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2458              (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2459                 if (!skip_emulated_instruction(vcpu))
2460                         return 0;
2461         }
2462
2463         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2464                 int_vec = -1;
2465
2466         return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2467                                has_error_code, error_code);
2468 }
2469
2470 static int iret_interception(struct kvm_vcpu *vcpu)
2471 {
2472         struct vcpu_svm *svm = to_svm(vcpu);
2473
2474         ++vcpu->stat.nmi_window_exits;
2475         vcpu->arch.hflags |= HF_IRET_MASK;
2476         if (!sev_es_guest(vcpu->kvm)) {
2477                 svm_clr_intercept(svm, INTERCEPT_IRET);
2478                 svm->nmi_iret_rip = kvm_rip_read(vcpu);
2479         }
2480         kvm_make_request(KVM_REQ_EVENT, vcpu);
2481         return 1;
2482 }
2483
2484 static int invlpg_interception(struct kvm_vcpu *vcpu)
2485 {
2486         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2487                 return kvm_emulate_instruction(vcpu, 0);
2488
2489         kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2490         return kvm_skip_emulated_instruction(vcpu);
2491 }
2492
2493 static int emulate_on_interception(struct kvm_vcpu *vcpu)
2494 {
2495         return kvm_emulate_instruction(vcpu, 0);
2496 }
2497
2498 static int rsm_interception(struct kvm_vcpu *vcpu)
2499 {
2500         return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2501 }
2502
2503 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2504                                             unsigned long val)
2505 {
2506         struct vcpu_svm *svm = to_svm(vcpu);
2507         unsigned long cr0 = vcpu->arch.cr0;
2508         bool ret = false;
2509
2510         if (!is_guest_mode(vcpu) ||
2511             (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2512                 return false;
2513
2514         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2515         val &= ~SVM_CR0_SELECTIVE_MASK;
2516
2517         if (cr0 ^ val) {
2518                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2519                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2520         }
2521
2522         return ret;
2523 }
2524
2525 #define CR_VALID (1ULL << 63)
2526
2527 static int cr_interception(struct kvm_vcpu *vcpu)
2528 {
2529         struct vcpu_svm *svm = to_svm(vcpu);
2530         int reg, cr;
2531         unsigned long val;
2532         int err;
2533
2534         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2535                 return emulate_on_interception(vcpu);
2536
2537         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2538                 return emulate_on_interception(vcpu);
2539
2540         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2541         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2542                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2543         else
2544                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2545
2546         err = 0;
2547         if (cr >= 16) { /* mov to cr */
2548                 cr -= 16;
2549                 val = kvm_register_read(vcpu, reg);
2550                 trace_kvm_cr_write(cr, val);
2551                 switch (cr) {
2552                 case 0:
2553                         if (!check_selective_cr0_intercepted(vcpu, val))
2554                                 err = kvm_set_cr0(vcpu, val);
2555                         else
2556                                 return 1;
2557
2558                         break;
2559                 case 3:
2560                         err = kvm_set_cr3(vcpu, val);
2561                         break;
2562                 case 4:
2563                         err = kvm_set_cr4(vcpu, val);
2564                         break;
2565                 case 8:
2566                         err = kvm_set_cr8(vcpu, val);
2567                         break;
2568                 default:
2569                         WARN(1, "unhandled write to CR%d", cr);
2570                         kvm_queue_exception(vcpu, UD_VECTOR);
2571                         return 1;
2572                 }
2573         } else { /* mov from cr */
2574                 switch (cr) {
2575                 case 0:
2576                         val = kvm_read_cr0(vcpu);
2577                         break;
2578                 case 2:
2579                         val = vcpu->arch.cr2;
2580                         break;
2581                 case 3:
2582                         val = kvm_read_cr3(vcpu);
2583                         break;
2584                 case 4:
2585                         val = kvm_read_cr4(vcpu);
2586                         break;
2587                 case 8:
2588                         val = kvm_get_cr8(vcpu);
2589                         break;
2590                 default:
2591                         WARN(1, "unhandled read from CR%d", cr);
2592                         kvm_queue_exception(vcpu, UD_VECTOR);
2593                         return 1;
2594                 }
2595                 kvm_register_write(vcpu, reg, val);
2596                 trace_kvm_cr_read(cr, val);
2597         }
2598         return kvm_complete_insn_gp(vcpu, err);
2599 }
2600
2601 static int cr_trap(struct kvm_vcpu *vcpu)
2602 {
2603         struct vcpu_svm *svm = to_svm(vcpu);
2604         unsigned long old_value, new_value;
2605         unsigned int cr;
2606         int ret = 0;
2607
2608         new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2609
2610         cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2611         switch (cr) {
2612         case 0:
2613                 old_value = kvm_read_cr0(vcpu);
2614                 svm_set_cr0(vcpu, new_value);
2615
2616                 kvm_post_set_cr0(vcpu, old_value, new_value);
2617                 break;
2618         case 4:
2619                 old_value = kvm_read_cr4(vcpu);
2620                 svm_set_cr4(vcpu, new_value);
2621
2622                 kvm_post_set_cr4(vcpu, old_value, new_value);
2623                 break;
2624         case 8:
2625                 ret = kvm_set_cr8(vcpu, new_value);
2626                 break;
2627         default:
2628                 WARN(1, "unhandled CR%d write trap", cr);
2629                 kvm_queue_exception(vcpu, UD_VECTOR);
2630                 return 1;
2631         }
2632
2633         return kvm_complete_insn_gp(vcpu, ret);
2634 }
2635
2636 static int dr_interception(struct kvm_vcpu *vcpu)
2637 {
2638         struct vcpu_svm *svm = to_svm(vcpu);
2639         int reg, dr;
2640         unsigned long val;
2641         int err = 0;
2642
2643         if (vcpu->guest_debug == 0) {
2644                 /*
2645                  * No more DR vmexits; force a reload of the debug registers
2646                  * and reenter on this instruction.  The next vmexit will
2647                  * retrieve the full state of the debug registers.
2648                  */
2649                 clr_dr_intercepts(svm);
2650                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2651                 return 1;
2652         }
2653
2654         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2655                 return emulate_on_interception(vcpu);
2656
2657         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2658         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2659         if (dr >= 16) { /* mov to DRn  */
2660                 dr -= 16;
2661                 val = kvm_register_read(vcpu, reg);
2662                 err = kvm_set_dr(vcpu, dr, val);
2663         } else {
2664                 kvm_get_dr(vcpu, dr, &val);
2665                 kvm_register_write(vcpu, reg, val);
2666         }
2667
2668         return kvm_complete_insn_gp(vcpu, err);
2669 }
2670
2671 static int cr8_write_interception(struct kvm_vcpu *vcpu)
2672 {
2673         int r;
2674
2675         u8 cr8_prev = kvm_get_cr8(vcpu);
2676         /* instruction emulation calls kvm_set_cr8() */
2677         r = cr_interception(vcpu);
2678         if (lapic_in_kernel(vcpu))
2679                 return r;
2680         if (cr8_prev <= kvm_get_cr8(vcpu))
2681                 return r;
2682         vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2683         return 0;
2684 }
2685
2686 static int efer_trap(struct kvm_vcpu *vcpu)
2687 {
2688         struct msr_data msr_info;
2689         int ret;
2690
2691         /*
2692          * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2693          * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2694          * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2695          * the guest doesn't have X86_FEATURE_SVM.
2696          */
2697         msr_info.host_initiated = false;
2698         msr_info.index = MSR_EFER;
2699         msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2700         ret = kvm_set_msr_common(vcpu, &msr_info);
2701
2702         return kvm_complete_insn_gp(vcpu, ret);
2703 }
2704
2705 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2706 {
2707         msr->data = 0;
2708
2709         switch (msr->index) {
2710         case MSR_F10H_DECFG:
2711                 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
2712                         msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
2713                 break;
2714         case MSR_IA32_PERF_CAPABILITIES:
2715                 return 0;
2716         default:
2717                 return KVM_MSR_RET_INVALID;
2718         }
2719
2720         return 0;
2721 }
2722
2723 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2724 {
2725         struct vcpu_svm *svm = to_svm(vcpu);
2726
2727         switch (msr_info->index) {
2728         case MSR_AMD64_TSC_RATIO:
2729                 if (!msr_info->host_initiated && !svm->tsc_scaling_enabled)
2730                         return 1;
2731                 msr_info->data = svm->tsc_ratio_msr;
2732                 break;
2733         case MSR_STAR:
2734                 msr_info->data = svm->vmcb01.ptr->save.star;
2735                 break;
2736 #ifdef CONFIG_X86_64
2737         case MSR_LSTAR:
2738                 msr_info->data = svm->vmcb01.ptr->save.lstar;
2739                 break;
2740         case MSR_CSTAR:
2741                 msr_info->data = svm->vmcb01.ptr->save.cstar;
2742                 break;
2743         case MSR_KERNEL_GS_BASE:
2744                 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2745                 break;
2746         case MSR_SYSCALL_MASK:
2747                 msr_info->data = svm->vmcb01.ptr->save.sfmask;
2748                 break;
2749 #endif
2750         case MSR_IA32_SYSENTER_CS:
2751                 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2752                 break;
2753         case MSR_IA32_SYSENTER_EIP:
2754                 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2755                 if (guest_cpuid_is_intel(vcpu))
2756                         msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2757                 break;
2758         case MSR_IA32_SYSENTER_ESP:
2759                 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2760                 if (guest_cpuid_is_intel(vcpu))
2761                         msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2762                 break;
2763         case MSR_TSC_AUX:
2764                 msr_info->data = svm->tsc_aux;
2765                 break;
2766         /*
2767          * Nobody will change the following 5 values in the VMCB so we can
2768          * safely return them on rdmsr. They will always be 0 until LBRV is
2769          * implemented.
2770          */
2771         case MSR_IA32_DEBUGCTLMSR:
2772                 msr_info->data = svm->vmcb->save.dbgctl;
2773                 break;
2774         case MSR_IA32_LASTBRANCHFROMIP:
2775                 msr_info->data = svm->vmcb->save.br_from;
2776                 break;
2777         case MSR_IA32_LASTBRANCHTOIP:
2778                 msr_info->data = svm->vmcb->save.br_to;
2779                 break;
2780         case MSR_IA32_LASTINTFROMIP:
2781                 msr_info->data = svm->vmcb->save.last_excp_from;
2782                 break;
2783         case MSR_IA32_LASTINTTOIP:
2784                 msr_info->data = svm->vmcb->save.last_excp_to;
2785                 break;
2786         case MSR_VM_HSAVE_PA:
2787                 msr_info->data = svm->nested.hsave_msr;
2788                 break;
2789         case MSR_VM_CR:
2790                 msr_info->data = svm->nested.vm_cr_msr;
2791                 break;
2792         case MSR_IA32_SPEC_CTRL:
2793                 if (!msr_info->host_initiated &&
2794                     !guest_has_spec_ctrl_msr(vcpu))
2795                         return 1;
2796
2797                 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2798                         msr_info->data = svm->vmcb->save.spec_ctrl;
2799                 else
2800                         msr_info->data = svm->spec_ctrl;
2801                 break;
2802         case MSR_AMD64_VIRT_SPEC_CTRL:
2803                 if (!msr_info->host_initiated &&
2804                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2805                         return 1;
2806
2807                 msr_info->data = svm->virt_spec_ctrl;
2808                 break;
2809         case MSR_F15H_IC_CFG: {
2810
2811                 int family, model;
2812
2813                 family = guest_cpuid_family(vcpu);
2814                 model  = guest_cpuid_model(vcpu);
2815
2816                 if (family < 0 || model < 0)
2817                         return kvm_get_msr_common(vcpu, msr_info);
2818
2819                 msr_info->data = 0;
2820
2821                 if (family == 0x15 &&
2822                     (model >= 0x2 && model < 0x20))
2823                         msr_info->data = 0x1E;
2824                 }
2825                 break;
2826         case MSR_F10H_DECFG:
2827                 msr_info->data = svm->msr_decfg;
2828                 break;
2829         default:
2830                 return kvm_get_msr_common(vcpu, msr_info);
2831         }
2832         return 0;
2833 }
2834
2835 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2836 {
2837         struct vcpu_svm *svm = to_svm(vcpu);
2838         if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2839                 return kvm_complete_insn_gp(vcpu, err);
2840
2841         ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1);
2842         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb,
2843                                 X86_TRAP_GP |
2844                                 SVM_EVTINJ_TYPE_EXEPT |
2845                                 SVM_EVTINJ_VALID);
2846         return 1;
2847 }
2848
2849 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2850 {
2851         struct vcpu_svm *svm = to_svm(vcpu);
2852         int svm_dis, chg_mask;
2853
2854         if (data & ~SVM_VM_CR_VALID_MASK)
2855                 return 1;
2856
2857         chg_mask = SVM_VM_CR_VALID_MASK;
2858
2859         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2860                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2861
2862         svm->nested.vm_cr_msr &= ~chg_mask;
2863         svm->nested.vm_cr_msr |= (data & chg_mask);
2864
2865         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2866
2867         /* check for svm_disable while efer.svme is set */
2868         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2869                 return 1;
2870
2871         return 0;
2872 }
2873
2874 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2875 {
2876         struct vcpu_svm *svm = to_svm(vcpu);
2877         int r;
2878
2879         u32 ecx = msr->index;
2880         u64 data = msr->data;
2881         switch (ecx) {
2882         case MSR_AMD64_TSC_RATIO:
2883                 if (!msr->host_initiated && !svm->tsc_scaling_enabled)
2884                         return 1;
2885
2886                 if (data & TSC_RATIO_RSVD)
2887                         return 1;
2888
2889                 svm->tsc_ratio_msr = data;
2890
2891                 if (svm->tsc_scaling_enabled && is_guest_mode(vcpu))
2892                         nested_svm_update_tsc_ratio_msr(vcpu);
2893
2894                 break;
2895         case MSR_IA32_CR_PAT:
2896                 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
2897                         return 1;
2898                 vcpu->arch.pat = data;
2899                 svm->vmcb01.ptr->save.g_pat = data;
2900                 if (is_guest_mode(vcpu))
2901                         nested_vmcb02_compute_g_pat(svm);
2902                 vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
2903                 break;
2904         case MSR_IA32_SPEC_CTRL:
2905                 if (!msr->host_initiated &&
2906                     !guest_has_spec_ctrl_msr(vcpu))
2907                         return 1;
2908
2909                 if (kvm_spec_ctrl_test_value(data))
2910                         return 1;
2911
2912                 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2913                         svm->vmcb->save.spec_ctrl = data;
2914                 else
2915                         svm->spec_ctrl = data;
2916                 if (!data)
2917                         break;
2918
2919                 /*
2920                  * For non-nested:
2921                  * When it's written (to non-zero) for the first time, pass
2922                  * it through.
2923                  *
2924                  * For nested:
2925                  * The handling of the MSR bitmap for L2 guests is done in
2926                  * nested_svm_vmrun_msrpm.
2927                  * We update the L1 MSR bit as well since it will end up
2928                  * touching the MSR anyway now.
2929                  */
2930                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
2931                 break;
2932         case MSR_IA32_PRED_CMD:
2933                 if (!msr->host_initiated &&
2934                     !guest_has_pred_cmd_msr(vcpu))
2935                         return 1;
2936
2937                 if (data & ~PRED_CMD_IBPB)
2938                         return 1;
2939                 if (!boot_cpu_has(X86_FEATURE_IBPB))
2940                         return 1;
2941                 if (!data)
2942                         break;
2943
2944                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2945                 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
2946                 break;
2947         case MSR_AMD64_VIRT_SPEC_CTRL:
2948                 if (!msr->host_initiated &&
2949                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2950                         return 1;
2951
2952                 if (data & ~SPEC_CTRL_SSBD)
2953                         return 1;
2954
2955                 svm->virt_spec_ctrl = data;
2956                 break;
2957         case MSR_STAR:
2958                 svm->vmcb01.ptr->save.star = data;
2959                 break;
2960 #ifdef CONFIG_X86_64
2961         case MSR_LSTAR:
2962                 svm->vmcb01.ptr->save.lstar = data;
2963                 break;
2964         case MSR_CSTAR:
2965                 svm->vmcb01.ptr->save.cstar = data;
2966                 break;
2967         case MSR_KERNEL_GS_BASE:
2968                 svm->vmcb01.ptr->save.kernel_gs_base = data;
2969                 break;
2970         case MSR_SYSCALL_MASK:
2971                 svm->vmcb01.ptr->save.sfmask = data;
2972                 break;
2973 #endif
2974         case MSR_IA32_SYSENTER_CS:
2975                 svm->vmcb01.ptr->save.sysenter_cs = data;
2976                 break;
2977         case MSR_IA32_SYSENTER_EIP:
2978                 svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
2979                 /*
2980                  * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
2981                  * when we spoof an Intel vendor ID (for cross vendor migration).
2982                  * In this case we use this intercept to track the high
2983                  * 32 bit part of these msrs to support Intel's
2984                  * implementation of SYSENTER/SYSEXIT.
2985                  */
2986                 svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
2987                 break;
2988         case MSR_IA32_SYSENTER_ESP:
2989                 svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
2990                 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
2991                 break;
2992         case MSR_TSC_AUX:
2993                 /*
2994                  * TSC_AUX is usually changed only during boot and never read
2995                  * directly.  Intercept TSC_AUX instead of exposing it to the
2996                  * guest via direct_access_msrs, and switch it via user return.
2997                  */
2998                 preempt_disable();
2999                 r = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
3000                 preempt_enable();
3001                 if (r)
3002                         return 1;
3003
3004                 svm->tsc_aux = data;
3005                 break;
3006         case MSR_IA32_DEBUGCTLMSR:
3007                 if (!lbrv) {
3008                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3009                                     __func__, data);
3010                         break;
3011                 }
3012                 if (data & DEBUGCTL_RESERVED_BITS)
3013                         return 1;
3014
3015                 svm->vmcb->save.dbgctl = data;
3016                 vmcb_mark_dirty(svm->vmcb, VMCB_LBR);
3017                 if (data & (1ULL<<0))
3018                         svm_enable_lbrv(vcpu);
3019                 else
3020                         svm_disable_lbrv(vcpu);
3021                 break;
3022         case MSR_VM_HSAVE_PA:
3023                 /*
3024                  * Old kernels did not validate the value written to
3025                  * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3026                  * value to allow live migrating buggy or malicious guests
3027                  * originating from those kernels.
3028                  */
3029                 if (!msr->host_initiated && !page_address_valid(vcpu, data))
3030                         return 1;
3031
3032                 svm->nested.hsave_msr = data & PAGE_MASK;
3033                 break;
3034         case MSR_VM_CR:
3035                 return svm_set_vm_cr(vcpu, data);
3036         case MSR_VM_IGNNE:
3037                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3038                 break;
3039         case MSR_F10H_DECFG: {
3040                 struct kvm_msr_entry msr_entry;
3041
3042                 msr_entry.index = msr->index;
3043                 if (svm_get_msr_feature(&msr_entry))
3044                         return 1;
3045
3046                 /* Check the supported bits */
3047                 if (data & ~msr_entry.data)
3048                         return 1;
3049
3050                 /* Don't allow the guest to change a bit, #GP */
3051                 if (!msr->host_initiated && (data ^ msr_entry.data))
3052                         return 1;
3053
3054                 svm->msr_decfg = data;
3055                 break;
3056         }
3057         default:
3058                 return kvm_set_msr_common(vcpu, msr);
3059         }
3060         return 0;
3061 }
3062
3063 static int msr_interception(struct kvm_vcpu *vcpu)
3064 {
3065         if (to_svm(vcpu)->vmcb->control.exit_info_1)
3066                 return kvm_emulate_wrmsr(vcpu);
3067         else
3068                 return kvm_emulate_rdmsr(vcpu);
3069 }
3070
3071 static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3072 {
3073         kvm_make_request(KVM_REQ_EVENT, vcpu);
3074         svm_clear_vintr(to_svm(vcpu));
3075
3076         /*
3077          * For AVIC, the only reason to end up here is ExtINTs.
3078          * In this case AVIC was temporarily disabled for
3079          * requesting the IRQ window and we have to re-enable it.
3080          */
3081         kvm_request_apicv_update(vcpu->kvm, true, APICV_INHIBIT_REASON_IRQWIN);
3082
3083         ++vcpu->stat.irq_window_exits;
3084         return 1;
3085 }
3086
3087 static int pause_interception(struct kvm_vcpu *vcpu)
3088 {
3089         bool in_kernel;
3090
3091         /*
3092          * CPL is not made available for an SEV-ES guest, therefore
3093          * vcpu->arch.preempted_in_kernel can never be true.  Just
3094          * set in_kernel to false as well.
3095          */
3096         in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3097
3098         if (!kvm_pause_in_guest(vcpu->kvm))
3099                 grow_ple_window(vcpu);
3100
3101         kvm_vcpu_on_spin(vcpu, in_kernel);
3102         return kvm_skip_emulated_instruction(vcpu);
3103 }
3104
3105 static int invpcid_interception(struct kvm_vcpu *vcpu)
3106 {
3107         struct vcpu_svm *svm = to_svm(vcpu);
3108         unsigned long type;
3109         gva_t gva;
3110
3111         if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3112                 kvm_queue_exception(vcpu, UD_VECTOR);
3113                 return 1;
3114         }
3115
3116         /*
3117          * For an INVPCID intercept:
3118          * EXITINFO1 provides the linear address of the memory operand.
3119          * EXITINFO2 provides the contents of the register operand.
3120          */
3121         type = svm->vmcb->control.exit_info_2;
3122         gva = svm->vmcb->control.exit_info_1;
3123
3124         if (type > 3) {
3125                 kvm_inject_gp(vcpu, 0);
3126                 return 1;
3127         }
3128
3129         return kvm_handle_invpcid(vcpu, type, gva);
3130 }
3131
3132 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3133         [SVM_EXIT_READ_CR0]                     = cr_interception,
3134         [SVM_EXIT_READ_CR3]                     = cr_interception,
3135         [SVM_EXIT_READ_CR4]                     = cr_interception,
3136         [SVM_EXIT_READ_CR8]                     = cr_interception,
3137         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
3138         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
3139         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
3140         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
3141         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
3142         [SVM_EXIT_READ_DR0]                     = dr_interception,
3143         [SVM_EXIT_READ_DR1]                     = dr_interception,
3144         [SVM_EXIT_READ_DR2]                     = dr_interception,
3145         [SVM_EXIT_READ_DR3]                     = dr_interception,
3146         [SVM_EXIT_READ_DR4]                     = dr_interception,
3147         [SVM_EXIT_READ_DR5]                     = dr_interception,
3148         [SVM_EXIT_READ_DR6]                     = dr_interception,
3149         [SVM_EXIT_READ_DR7]                     = dr_interception,
3150         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
3151         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
3152         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
3153         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
3154         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
3155         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
3156         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
3157         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
3158         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
3159         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
3160         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
3161         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
3162         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
3163         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
3164         [SVM_EXIT_EXCP_BASE + GP_VECTOR]        = gp_interception,
3165         [SVM_EXIT_INTR]                         = intr_interception,
3166         [SVM_EXIT_NMI]                          = nmi_interception,
3167         [SVM_EXIT_SMI]                          = smi_interception,
3168         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
3169         [SVM_EXIT_RDPMC]                        = kvm_emulate_rdpmc,
3170         [SVM_EXIT_CPUID]                        = kvm_emulate_cpuid,
3171         [SVM_EXIT_IRET]                         = iret_interception,
3172         [SVM_EXIT_INVD]                         = kvm_emulate_invd,
3173         [SVM_EXIT_PAUSE]                        = pause_interception,
3174         [SVM_EXIT_HLT]                          = kvm_emulate_halt,
3175         [SVM_EXIT_INVLPG]                       = invlpg_interception,
3176         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
3177         [SVM_EXIT_IOIO]                         = io_interception,
3178         [SVM_EXIT_MSR]                          = msr_interception,
3179         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
3180         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
3181         [SVM_EXIT_VMRUN]                        = vmrun_interception,
3182         [SVM_EXIT_VMMCALL]                      = kvm_emulate_hypercall,
3183         [SVM_EXIT_VMLOAD]                       = vmload_interception,
3184         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
3185         [SVM_EXIT_STGI]                         = stgi_interception,
3186         [SVM_EXIT_CLGI]                         = clgi_interception,
3187         [SVM_EXIT_SKINIT]                       = skinit_interception,
3188         [SVM_EXIT_RDTSCP]                       = kvm_handle_invalid_op,
3189         [SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3190         [SVM_EXIT_MONITOR]                      = kvm_emulate_monitor,
3191         [SVM_EXIT_MWAIT]                        = kvm_emulate_mwait,
3192         [SVM_EXIT_XSETBV]                       = kvm_emulate_xsetbv,
3193         [SVM_EXIT_RDPRU]                        = kvm_handle_invalid_op,
3194         [SVM_EXIT_EFER_WRITE_TRAP]              = efer_trap,
3195         [SVM_EXIT_CR0_WRITE_TRAP]               = cr_trap,
3196         [SVM_EXIT_CR4_WRITE_TRAP]               = cr_trap,
3197         [SVM_EXIT_CR8_WRITE_TRAP]               = cr_trap,
3198         [SVM_EXIT_INVPCID]                      = invpcid_interception,
3199         [SVM_EXIT_NPF]                          = npf_interception,
3200         [SVM_EXIT_RSM]                          = rsm_interception,
3201         [SVM_EXIT_AVIC_INCOMPLETE_IPI]          = avic_incomplete_ipi_interception,
3202         [SVM_EXIT_AVIC_UNACCELERATED_ACCESS]    = avic_unaccelerated_access_interception,
3203         [SVM_EXIT_VMGEXIT]                      = sev_handle_vmgexit,
3204 };
3205
3206 static void dump_vmcb(struct kvm_vcpu *vcpu)
3207 {
3208         struct vcpu_svm *svm = to_svm(vcpu);
3209         struct vmcb_control_area *control = &svm->vmcb->control;
3210         struct vmcb_save_area *save = &svm->vmcb->save;
3211         struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3212
3213         if (!dump_invalid_vmcb) {
3214                 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3215                 return;
3216         }
3217
3218         pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3219                svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3220         pr_err("VMCB Control Area:\n");
3221         pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3222         pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3223         pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3224         pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3225         pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3226         pr_err("%-20s%08x %08x\n", "intercepts:",
3227               control->intercepts[INTERCEPT_WORD3],
3228                control->intercepts[INTERCEPT_WORD4]);
3229         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3230         pr_err("%-20s%d\n", "pause filter threshold:",
3231                control->pause_filter_thresh);
3232         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3233         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3234         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3235         pr_err("%-20s%d\n", "asid:", control->asid);
3236         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3237         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3238         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3239         pr_err("%-20s%08x\n", "int_state:", control->int_state);
3240         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3241         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3242         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3243         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3244         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3245         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3246         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3247         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3248         pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3249         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3250         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3251         pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3252         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3253         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3254         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3255         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3256         pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3257         pr_err("VMCB State Save Area:\n");
3258         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3259                "es:",
3260                save->es.selector, save->es.attrib,
3261                save->es.limit, save->es.base);
3262         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3263                "cs:",
3264                save->cs.selector, save->cs.attrib,
3265                save->cs.limit, save->cs.base);
3266         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3267                "ss:",
3268                save->ss.selector, save->ss.attrib,
3269                save->ss.limit, save->ss.base);
3270         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3271                "ds:",
3272                save->ds.selector, save->ds.attrib,
3273                save->ds.limit, save->ds.base);
3274         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3275                "fs:",
3276                save01->fs.selector, save01->fs.attrib,
3277                save01->fs.limit, save01->fs.base);
3278         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3279                "gs:",
3280                save01->gs.selector, save01->gs.attrib,
3281                save01->gs.limit, save01->gs.base);
3282         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3283                "gdtr:",
3284                save->gdtr.selector, save->gdtr.attrib,
3285                save->gdtr.limit, save->gdtr.base);
3286         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3287                "ldtr:",
3288                save01->ldtr.selector, save01->ldtr.attrib,
3289                save01->ldtr.limit, save01->ldtr.base);
3290         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3291                "idtr:",
3292                save->idtr.selector, save->idtr.attrib,
3293                save->idtr.limit, save->idtr.base);
3294         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3295                "tr:",
3296                save01->tr.selector, save01->tr.attrib,
3297                save01->tr.limit, save01->tr.base);
3298         pr_err("cpl:            %d                efer:         %016llx\n",
3299                 save->cpl, save->efer);
3300         pr_err("%-15s %016llx %-13s %016llx\n",
3301                "cr0:", save->cr0, "cr2:", save->cr2);
3302         pr_err("%-15s %016llx %-13s %016llx\n",
3303                "cr3:", save->cr3, "cr4:", save->cr4);
3304         pr_err("%-15s %016llx %-13s %016llx\n",
3305                "dr6:", save->dr6, "dr7:", save->dr7);
3306         pr_err("%-15s %016llx %-13s %016llx\n",
3307                "rip:", save->rip, "rflags:", save->rflags);
3308         pr_err("%-15s %016llx %-13s %016llx\n",
3309                "rsp:", save->rsp, "rax:", save->rax);
3310         pr_err("%-15s %016llx %-13s %016llx\n",
3311                "star:", save01->star, "lstar:", save01->lstar);
3312         pr_err("%-15s %016llx %-13s %016llx\n",
3313                "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3314         pr_err("%-15s %016llx %-13s %016llx\n",
3315                "kernel_gs_base:", save01->kernel_gs_base,
3316                "sysenter_cs:", save01->sysenter_cs);
3317         pr_err("%-15s %016llx %-13s %016llx\n",
3318                "sysenter_esp:", save01->sysenter_esp,
3319                "sysenter_eip:", save01->sysenter_eip);
3320         pr_err("%-15s %016llx %-13s %016llx\n",
3321                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3322         pr_err("%-15s %016llx %-13s %016llx\n",
3323                "br_from:", save->br_from, "br_to:", save->br_to);
3324         pr_err("%-15s %016llx %-13s %016llx\n",
3325                "excp_from:", save->last_excp_from,
3326                "excp_to:", save->last_excp_to);
3327 }
3328
3329 static bool svm_check_exit_valid(struct kvm_vcpu *vcpu, u64 exit_code)
3330 {
3331         return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3332                 svm_exit_handlers[exit_code]);
3333 }
3334
3335 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3336 {
3337         vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3338         dump_vmcb(vcpu);
3339         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3340         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3341         vcpu->run->internal.ndata = 2;
3342         vcpu->run->internal.data[0] = exit_code;
3343         vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3344         return 0;
3345 }
3346
3347 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3348 {
3349         if (!svm_check_exit_valid(vcpu, exit_code))
3350                 return svm_handle_invalid_exit(vcpu, exit_code);
3351
3352 #ifdef CONFIG_RETPOLINE
3353         if (exit_code == SVM_EXIT_MSR)
3354                 return msr_interception(vcpu);
3355         else if (exit_code == SVM_EXIT_VINTR)
3356                 return interrupt_window_interception(vcpu);
3357         else if (exit_code == SVM_EXIT_INTR)
3358                 return intr_interception(vcpu);
3359         else if (exit_code == SVM_EXIT_HLT)
3360                 return kvm_emulate_halt(vcpu);
3361         else if (exit_code == SVM_EXIT_NPF)
3362                 return npf_interception(vcpu);
3363 #endif
3364         return svm_exit_handlers[exit_code](vcpu);
3365 }
3366
3367 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3368                               u64 *info1, u64 *info2,
3369                               u32 *intr_info, u32 *error_code)
3370 {
3371         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3372
3373         *reason = control->exit_code;
3374         *info1 = control->exit_info_1;
3375         *info2 = control->exit_info_2;
3376         *intr_info = control->exit_int_info;
3377         if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3378             (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3379                 *error_code = control->exit_int_info_err;
3380         else
3381                 *error_code = 0;
3382 }
3383
3384 static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3385 {
3386         struct vcpu_svm *svm = to_svm(vcpu);
3387         struct kvm_run *kvm_run = vcpu->run;
3388         u32 exit_code = svm->vmcb->control.exit_code;
3389
3390         trace_kvm_exit(vcpu, KVM_ISA_SVM);
3391
3392         /* SEV-ES guests must use the CR write traps to track CR registers. */
3393         if (!sev_es_guest(vcpu->kvm)) {
3394                 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3395                         vcpu->arch.cr0 = svm->vmcb->save.cr0;
3396                 if (npt_enabled)
3397                         vcpu->arch.cr3 = svm->vmcb->save.cr3;
3398         }
3399
3400         if (is_guest_mode(vcpu)) {
3401                 int vmexit;
3402
3403                 trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3404
3405                 vmexit = nested_svm_exit_special(svm);
3406
3407                 if (vmexit == NESTED_EXIT_CONTINUE)
3408                         vmexit = nested_svm_exit_handled(svm);
3409
3410                 if (vmexit == NESTED_EXIT_DONE)
3411                         return 1;
3412         }
3413
3414         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3415                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3416                 kvm_run->fail_entry.hardware_entry_failure_reason
3417                         = svm->vmcb->control.exit_code;
3418                 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3419                 dump_vmcb(vcpu);
3420                 return 0;
3421         }
3422
3423         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3424             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3425             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3426             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3427                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3428                        "exit_code 0x%x\n",
3429                        __func__, svm->vmcb->control.exit_int_info,
3430                        exit_code);
3431
3432         if (exit_fastpath != EXIT_FASTPATH_NONE)
3433                 return 1;
3434
3435         return svm_invoke_exit_handler(vcpu, exit_code);
3436 }
3437
3438 static void reload_tss(struct kvm_vcpu *vcpu)
3439 {
3440         struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3441
3442         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3443         load_TR_desc();
3444 }
3445
3446 static void pre_svm_run(struct kvm_vcpu *vcpu)
3447 {
3448         struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3449         struct vcpu_svm *svm = to_svm(vcpu);
3450
3451         /*
3452          * If the previous vmrun of the vmcb occurred on a different physical
3453          * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3454          * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3455          */
3456         if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3457                 svm->current_vmcb->asid_generation = 0;
3458                 vmcb_mark_all_dirty(svm->vmcb);
3459                 svm->current_vmcb->cpu = vcpu->cpu;
3460         }
3461
3462         if (sev_guest(vcpu->kvm))
3463                 return pre_sev_run(svm, vcpu->cpu);
3464
3465         /* FIXME: handle wraparound of asid_generation */
3466         if (svm->current_vmcb->asid_generation != sd->asid_generation)
3467                 new_asid(svm, sd);
3468 }
3469
3470 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3471 {
3472         struct vcpu_svm *svm = to_svm(vcpu);
3473
3474         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3475         vcpu->arch.hflags |= HF_NMI_MASK;
3476         if (!sev_es_guest(vcpu->kvm))
3477                 svm_set_intercept(svm, INTERCEPT_IRET);
3478         ++vcpu->stat.nmi_injections;
3479 }
3480
3481 static void svm_set_irq(struct kvm_vcpu *vcpu)
3482 {
3483         struct vcpu_svm *svm = to_svm(vcpu);
3484
3485         BUG_ON(!(gif_set(svm)));
3486
3487         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3488         ++vcpu->stat.irq_injections;
3489
3490         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3491                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3492 }
3493
3494 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3495 {
3496         struct vcpu_svm *svm = to_svm(vcpu);
3497
3498         /*
3499          * SEV-ES guests must always keep the CR intercepts cleared. CR
3500          * tracking is done using the CR write traps.
3501          */
3502         if (sev_es_guest(vcpu->kvm))
3503                 return;
3504
3505         if (nested_svm_virtualize_tpr(vcpu))
3506                 return;
3507
3508         svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3509
3510         if (irr == -1)
3511                 return;
3512
3513         if (tpr >= irr)
3514                 svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3515 }
3516
3517 bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3518 {
3519         struct vcpu_svm *svm = to_svm(vcpu);
3520         struct vmcb *vmcb = svm->vmcb;
3521         bool ret;
3522
3523         if (!gif_set(svm))
3524                 return true;
3525
3526         if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3527                 return false;
3528
3529         ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
3530               (vcpu->arch.hflags & HF_NMI_MASK);
3531
3532         return ret;
3533 }
3534
3535 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3536 {
3537         struct vcpu_svm *svm = to_svm(vcpu);
3538         if (svm->nested.nested_run_pending)
3539                 return -EBUSY;
3540
3541         /* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3542         if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3543                 return -EBUSY;
3544
3545         return !svm_nmi_blocked(vcpu);
3546 }
3547
3548 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3549 {
3550         return !!(vcpu->arch.hflags & HF_NMI_MASK);
3551 }
3552
3553 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3554 {
3555         struct vcpu_svm *svm = to_svm(vcpu);
3556
3557         if (masked) {
3558                 vcpu->arch.hflags |= HF_NMI_MASK;
3559                 if (!sev_es_guest(vcpu->kvm))
3560                         svm_set_intercept(svm, INTERCEPT_IRET);
3561         } else {
3562                 vcpu->arch.hflags &= ~HF_NMI_MASK;
3563                 if (!sev_es_guest(vcpu->kvm))
3564                         svm_clr_intercept(svm, INTERCEPT_IRET);
3565         }
3566 }
3567
3568 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3569 {
3570         struct vcpu_svm *svm = to_svm(vcpu);
3571         struct vmcb *vmcb = svm->vmcb;
3572
3573         if (!gif_set(svm))
3574                 return true;
3575
3576         if (sev_es_guest(vcpu->kvm)) {
3577                 /*
3578                  * SEV-ES guests to not expose RFLAGS. Use the VMCB interrupt mask
3579                  * bit to determine the state of the IF flag.
3580                  */
3581                 if (!(vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK))
3582                         return true;
3583         } else if (is_guest_mode(vcpu)) {
3584                 /* As long as interrupts are being delivered...  */
3585                 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3586                     ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3587                     : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3588                         return true;
3589
3590                 /* ... vmexits aren't blocked by the interrupt shadow  */
3591                 if (nested_exit_on_intr(svm))
3592                         return false;
3593         } else {
3594                 if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3595                         return true;
3596         }
3597
3598         return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3599 }
3600
3601 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3602 {
3603         struct vcpu_svm *svm = to_svm(vcpu);
3604         if (svm->nested.nested_run_pending)
3605                 return -EBUSY;
3606
3607         /*
3608          * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3609          * e.g. if the IRQ arrived asynchronously after checking nested events.
3610          */
3611         if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3612                 return -EBUSY;
3613
3614         return !svm_interrupt_blocked(vcpu);
3615 }
3616
3617 static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3618 {
3619         struct vcpu_svm *svm = to_svm(vcpu);
3620
3621         /*
3622          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3623          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3624          * get that intercept, this function will be called again though and
3625          * we'll get the vintr intercept. However, if the vGIF feature is
3626          * enabled, the STGI interception will not occur. Enable the irq
3627          * window under the assumption that the hardware will set the GIF.
3628          */
3629         if (vgif_enabled(svm) || gif_set(svm)) {
3630                 /*
3631                  * IRQ window is not needed when AVIC is enabled,
3632                  * unless we have pending ExtINT since it cannot be injected
3633                  * via AVIC. In such case, we need to temporarily disable AVIC,
3634                  * and fallback to injecting IRQ via V_IRQ.
3635                  */
3636                 kvm_request_apicv_update(vcpu->kvm, false, APICV_INHIBIT_REASON_IRQWIN);
3637                 svm_set_vintr(svm);
3638         }
3639 }
3640
3641 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3642 {
3643         struct vcpu_svm *svm = to_svm(vcpu);
3644
3645         if ((vcpu->arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) == HF_NMI_MASK)
3646                 return; /* IRET will cause a vm exit */
3647
3648         if (!gif_set(svm)) {
3649                 if (vgif_enabled(svm))
3650                         svm_set_intercept(svm, INTERCEPT_STGI);
3651                 return; /* STGI will cause a vm exit */
3652         }
3653
3654         /*
3655          * Something prevents NMI from been injected. Single step over possible
3656          * problem (IRET or exception injection or interrupt shadow)
3657          */
3658         svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3659         svm->nmi_singlestep = true;
3660         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3661 }
3662
3663 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3664 {
3665         return 0;
3666 }
3667
3668 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
3669 {
3670         return 0;
3671 }
3672
3673 void svm_flush_tlb(struct kvm_vcpu *vcpu)
3674 {
3675         struct vcpu_svm *svm = to_svm(vcpu);
3676
3677         /*
3678          * Flush only the current ASID even if the TLB flush was invoked via
3679          * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3680          * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3681          * unconditionally does a TLB flush on both nested VM-Enter and nested
3682          * VM-Exit (via kvm_mmu_reset_context()).
3683          */
3684         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3685                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3686         else
3687                 svm->current_vmcb->asid_generation--;
3688 }
3689
3690 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3691 {
3692         struct vcpu_svm *svm = to_svm(vcpu);
3693
3694         invlpga(gva, svm->vmcb->control.asid);
3695 }
3696
3697 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3698 {
3699         struct vcpu_svm *svm = to_svm(vcpu);
3700
3701         if (nested_svm_virtualize_tpr(vcpu))
3702                 return;
3703
3704         if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3705                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3706                 kvm_set_cr8(vcpu, cr8);
3707         }
3708 }
3709
3710 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3711 {
3712         struct vcpu_svm *svm = to_svm(vcpu);
3713         u64 cr8;
3714
3715         if (nested_svm_virtualize_tpr(vcpu) ||
3716             kvm_vcpu_apicv_active(vcpu))
3717                 return;
3718
3719         cr8 = kvm_get_cr8(vcpu);
3720         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3721         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3722 }
3723
3724 static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
3725 {
3726         struct vcpu_svm *svm = to_svm(vcpu);
3727         u8 vector;
3728         int type;
3729         u32 exitintinfo = svm->vmcb->control.exit_int_info;
3730         unsigned int3_injected = svm->int3_injected;
3731
3732         svm->int3_injected = 0;
3733
3734         /*
3735          * If we've made progress since setting HF_IRET_MASK, we've
3736          * executed an IRET and can allow NMI injection.
3737          */
3738         if ((vcpu->arch.hflags & HF_IRET_MASK) &&
3739             (sev_es_guest(vcpu->kvm) ||
3740              kvm_rip_read(vcpu) != svm->nmi_iret_rip)) {
3741                 vcpu->arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3742                 kvm_make_request(KVM_REQ_EVENT, vcpu);
3743         }
3744
3745         vcpu->arch.nmi_injected = false;
3746         kvm_clear_exception_queue(vcpu);
3747         kvm_clear_interrupt_queue(vcpu);
3748
3749         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3750                 return;
3751
3752         kvm_make_request(KVM_REQ_EVENT, vcpu);
3753
3754         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3755         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3756
3757         switch (type) {
3758         case SVM_EXITINTINFO_TYPE_NMI:
3759                 vcpu->arch.nmi_injected = true;
3760                 break;
3761         case SVM_EXITINTINFO_TYPE_EXEPT:
3762                 /*
3763                  * Never re-inject a #VC exception.
3764                  */
3765                 if (vector == X86_TRAP_VC)
3766                         break;
3767
3768                 /*
3769                  * In case of software exceptions, do not reinject the vector,
3770                  * but re-execute the instruction instead. Rewind RIP first
3771                  * if we emulated INT3 before.
3772                  */
3773                 if (kvm_exception_is_soft(vector)) {
3774                         if (vector == BP_VECTOR && int3_injected &&
3775                             kvm_is_linear_rip(vcpu, svm->int3_rip))
3776                                 kvm_rip_write(vcpu,
3777                                               kvm_rip_read(vcpu) - int3_injected);
3778                         break;
3779                 }
3780                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3781                         u32 err = svm->vmcb->control.exit_int_info_err;
3782                         kvm_requeue_exception_e(vcpu, vector, err);
3783
3784                 } else
3785                         kvm_requeue_exception(vcpu, vector);
3786                 break;
3787         case SVM_EXITINTINFO_TYPE_INTR:
3788                 kvm_queue_interrupt(vcpu, vector, false);
3789                 break;
3790         default:
3791                 break;
3792         }
3793 }
3794
3795 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3796 {
3797         struct vcpu_svm *svm = to_svm(vcpu);
3798         struct vmcb_control_area *control = &svm->vmcb->control;
3799
3800         control->exit_int_info = control->event_inj;
3801         control->exit_int_info_err = control->event_inj_err;
3802         control->event_inj = 0;
3803         svm_complete_interrupts(vcpu);
3804 }
3805
3806 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
3807 {
3808         if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
3809             to_svm(vcpu)->vmcb->control.exit_info_1)
3810                 return handle_fastpath_set_msr_irqoff(vcpu);
3811
3812         return EXIT_FASTPATH_NONE;
3813 }
3814
3815 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
3816 {
3817         struct vcpu_svm *svm = to_svm(vcpu);
3818         unsigned long vmcb_pa = svm->current_vmcb->pa;
3819
3820         kvm_guest_enter_irqoff();
3821
3822         if (sev_es_guest(vcpu->kvm)) {
3823                 __svm_sev_es_vcpu_run(vmcb_pa);
3824         } else {
3825                 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3826
3827                 /*
3828                  * Use a single vmcb (vmcb01 because it's always valid) for
3829                  * context switching guest state via VMLOAD/VMSAVE, that way
3830                  * the state doesn't need to be copied between vmcb01 and
3831                  * vmcb02 when switching vmcbs for nested virtualization.
3832                  */
3833                 vmload(svm->vmcb01.pa);
3834                 __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs);
3835                 vmsave(svm->vmcb01.pa);
3836
3837                 vmload(__sme_page_pa(sd->save_area));
3838         }
3839
3840         kvm_guest_exit_irqoff();
3841 }
3842
3843 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
3844 {
3845         struct vcpu_svm *svm = to_svm(vcpu);
3846
3847         trace_kvm_entry(vcpu);
3848
3849         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3850         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3851         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3852
3853         /*
3854          * Disable singlestep if we're injecting an interrupt/exception.
3855          * We don't want our modified rflags to be pushed on the stack where
3856          * we might not be able to easily reset them if we disabled NMI
3857          * singlestep later.
3858          */
3859         if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
3860                 /*
3861                  * Event injection happens before external interrupts cause a
3862                  * vmexit and interrupts are disabled here, so smp_send_reschedule
3863                  * is enough to force an immediate vmexit.
3864                  */
3865                 disable_nmi_singlestep(svm);
3866                 smp_send_reschedule(vcpu->cpu);
3867         }
3868
3869         pre_svm_run(vcpu);
3870
3871         sync_lapic_to_cr8(vcpu);
3872
3873         if (unlikely(svm->asid != svm->vmcb->control.asid)) {
3874                 svm->vmcb->control.asid = svm->asid;
3875                 vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
3876         }
3877         svm->vmcb->save.cr2 = vcpu->arch.cr2;
3878
3879         svm_hv_update_vp_id(svm->vmcb, vcpu);
3880
3881         /*
3882          * Run with all-zero DR6 unless needed, so that we can get the exact cause
3883          * of a #DB.
3884          */
3885         if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3886                 svm_set_dr6(svm, vcpu->arch.dr6);
3887         else
3888                 svm_set_dr6(svm, DR6_ACTIVE_LOW);
3889
3890         clgi();
3891         kvm_load_guest_xsave_state(vcpu);
3892
3893         kvm_wait_lapic_expire(vcpu);
3894
3895         /*
3896          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
3897          * it's non-zero. Since vmentry is serialising on affected CPUs, there
3898          * is no need to worry about the conditional branch over the wrmsr
3899          * being speculatively taken.
3900          */
3901         if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3902                 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
3903
3904         svm_vcpu_enter_exit(vcpu);
3905
3906         /*
3907          * We do not use IBRS in the kernel. If this vCPU has used the
3908          * SPEC_CTRL MSR it may have left it on; save the value and
3909          * turn it off. This is much more efficient than blindly adding
3910          * it to the atomic save/restore list. Especially as the former
3911          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
3912          *
3913          * For non-nested case:
3914          * If the L01 MSR bitmap does not intercept the MSR, then we need to
3915          * save it.
3916          *
3917          * For nested case:
3918          * If the L02 MSR bitmap does not intercept the MSR, then we need to
3919          * save it.
3920          */
3921         if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) &&
3922             unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
3923                 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
3924
3925         if (!sev_es_guest(vcpu->kvm))
3926                 reload_tss(vcpu);
3927
3928         if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3929                 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
3930
3931         if (!sev_es_guest(vcpu->kvm)) {
3932                 vcpu->arch.cr2 = svm->vmcb->save.cr2;
3933                 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3934                 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3935                 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3936         }
3937
3938         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3939                 kvm_before_interrupt(vcpu);
3940
3941         kvm_load_host_xsave_state(vcpu);
3942         stgi();
3943
3944         /* Any pending NMI will happen here */
3945
3946         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3947                 kvm_after_interrupt(vcpu);
3948
3949         sync_cr8_to_lapic(vcpu);
3950
3951         svm->next_rip = 0;
3952         if (is_guest_mode(vcpu)) {
3953                 nested_sync_control_from_vmcb02(svm);
3954
3955                 /* Track VMRUNs that have made past consistency checking */
3956                 if (svm->nested.nested_run_pending &&
3957                     svm->vmcb->control.exit_code != SVM_EXIT_ERR)
3958                         ++vcpu->stat.nested_run;
3959
3960                 svm->nested.nested_run_pending = 0;
3961         }
3962
3963         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3964         vmcb_mark_all_clean(svm->vmcb);
3965
3966         /* if exit due to PF check for async PF */
3967         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3968                 vcpu->arch.apf.host_apf_flags =
3969                         kvm_read_and_reset_apf_flags();
3970
3971         if (npt_enabled)
3972                 kvm_register_clear_available(vcpu, VCPU_EXREG_PDPTR);
3973
3974         /*
3975          * We need to handle MC intercepts here before the vcpu has a chance to
3976          * change the physical cpu
3977          */
3978         if (unlikely(svm->vmcb->control.exit_code ==
3979                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
3980                 svm_handle_mce(vcpu);
3981
3982         svm_complete_interrupts(vcpu);
3983
3984         if (is_guest_mode(vcpu))
3985                 return EXIT_FASTPATH_NONE;
3986
3987         return svm_exit_handlers_fastpath(vcpu);
3988 }
3989
3990 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
3991                              int root_level)
3992 {
3993         struct vcpu_svm *svm = to_svm(vcpu);
3994         unsigned long cr3;
3995
3996         if (npt_enabled) {
3997                 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
3998                 vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3999
4000                 hv_track_root_tdp(vcpu, root_hpa);
4001
4002                 /* Loading L2's CR3 is handled by enter_svm_guest_mode.  */
4003                 if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
4004                         return;
4005                 cr3 = vcpu->arch.cr3;
4006         } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) {
4007                 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4008         } else {
4009                 /* PCID in the guest should be impossible with a 32-bit MMU. */
4010                 WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4011                 cr3 = root_hpa;
4012         }
4013
4014         svm->vmcb->save.cr3 = cr3;
4015         vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4016 }
4017
4018 static int is_disabled(void)
4019 {
4020         u64 vm_cr;
4021
4022         rdmsrl(MSR_VM_CR, vm_cr);
4023         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
4024                 return 1;
4025
4026         return 0;
4027 }
4028
4029 static void
4030 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4031 {
4032         /*
4033          * Patch in the VMMCALL instruction:
4034          */
4035         hypercall[0] = 0x0f;
4036         hypercall[1] = 0x01;
4037         hypercall[2] = 0xd9;
4038 }
4039
4040 static int __init svm_check_processor_compat(void)
4041 {
4042         return 0;
4043 }
4044
4045 static bool svm_cpu_has_accelerated_tpr(void)
4046 {
4047         return false;
4048 }
4049
4050 /*
4051  * The kvm parameter can be NULL (module initialization, or invocation before
4052  * VM creation). Be sure to check the kvm parameter before using it.
4053  */
4054 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4055 {
4056         switch (index) {
4057         case MSR_IA32_MCG_EXT_CTL:
4058         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
4059                 return false;
4060         case MSR_IA32_SMBASE:
4061                 /* SEV-ES guests do not support SMM, so report false */
4062                 if (kvm && sev_es_guest(kvm))
4063                         return false;
4064                 break;
4065         default:
4066                 break;
4067         }
4068
4069         return true;
4070 }
4071
4072 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4073 {
4074         return 0;
4075 }
4076
4077 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4078 {
4079         struct vcpu_svm *svm = to_svm(vcpu);
4080         struct kvm_cpuid_entry2 *best;
4081
4082         vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
4083                                     boot_cpu_has(X86_FEATURE_XSAVE) &&
4084                                     boot_cpu_has(X86_FEATURE_XSAVES);
4085
4086         /* Update nrips enabled cache */
4087         svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) &&
4088                              guest_cpuid_has(vcpu, X86_FEATURE_NRIPS);
4089
4090         svm->tsc_scaling_enabled = tsc_scaling && guest_cpuid_has(vcpu, X86_FEATURE_TSCRATEMSR);
4091
4092         svm_recalc_instruction_intercepts(vcpu, svm);
4093
4094         /* For sev guests, the memory encryption bit is not reserved in CR3.  */
4095         if (sev_guest(vcpu->kvm)) {
4096                 best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);
4097                 if (best)
4098                         vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
4099         }
4100
4101         if (kvm_vcpu_apicv_active(vcpu)) {
4102                 /*
4103                  * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
4104                  * is exposed to the guest, disable AVIC.
4105                  */
4106                 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
4107                         kvm_request_apicv_update(vcpu->kvm, false,
4108                                                  APICV_INHIBIT_REASON_X2APIC);
4109
4110                 /*
4111                  * Currently, AVIC does not work with nested virtualization.
4112                  * So, we disable AVIC when cpuid for SVM is set in the L1 guest.
4113                  */
4114                 if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4115                         kvm_request_apicv_update(vcpu->kvm, false,
4116                                                  APICV_INHIBIT_REASON_NESTED);
4117         }
4118         init_vmcb_after_set_cpuid(vcpu);
4119 }
4120
4121 static bool svm_has_wbinvd_exit(void)
4122 {
4123         return true;
4124 }
4125
4126 #define PRE_EX(exit)  { .exit_code = (exit), \
4127                         .stage = X86_ICPT_PRE_EXCEPT, }
4128 #define POST_EX(exit) { .exit_code = (exit), \
4129                         .stage = X86_ICPT_POST_EXCEPT, }
4130 #define POST_MEM(exit) { .exit_code = (exit), \
4131                         .stage = X86_ICPT_POST_MEMACCESS, }
4132
4133 static const struct __x86_intercept {
4134         u32 exit_code;
4135         enum x86_intercept_stage stage;
4136 } x86_intercept_map[] = {
4137         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
4138         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
4139         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
4140         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
4141         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
4142         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
4143         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
4144         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
4145         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
4146         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
4147         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
4148         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
4149         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
4150         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
4151         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
4152         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
4153         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
4154         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
4155         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
4156         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
4157         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
4158         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
4159         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
4160         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
4161         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
4162         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
4163         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
4164         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
4165         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
4166         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
4167         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
4168         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
4169         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
4170         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
4171         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
4172         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
4173         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
4174         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
4175         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
4176         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
4177         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
4178         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
4179         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
4180         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
4181         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
4182         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
4183         [x86_intercept_xsetbv]          = PRE_EX(SVM_EXIT_XSETBV),
4184 };
4185
4186 #undef PRE_EX
4187 #undef POST_EX
4188 #undef POST_MEM
4189
4190 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4191                                struct x86_instruction_info *info,
4192                                enum x86_intercept_stage stage,
4193                                struct x86_exception *exception)
4194 {
4195         struct vcpu_svm *svm = to_svm(vcpu);
4196         int vmexit, ret = X86EMUL_CONTINUE;
4197         struct __x86_intercept icpt_info;
4198         struct vmcb *vmcb = svm->vmcb;
4199
4200         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4201                 goto out;
4202
4203         icpt_info = x86_intercept_map[info->intercept];
4204
4205         if (stage != icpt_info.stage)
4206                 goto out;
4207
4208         switch (icpt_info.exit_code) {
4209         case SVM_EXIT_READ_CR0:
4210                 if (info->intercept == x86_intercept_cr_read)
4211                         icpt_info.exit_code += info->modrm_reg;
4212                 break;
4213         case SVM_EXIT_WRITE_CR0: {
4214                 unsigned long cr0, val;
4215
4216                 if (info->intercept == x86_intercept_cr_write)
4217                         icpt_info.exit_code += info->modrm_reg;
4218
4219                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4220                     info->intercept == x86_intercept_clts)
4221                         break;
4222
4223                 if (!(vmcb_is_intercept(&svm->nested.ctl,
4224                                         INTERCEPT_SELECTIVE_CR0)))
4225                         break;
4226
4227                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4228                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4229
4230                 if (info->intercept == x86_intercept_lmsw) {
4231                         cr0 &= 0xfUL;
4232                         val &= 0xfUL;
4233                         /* lmsw can't clear PE - catch this here */
4234                         if (cr0 & X86_CR0_PE)
4235                                 val |= X86_CR0_PE;
4236                 }
4237
4238                 if (cr0 ^ val)
4239                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4240
4241                 break;
4242         }
4243         case SVM_EXIT_READ_DR0:
4244         case SVM_EXIT_WRITE_DR0:
4245                 icpt_info.exit_code += info->modrm_reg;
4246                 break;
4247         case SVM_EXIT_MSR:
4248                 if (info->intercept == x86_intercept_wrmsr)
4249                         vmcb->control.exit_info_1 = 1;
4250                 else
4251                         vmcb->control.exit_info_1 = 0;
4252                 break;
4253         case SVM_EXIT_PAUSE:
4254                 /*
4255                  * We get this for NOP only, but pause
4256                  * is rep not, check this here
4257                  */
4258                 if (info->rep_prefix != REPE_PREFIX)
4259                         goto out;
4260                 break;
4261         case SVM_EXIT_IOIO: {
4262                 u64 exit_info;
4263                 u32 bytes;
4264
4265                 if (info->intercept == x86_intercept_in ||
4266                     info->intercept == x86_intercept_ins) {
4267                         exit_info = ((info->src_val & 0xffff) << 16) |
4268                                 SVM_IOIO_TYPE_MASK;
4269                         bytes = info->dst_bytes;
4270                 } else {
4271                         exit_info = (info->dst_val & 0xffff) << 16;
4272                         bytes = info->src_bytes;
4273                 }
4274
4275                 if (info->intercept == x86_intercept_outs ||
4276                     info->intercept == x86_intercept_ins)
4277                         exit_info |= SVM_IOIO_STR_MASK;
4278
4279                 if (info->rep_prefix)
4280                         exit_info |= SVM_IOIO_REP_MASK;
4281
4282                 bytes = min(bytes, 4u);
4283
4284                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4285
4286                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4287
4288                 vmcb->control.exit_info_1 = exit_info;
4289                 vmcb->control.exit_info_2 = info->next_rip;
4290
4291                 break;
4292         }
4293         default:
4294                 break;
4295         }
4296
4297         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4298         if (static_cpu_has(X86_FEATURE_NRIPS))
4299                 vmcb->control.next_rip  = info->next_rip;
4300         vmcb->control.exit_code = icpt_info.exit_code;
4301         vmexit = nested_svm_exit_handled(svm);
4302
4303         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4304                                            : X86EMUL_CONTINUE;
4305
4306 out:
4307         return ret;
4308 }
4309
4310 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4311 {
4312 }
4313
4314 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4315 {
4316         if (!kvm_pause_in_guest(vcpu->kvm))
4317                 shrink_ple_window(vcpu);
4318 }
4319
4320 static void svm_setup_mce(struct kvm_vcpu *vcpu)
4321 {
4322         /* [63:9] are reserved. */
4323         vcpu->arch.mcg_cap &= 0x1ff;
4324 }
4325
4326 bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4327 {
4328         struct vcpu_svm *svm = to_svm(vcpu);
4329
4330         /* Per APM Vol.2 15.22.2 "Response to SMI" */
4331         if (!gif_set(svm))
4332                 return true;
4333
4334         return is_smm(vcpu);
4335 }
4336
4337 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4338 {
4339         struct vcpu_svm *svm = to_svm(vcpu);
4340         if (svm->nested.nested_run_pending)
4341                 return -EBUSY;
4342
4343         /* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4344         if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4345                 return -EBUSY;
4346
4347         return !svm_smi_blocked(vcpu);
4348 }
4349
4350 static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
4351 {
4352         struct vcpu_svm *svm = to_svm(vcpu);
4353         struct kvm_host_map map_save;
4354         int ret;
4355
4356         if (!is_guest_mode(vcpu))
4357                 return 0;
4358
4359         /* FED8h - SVM Guest */
4360         put_smstate(u64, smstate, 0x7ed8, 1);
4361         /* FEE0h - SVM Guest VMCB Physical Address */
4362         put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa);
4363
4364         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4365         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4366         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4367
4368         ret = nested_svm_vmexit(svm);
4369         if (ret)
4370                 return ret;
4371
4372         /*
4373          * KVM uses VMCB01 to store L1 host state while L2 runs but
4374          * VMCB01 is going to be used during SMM and thus the state will
4375          * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4376          * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4377          * format of the area is identical to guest save area offsetted
4378          * by 0x400 (matches the offset of 'struct vmcb_save_area'
4379          * within 'struct vmcb'). Note: HSAVE area may also be used by
4380          * L1 hypervisor to save additional host context (e.g. KVM does
4381          * that, see svm_prepare_guest_switch()) which must be
4382          * preserved.
4383          */
4384         if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr),
4385                          &map_save) == -EINVAL)
4386                 return 1;
4387
4388         BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4389
4390         svm_copy_vmrun_state(map_save.hva + 0x400,
4391                              &svm->vmcb01.ptr->save);
4392
4393         kvm_vcpu_unmap(vcpu, &map_save, true);
4394         return 0;
4395 }
4396
4397 static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
4398 {
4399         struct vcpu_svm *svm = to_svm(vcpu);
4400         struct kvm_host_map map, map_save;
4401         u64 saved_efer, vmcb12_gpa;
4402         struct vmcb *vmcb12;
4403         int ret;
4404
4405         if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4406                 return 0;
4407
4408         /* Non-zero if SMI arrived while vCPU was in guest mode. */
4409         if (!GET_SMSTATE(u64, smstate, 0x7ed8))
4410                 return 0;
4411
4412         if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4413                 return 1;
4414
4415         saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0);
4416         if (!(saved_efer & EFER_SVME))
4417                 return 1;
4418
4419         vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0);
4420         if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL)
4421                 return 1;
4422
4423         ret = 1;
4424         if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save) == -EINVAL)
4425                 goto unmap_map;
4426
4427         if (svm_allocate_nested(svm))
4428                 goto unmap_save;
4429
4430         /*
4431          * Restore L1 host state from L1 HSAVE area as VMCB01 was
4432          * used during SMM (see svm_enter_smm())
4433          */
4434
4435         svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4436
4437         /*
4438          * Enter the nested guest now
4439          */
4440
4441         vmcb12 = map.hva;
4442         nested_load_control_from_vmcb12(svm, &vmcb12->control);
4443         ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false);
4444
4445 unmap_save:
4446         kvm_vcpu_unmap(vcpu, &map_save, true);
4447 unmap_map:
4448         kvm_vcpu_unmap(vcpu, &map, true);
4449         return ret;
4450 }
4451
4452 static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4453 {
4454         struct vcpu_svm *svm = to_svm(vcpu);
4455
4456         if (!gif_set(svm)) {
4457                 if (vgif_enabled(svm))
4458                         svm_set_intercept(svm, INTERCEPT_STGI);
4459                 /* STGI will cause a vm exit */
4460         } else {
4461                 /* We must be in SMM; RSM will cause a vmexit anyway.  */
4462         }
4463 }
4464
4465 static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len)
4466 {
4467         bool smep, smap, is_user;
4468         unsigned long cr4;
4469
4470         /*
4471          * When the guest is an SEV-ES guest, emulation is not possible.
4472          */
4473         if (sev_es_guest(vcpu->kvm))
4474                 return false;
4475
4476         /*
4477          * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4478          *
4479          * Errata:
4480          * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
4481          * possible that CPU microcode implementing DecodeAssist will fail
4482          * to read bytes of instruction which caused #NPF. In this case,
4483          * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
4484          * return 0 instead of the correct guest instruction bytes.
4485          *
4486          * This happens because CPU microcode reading instruction bytes
4487          * uses a special opcode which attempts to read data using CPL=0
4488          * privileges. The microcode reads CS:RIP and if it hits a SMAP
4489          * fault, it gives up and returns no instruction bytes.
4490          *
4491          * Detection:
4492          * We reach here in case CPU supports DecodeAssist, raised #NPF and
4493          * returned 0 in GuestIntrBytes field of the VMCB.
4494          * First, errata can only be triggered in case vCPU CR4.SMAP=1.
4495          * Second, if vCPU CR4.SMEP=1, errata could only be triggered
4496          * in case vCPU CPL==3 (Because otherwise guest would have triggered
4497          * a SMEP fault instead of #NPF).
4498          * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
4499          * As most guests enable SMAP if they have also enabled SMEP, use above
4500          * logic in order to attempt minimize false-positive of detecting errata
4501          * while still preserving all cases semantic correctness.
4502          *
4503          * Workaround:
4504          * To determine what instruction the guest was executing, the hypervisor
4505          * will have to decode the instruction at the instruction pointer.
4506          *
4507          * In non SEV guest, hypervisor will be able to read the guest
4508          * memory to decode the instruction pointer when insn_len is zero
4509          * so we return true to indicate that decoding is possible.
4510          *
4511          * But in the SEV guest, the guest memory is encrypted with the
4512          * guest specific key and hypervisor will not be able to decode the
4513          * instruction pointer so we will not able to workaround it. Lets
4514          * print the error and request to kill the guest.
4515          */
4516         if (likely(!insn || insn_len))
4517                 return true;
4518
4519         /*
4520          * If RIP is invalid, go ahead with emulation which will cause an
4521          * internal error exit.
4522          */
4523         if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT))
4524                 return true;
4525
4526         cr4 = kvm_read_cr4(vcpu);
4527         smep = cr4 & X86_CR4_SMEP;
4528         smap = cr4 & X86_CR4_SMAP;
4529         is_user = svm_get_cpl(vcpu) == 3;
4530         if (smap && (!smep || is_user)) {
4531                 if (!sev_guest(vcpu->kvm))
4532                         return true;
4533
4534                 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
4535                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4536         }
4537
4538         return false;
4539 }
4540
4541 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4542 {
4543         struct vcpu_svm *svm = to_svm(vcpu);
4544
4545         /*
4546          * TODO: Last condition latch INIT signals on vCPU when
4547          * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
4548          * To properly emulate the INIT intercept,
4549          * svm_check_nested_events() should call nested_svm_vmexit()
4550          * if an INIT signal is pending.
4551          */
4552         return !gif_set(svm) ||
4553                    (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT));
4554 }
4555
4556 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4557 {
4558         if (!sev_es_guest(vcpu->kvm))
4559                 return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4560
4561         sev_vcpu_deliver_sipi_vector(vcpu, vector);
4562 }
4563
4564 static void svm_vm_destroy(struct kvm *kvm)
4565 {
4566         avic_vm_destroy(kvm);
4567         sev_vm_destroy(kvm);
4568 }
4569
4570 static int svm_vm_init(struct kvm *kvm)
4571 {
4572         if (!pause_filter_count || !pause_filter_thresh)
4573                 kvm->arch.pause_in_guest = true;
4574
4575         if (enable_apicv) {
4576                 int ret = avic_vm_init(kvm);
4577                 if (ret)
4578                         return ret;
4579         }
4580
4581         return 0;
4582 }
4583
4584 static struct kvm_x86_ops svm_x86_ops __initdata = {
4585         .name = "kvm_amd",
4586
4587         .hardware_unsetup = svm_hardware_teardown,
4588         .hardware_enable = svm_hardware_enable,
4589         .hardware_disable = svm_hardware_disable,
4590         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4591         .has_emulated_msr = svm_has_emulated_msr,
4592
4593         .vcpu_create = svm_create_vcpu,
4594         .vcpu_free = svm_free_vcpu,
4595         .vcpu_reset = svm_vcpu_reset,
4596
4597         .vm_size = sizeof(struct kvm_svm),
4598         .vm_init = svm_vm_init,
4599         .vm_destroy = svm_vm_destroy,
4600
4601         .prepare_guest_switch = svm_prepare_guest_switch,
4602         .vcpu_load = svm_vcpu_load,
4603         .vcpu_put = svm_vcpu_put,
4604         .vcpu_blocking = svm_vcpu_blocking,
4605         .vcpu_unblocking = svm_vcpu_unblocking,
4606
4607         .update_exception_bitmap = svm_update_exception_bitmap,
4608         .get_msr_feature = svm_get_msr_feature,
4609         .get_msr = svm_get_msr,
4610         .set_msr = svm_set_msr,
4611         .get_segment_base = svm_get_segment_base,
4612         .get_segment = svm_get_segment,
4613         .set_segment = svm_set_segment,
4614         .get_cpl = svm_get_cpl,
4615         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4616         .set_cr0 = svm_set_cr0,
4617         .is_valid_cr4 = svm_is_valid_cr4,
4618         .set_cr4 = svm_set_cr4,
4619         .set_efer = svm_set_efer,
4620         .get_idt = svm_get_idt,
4621         .set_idt = svm_set_idt,
4622         .get_gdt = svm_get_gdt,
4623         .set_gdt = svm_set_gdt,
4624         .set_dr7 = svm_set_dr7,
4625         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4626         .cache_reg = svm_cache_reg,
4627         .get_rflags = svm_get_rflags,
4628         .set_rflags = svm_set_rflags,
4629
4630         .tlb_flush_all = svm_flush_tlb,
4631         .tlb_flush_current = svm_flush_tlb,
4632         .tlb_flush_gva = svm_flush_tlb_gva,
4633         .tlb_flush_guest = svm_flush_tlb,
4634
4635         .run = svm_vcpu_run,
4636         .handle_exit = handle_exit,
4637         .skip_emulated_instruction = skip_emulated_instruction,
4638         .update_emulated_instruction = NULL,
4639         .set_interrupt_shadow = svm_set_interrupt_shadow,
4640         .get_interrupt_shadow = svm_get_interrupt_shadow,
4641         .patch_hypercall = svm_patch_hypercall,
4642         .set_irq = svm_set_irq,
4643         .set_nmi = svm_inject_nmi,
4644         .queue_exception = svm_queue_exception,
4645         .cancel_injection = svm_cancel_injection,
4646         .interrupt_allowed = svm_interrupt_allowed,
4647         .nmi_allowed = svm_nmi_allowed,
4648         .get_nmi_mask = svm_get_nmi_mask,
4649         .set_nmi_mask = svm_set_nmi_mask,
4650         .enable_nmi_window = svm_enable_nmi_window,
4651         .enable_irq_window = svm_enable_irq_window,
4652         .update_cr8_intercept = svm_update_cr8_intercept,
4653         .set_virtual_apic_mode = svm_set_virtual_apic_mode,
4654         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4655         .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
4656         .load_eoi_exitmap = svm_load_eoi_exitmap,
4657         .hwapic_irr_update = svm_hwapic_irr_update,
4658         .hwapic_isr_update = svm_hwapic_isr_update,
4659         .sync_pir_to_irr = kvm_lapic_find_highest_irr,
4660         .apicv_post_state_restore = avic_post_state_restore,
4661
4662         .set_tss_addr = svm_set_tss_addr,
4663         .set_identity_map_addr = svm_set_identity_map_addr,
4664         .get_mt_mask = svm_get_mt_mask,
4665
4666         .get_exit_info = svm_get_exit_info,
4667
4668         .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
4669
4670         .has_wbinvd_exit = svm_has_wbinvd_exit,
4671
4672         .get_l2_tsc_offset = svm_get_l2_tsc_offset,
4673         .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
4674         .write_tsc_offset = svm_write_tsc_offset,
4675         .write_tsc_multiplier = svm_write_tsc_multiplier,
4676
4677         .load_mmu_pgd = svm_load_mmu_pgd,
4678
4679         .check_intercept = svm_check_intercept,
4680         .handle_exit_irqoff = svm_handle_exit_irqoff,
4681
4682         .request_immediate_exit = __kvm_request_immediate_exit,
4683
4684         .sched_in = svm_sched_in,
4685
4686         .pmu_ops = &amd_pmu_ops,
4687         .nested_ops = &svm_nested_ops,
4688
4689         .deliver_posted_interrupt = svm_deliver_avic_intr,
4690         .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
4691         .update_pi_irte = svm_update_pi_irte,
4692         .setup_mce = svm_setup_mce,
4693
4694         .smi_allowed = svm_smi_allowed,
4695         .enter_smm = svm_enter_smm,
4696         .leave_smm = svm_leave_smm,
4697         .enable_smi_window = svm_enable_smi_window,
4698
4699         .mem_enc_op = svm_mem_enc_op,
4700         .mem_enc_reg_region = svm_register_enc_region,
4701         .mem_enc_unreg_region = svm_unregister_enc_region,
4702
4703         .vm_copy_enc_context_from = svm_vm_copy_asid_from,
4704         .vm_move_enc_context_from = svm_vm_migrate_from,
4705
4706         .can_emulate_instruction = svm_can_emulate_instruction,
4707
4708         .apic_init_signal_blocked = svm_apic_init_signal_blocked,
4709
4710         .msr_filter_changed = svm_msr_filter_changed,
4711         .complete_emulated_msr = svm_complete_emulated_msr,
4712
4713         .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
4714 };
4715
4716 static struct kvm_x86_init_ops svm_init_ops __initdata = {
4717         .cpu_has_kvm_support = has_svm,
4718         .disabled_by_bios = is_disabled,
4719         .hardware_setup = svm_hardware_setup,
4720         .check_processor_compatibility = svm_check_processor_compat,
4721
4722         .runtime_ops = &svm_x86_ops,
4723 };
4724
4725 static int __init svm_init(void)
4726 {
4727         __unused_size_checks();
4728
4729         return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm),
4730                         __alignof__(struct vcpu_svm), THIS_MODULE);
4731 }
4732
4733 static void __exit svm_exit(void)
4734 {
4735         kvm_exit();
4736 }
4737
4738 module_init(svm_init)
4739 module_exit(svm_exit)