arch/x86/kvm/svm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM support
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *   Avi Kivity   <avi@qumranet.com>
  13  */
  14
  15 #define pr_fmt(fmt) "SVM: " fmt
  16
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34 #include <linux/amd-iommu.h>
  35 #include <linux/hashtable.h>
  36 #include <linux/frame.h>
  37 #include <linux/psp-sev.h>
  38 #include <linux/file.h>
  39 #include <linux/pagemap.h>
  40 #include <linux/swap.h>
  41 #include <linux/rwsem.h>
  42
  43 #include <asm/apic.h>
  44 #include <asm/perf_event.h>
  45 #include <asm/tlbflush.h>
  46 #include <asm/desc.h>
  47 #include <asm/debugreg.h>
  48 #include <asm/kvm_para.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/spec-ctrl.h>
  51
  52 #include <asm/virtext.h>
  53 #include "trace.h"
  54
  55 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  56
  57 MODULE_AUTHOR("Qumranet");
  58 MODULE_LICENSE("GPL");
  59
  60 #ifdef MODULE
  61 static const struct x86_cpu_id svm_cpu_id[] = {
  62         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  63         {}
  64 };
  65 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  66 #endif
  67
  68 #define IOPM_ALLOC_ORDER 2
  69 #define MSRPM_ALLOC_ORDER 1
  70
  71 #define SEG_TYPE_LDT 2
  72 #define SEG_TYPE_BUSY_TSS16 3
  73
  74 #define SVM_FEATURE_LBRV           (1 <<  1)
  75 #define SVM_FEATURE_SVML           (1 <<  2)
  76 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  77 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  78 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  79 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  80 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  81
  82 #define SVM_AVIC_DOORBELL       0xc001011b
  83
  84 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  85 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  86 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  87
  88 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  89
  90 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  91 #define TSC_RATIO_MIN           0x0000000000000001ULL
  92 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  93
  94 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) | 0xFFF)
  95
  96 /*
  97  * 0xff is broadcast, so the max index allowed for physical APIC ID
  98  * table is 0xfe.  APIC IDs above 0xff are reserved.
  99  */
 100 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
 101
 102 #define AVIC_UNACCEL_ACCESS_WRITE_MASK          1
 103 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK         0xFF0
 104 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK         0xFFFFFFFF
 105
 106 /* AVIC GATAG is encoded using VM and VCPU IDs */
 107 #define AVIC_VCPU_ID_BITS               8
 108 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
 109
 110 #define AVIC_VM_ID_BITS                 24
 111 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
 112 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
 113
 114 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
 115                                                 (y & AVIC_VCPU_ID_MASK))
 116 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
 117 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
 118
 119 static bool erratum_383_found __read_mostly;
 120
 121 static const u32 host_save_user_msrs[] = {
 122 #ifdef CONFIG_X86_64
 123         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
 124         MSR_FS_BASE,
 125 #endif
 126         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 127         MSR_TSC_AUX,
 128 };
 129
 130 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 131
 132 struct kvm_sev_info {
 133         bool active;            /* SEV enabled guest */
 134         unsigned int asid;      /* ASID used for this guest */
 135         unsigned int handle;    /* SEV firmware handle */
 136         int fd;                 /* SEV device fd */
 137         unsigned long pages_locked; /* Number of pages locked */
 138         struct list_head regions_list;  /* List of registered regions */
 139 };
 140
 141 struct kvm_svm {
 142         struct kvm kvm;
 143
 144         /* Struct members for AVIC */
 145         u32 avic_vm_id;
 146         struct page *avic_logical_id_table_page;
 147         struct page *avic_physical_id_table_page;
 148         struct hlist_node hnode;
 149
 150         struct kvm_sev_info sev_info;
 151 };
 152
 153 struct kvm_vcpu;
 154
 155 struct nested_state {
 156         struct vmcb *hsave;
 157         u64 hsave_msr;
 158         u64 vm_cr_msr;
 159         u64 vmcb;
 160
 161         /* These are the merged vectors */
 162         u32 *msrpm;
 163
 164         /* gpa pointers to the real vectors */
 165         u64 vmcb_msrpm;
 166         u64 vmcb_iopm;
 167
 168         /* A VMEXIT is required but not yet emulated */
 169         bool exit_required;
 170
 171         /* cache for intercepts of the guest */
 172         u32 intercept_cr;
 173         u32 intercept_dr;
 174         u32 intercept_exceptions;
 175         u64 intercept;
 176
 177         /* Nested Paging related state */
 178         u64 nested_cr3;
 179 };
 180
 181 #define MSRPM_OFFSETS   16
 182 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 183
 184 /*
 185  * Set osvw_len to higher value when updated Revision Guides
 186  * are published and we know what the new status bits are
 187  */
 188 static uint64_t osvw_len = 4, osvw_status;
 189
 190 struct vcpu_svm {
 191         struct kvm_vcpu vcpu;
 192         struct vmcb *vmcb;
 193         unsigned long vmcb_pa;
 194         struct svm_cpu_data *svm_data;
 195         uint64_t asid_generation;
 196         uint64_t sysenter_esp;
 197         uint64_t sysenter_eip;
 198         uint64_t tsc_aux;
 199
 200         u64 msr_decfg;
 201
 202         u64 next_rip;
 203
 204         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 205         struct {
 206                 u16 fs;
 207                 u16 gs;
 208                 u16 ldt;
 209                 u64 gs_base;
 210         } host;
 211
 212         u64 spec_ctrl;
 213         /*
 214          * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
 215          * translated into the appropriate L2_CFG bits on the host to
 216          * perform speculative control.
 217          */
 218         u64 virt_spec_ctrl;
 219
 220         u32 *msrpm;
 221
 222         ulong nmi_iret_rip;
 223
 224         struct nested_state nested;
 225
 226         bool nmi_singlestep;
 227         u64 nmi_singlestep_guest_rflags;
 228
 229         unsigned int3_injected;
 230         unsigned long int3_rip;
 231
 232         /* cached guest cpuid flags for faster access */
 233         bool nrips_enabled      : 1;
 234
 235         u32 ldr_reg;
 236         u32 dfr_reg;
 237         struct page *avic_backing_page;
 238         u64 *avic_physical_id_cache;
 239         bool avic_is_running;
 240
 241         /*
 242          * Per-vcpu list of struct amd_svm_iommu_ir:
 243          * This is used mainly to store interrupt remapping information used
 244          * when update the vcpu affinity. This avoids the need to scan for
 245          * IRTE and try to match ga_tag in the IOMMU driver.
 246          */
 247         struct list_head ir_list;
 248         spinlock_t ir_list_lock;
 249
 250         /* which host CPU was used for running this vcpu */
 251         unsigned int last_cpu;
 252 };
 253
 254 /*
 255  * This is a wrapper of struct amd_iommu_ir_data.
 256  */
 257 struct amd_svm_iommu_ir {
 258         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
 259         void *data;             /* Storing pointer to struct amd_ir_data */
 260 };
 261
 262 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
 263 #define AVIC_LOGICAL_ID_ENTRY_VALID_BIT                 31
 264 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
 265
 266 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
 267 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
 268 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
 269 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
 270
 271 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 272 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 273
 274 #define MSR_INVALID                     0xffffffffU
 275
 276 static const struct svm_direct_access_msrs {
 277         u32 index;   /* Index of the MSR */
 278         bool always; /* True if intercept is always on */
 279 } direct_access_msrs[] = {
 280         { .index = MSR_STAR,                            .always = true  },
 281         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 282 #ifdef CONFIG_X86_64
 283         { .index = MSR_GS_BASE,                         .always = true  },
 284         { .index = MSR_FS_BASE,                         .always = true  },
 285         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 286         { .index = MSR_LSTAR,                           .always = true  },
 287         { .index = MSR_CSTAR,                           .always = true  },
 288         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 289 #endif
 290         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
 291         { .index = MSR_IA32_PRED_CMD,                   .always = false },
 292         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 293         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 294         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 295         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 296         { .index = MSR_INVALID,                         .always = false },
 297 };
 298
 299 /* enable NPT for AMD64 and X86 with PAE */
 300 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 301 static bool npt_enabled = true;
 302 #else
 303 static bool npt_enabled;
 304 #endif
 305
 306 /*
 307  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 308  * pause_filter_count: On processors that support Pause filtering(indicated
 309  *      by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
 310  *      count value. On VMRUN this value is loaded into an internal counter.
 311  *      Each time a pause instruction is executed, this counter is decremented
 312  *      until it reaches zero at which time a #VMEXIT is generated if pause
 313  *      intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
 314  *      Intercept Filtering for more details.
 315  *      This also indicate if ple logic enabled.
 316  *
 317  * pause_filter_thresh: In addition, some processor families support advanced
 318  *      pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
 319  *      the amount of time a guest is allowed to execute in a pause loop.
 320  *      In this mode, a 16-bit pause filter threshold field is added in the
 321  *      VMCB. The threshold value is a cycle count that is used to reset the
 322  *      pause counter. As with simple pause filtering, VMRUN loads the pause
 323  *      count value from VMCB into an internal counter. Then, on each pause
 324  *      instruction the hardware checks the elapsed number of cycles since
 325  *      the most recent pause instruction against the pause filter threshold.
 326  *      If the elapsed cycle count is greater than the pause filter threshold,
 327  *      then the internal pause count is reloaded from the VMCB and execution
 328  *      continues. If the elapsed cycle count is less than the pause filter
 329  *      threshold, then the internal pause count is decremented. If the count
 330  *      value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 331  *      triggered. If advanced pause filtering is supported and pause filter
 332  *      threshold field is set to zero, the filter will operate in the simpler,
 333  *      count only mode.
 334  */
 335
 336 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 337 module_param(pause_filter_thresh, ushort, 0444);
 338
 339 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 340 module_param(pause_filter_count, ushort, 0444);
 341
 342 /* Default doubles per-vcpu window every exit. */
 343 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 344 module_param(pause_filter_count_grow, ushort, 0444);
 345
 346 /* Default resets per-vcpu window every exit to pause_filter_count. */
 347 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 348 module_param(pause_filter_count_shrink, ushort, 0444);
 349
 350 /* Default is to compute the maximum so we can never overflow. */
 351 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 352 module_param(pause_filter_count_max, ushort, 0444);
 353
 354 /* allow nested paging (virtualized MMU) for all guests */
 355 static int npt = true;
 356 module_param(npt, int, S_IRUGO);
 357
 358 /* allow nested virtualization in KVM/SVM */
 359 static int nested = true;
 360 module_param(nested, int, S_IRUGO);
 361
 362 /* enable / disable AVIC */
 363 static int avic;
 364 #ifdef CONFIG_X86_LOCAL_APIC
 365 module_param(avic, int, S_IRUGO);
 366 #endif
 367
 368 /* enable/disable Next RIP Save */
 369 static int nrips = true;
 370 module_param(nrips, int, 0444);
 371
 372 /* enable/disable Virtual VMLOAD VMSAVE */
 373 static int vls = true;
 374 module_param(vls, int, 0444);
 375
 376 /* enable/disable Virtual GIF */
 377 static int vgif = true;
 378 module_param(vgif, int, 0444);
 379
 380 /* enable/disable SEV support */
 381 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
 382 module_param(sev, int, 0444);
 383
 384 static bool __read_mostly dump_invalid_vmcb = 0;
 385 module_param(dump_invalid_vmcb, bool, 0644);
 386
 387 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 388
 389 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 390 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 391 static void svm_complete_interrupts(struct vcpu_svm *svm);
 392 static void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate);
 393 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu);
 394
 395 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 396 static int nested_svm_intercept(struct vcpu_svm *svm);
 397 static int nested_svm_vmexit(struct vcpu_svm *svm);
 398 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 399                                       bool has_error_code, u32 error_code);
 400
 401 enum {
 402         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 403                             pause filter count */
 404         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 405         VMCB_ASID,       /* ASID */
 406         VMCB_INTR,       /* int_ctl, int_vector */
 407         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 408         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 409         VMCB_DR,         /* DR6, DR7 */
 410         VMCB_DT,         /* GDT, IDT */
 411         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 412         VMCB_CR2,        /* CR2 only */
 413         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 414         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
 415                           * AVIC PHYSICAL_TABLE pointer,
 416                           * AVIC LOGICAL_TABLE pointer
 417                           */
 418         VMCB_DIRTY_MAX,
 419 };
 420
 421 /* TPR and CR2 are always written before VMRUN */
 422 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 423
 424 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
 425
 426 static int sev_flush_asids(void);
 427 static DECLARE_RWSEM(sev_deactivate_lock);
 428 static DEFINE_MUTEX(sev_bitmap_lock);
 429 static unsigned int max_sev_asid;
 430 static unsigned int min_sev_asid;
 431 static unsigned long *sev_asid_bitmap;
 432 static unsigned long *sev_reclaim_asid_bitmap;
 433 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
 434
 435 struct enc_region {
 436         struct list_head list;
 437         unsigned long npages;
 438         struct page **pages;
 439         unsigned long uaddr;
 440         unsigned long size;
 441 };
 442
 443
 444 static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
 445 {
 446         return container_of(kvm, struct kvm_svm, kvm);
 447 }
 448
 449 static inline bool svm_sev_enabled(void)
 450 {
 451         return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0;
 452 }
 453
 454 static inline bool sev_guest(struct kvm *kvm)
 455 {
 456 #ifdef CONFIG_KVM_AMD_SEV
 457         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 458
 459         return sev->active;
 460 #else
 461         return false;
 462 #endif
 463 }
 464
 465 static inline int sev_get_asid(struct kvm *kvm)
 466 {
 467         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 468
 469         return sev->asid;
 470 }
 471
 472 static inline void mark_all_dirty(struct vmcb *vmcb)
 473 {
 474         vmcb->control.clean = 0;
 475 }
 476
 477 static inline void mark_all_clean(struct vmcb *vmcb)
 478 {
 479         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 480                                & ~VMCB_ALWAYS_DIRTY_MASK;
 481 }
 482
 483 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 484 {
 485         vmcb->control.clean &= ~(1 << bit);
 486 }
 487
 488 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 489 {
 490         return container_of(vcpu, struct vcpu_svm, vcpu);
 491 }
 492
 493 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
 494 {
 495         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
 496         mark_dirty(svm->vmcb, VMCB_AVIC);
 497 }
 498
 499 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
 500 {
 501         struct vcpu_svm *svm = to_svm(vcpu);
 502         u64 *entry = svm->avic_physical_id_cache;
 503
 504         if (!entry)
 505                 return false;
 506
 507         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 508 }
 509
 510 static void recalc_intercepts(struct vcpu_svm *svm)
 511 {
 512         struct vmcb_control_area *c, *h;
 513         struct nested_state *g;
 514
 515         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 516
 517         if (!is_guest_mode(&svm->vcpu))
 518                 return;
 519
 520         c = &svm->vmcb->control;
 521         h = &svm->nested.hsave->control;
 522         g = &svm->nested;
 523
 524         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 525         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 526         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 527         c->intercept = h->intercept | g->intercept;
 528 }
 529
 530 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 531 {
 532         if (is_guest_mode(&svm->vcpu))
 533                 return svm->nested.hsave;
 534         else
 535                 return svm->vmcb;
 536 }
 537
 538 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 539 {
 540         struct vmcb *vmcb = get_host_vmcb(svm);
 541
 542         vmcb->control.intercept_cr |= (1U << bit);
 543
 544         recalc_intercepts(svm);
 545 }
 546
 547 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 548 {
 549         struct vmcb *vmcb = get_host_vmcb(svm);
 550
 551         vmcb->control.intercept_cr &= ~(1U << bit);
 552
 553         recalc_intercepts(svm);
 554 }
 555
 556 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 557 {
 558         struct vmcb *vmcb = get_host_vmcb(svm);
 559
 560         return vmcb->control.intercept_cr & (1U << bit);
 561 }
 562
 563 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 564 {
 565         struct vmcb *vmcb = get_host_vmcb(svm);
 566
 567         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 568                 | (1 << INTERCEPT_DR1_READ)
 569                 | (1 << INTERCEPT_DR2_READ)
 570                 | (1 << INTERCEPT_DR3_READ)
 571                 | (1 << INTERCEPT_DR4_READ)
 572                 | (1 << INTERCEPT_DR5_READ)
 573                 | (1 << INTERCEPT_DR6_READ)
 574                 | (1 << INTERCEPT_DR7_READ)
 575                 | (1 << INTERCEPT_DR0_WRITE)
 576                 | (1 << INTERCEPT_DR1_WRITE)
 577                 | (1 << INTERCEPT_DR2_WRITE)
 578                 | (1 << INTERCEPT_DR3_WRITE)
 579                 | (1 << INTERCEPT_DR4_WRITE)
 580                 | (1 << INTERCEPT_DR5_WRITE)
 581                 | (1 << INTERCEPT_DR6_WRITE)
 582                 | (1 << INTERCEPT_DR7_WRITE);
 583
 584         recalc_intercepts(svm);
 585 }
 586
 587 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 588 {
 589         struct vmcb *vmcb = get_host_vmcb(svm);
 590
 591         vmcb->control.intercept_dr = 0;
 592
 593         recalc_intercepts(svm);
 594 }
 595
 596 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 597 {
 598         struct vmcb *vmcb = get_host_vmcb(svm);
 599
 600         vmcb->control.intercept_exceptions |= (1U << bit);
 601
 602         recalc_intercepts(svm);
 603 }
 604
 605 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 606 {
 607         struct vmcb *vmcb = get_host_vmcb(svm);
 608
 609         vmcb->control.intercept_exceptions &= ~(1U << bit);
 610
 611         recalc_intercepts(svm);
 612 }
 613
 614 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 615 {
 616         struct vmcb *vmcb = get_host_vmcb(svm);
 617
 618         vmcb->control.intercept |= (1ULL << bit);
 619
 620         recalc_intercepts(svm);
 621 }
 622
 623 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 624 {
 625         struct vmcb *vmcb = get_host_vmcb(svm);
 626
 627         vmcb->control.intercept &= ~(1ULL << bit);
 628
 629         recalc_intercepts(svm);
 630 }
 631
 632 static inline bool vgif_enabled(struct vcpu_svm *svm)
 633 {
 634         return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
 635 }
 636
 637 static inline void enable_gif(struct vcpu_svm *svm)
 638 {
 639         if (vgif_enabled(svm))
 640                 svm->vmcb->control.int_ctl |= V_GIF_MASK;
 641         else
 642                 svm->vcpu.arch.hflags |= HF_GIF_MASK;
 643 }
 644
 645 static inline void disable_gif(struct vcpu_svm *svm)
 646 {
 647         if (vgif_enabled(svm))
 648                 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
 649         else
 650                 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 651 }
 652
 653 static inline bool gif_set(struct vcpu_svm *svm)
 654 {
 655         if (vgif_enabled(svm))
 656                 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
 657         else
 658                 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 659 }
 660
 661 static unsigned long iopm_base;
 662
 663 struct kvm_ldttss_desc {
 664         u16 limit0;
 665         u16 base0;
 666         unsigned base1:8, type:5, dpl:2, p:1;
 667         unsigned limit1:4, zero0:3, g:1, base2:8;
 668         u32 base3;
 669         u32 zero1;
 670 } __attribute__((packed));
 671
 672 struct svm_cpu_data {
 673         int cpu;
 674
 675         u64 asid_generation;
 676         u32 max_asid;
 677         u32 next_asid;
 678         u32 min_asid;
 679         struct kvm_ldttss_desc *tss_desc;
 680
 681         struct page *save_area;
 682         struct vmcb *current_vmcb;
 683
 684         /* index = sev_asid, value = vmcb pointer */
 685         struct vmcb **sev_vmcbs;
 686 };
 687
 688 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 689
 690 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 691
 692 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 693 #define MSRS_RANGE_SIZE 2048
 694 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 695
 696 static u32 svm_msrpm_offset(u32 msr)
 697 {
 698         u32 offset;
 699         int i;
 700
 701         for (i = 0; i < NUM_MSR_MAPS; i++) {
 702                 if (msr < msrpm_ranges[i] ||
 703                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 704                         continue;
 705
 706                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 707                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 708
 709                 /* Now we have the u8 offset - but need the u32 offset */
 710                 return offset / 4;
 711         }
 712
 713         /* MSR not in any range */
 714         return MSR_INVALID;
 715 }
 716
 717 #define MAX_INST_SIZE 15
 718
 719 static inline void clgi(void)
 720 {
 721         asm volatile (__ex("clgi"));
 722 }
 723
 724 static inline void stgi(void)
 725 {
 726         asm volatile (__ex("stgi"));
 727 }
 728
 729 static inline void invlpga(unsigned long addr, u32 asid)
 730 {
 731         asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
 732 }
 733
 734 static int get_npt_level(struct kvm_vcpu *vcpu)
 735 {
 736 #ifdef CONFIG_X86_64
 737         return PT64_ROOT_4LEVEL;
 738 #else
 739         return PT32E_ROOT_LEVEL;
 740 #endif
 741 }
 742
 743 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 744 {
 745         vcpu->arch.efer = efer;
 746
 747         if (!npt_enabled) {
 748                 /* Shadow paging assumes NX to be available.  */
 749                 efer |= EFER_NX;
 750
 751                 if (!(efer & EFER_LMA))
 752                         efer &= ~EFER_LME;
 753         }
 754
 755         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 756         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 757 }
 758
 759 static int is_external_interrupt(u32 info)
 760 {
 761         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 762         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 763 }
 764
 765 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 766 {
 767         struct vcpu_svm *svm = to_svm(vcpu);
 768         u32 ret = 0;
 769
 770         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 771                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 772         return ret;
 773 }
 774
 775 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 776 {
 777         struct vcpu_svm *svm = to_svm(vcpu);
 778
 779         if (mask == 0)
 780                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 781         else
 782                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 783
 784 }
 785
 786 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
 787 {
 788         struct vcpu_svm *svm = to_svm(vcpu);
 789
 790         if (nrips && svm->vmcb->control.next_rip != 0) {
 791                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 792                 svm->next_rip = svm->vmcb->control.next_rip;
 793         }
 794
 795         if (!svm->next_rip) {
 796                 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
 797                         return 0;
 798         } else {
 799                 if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 800                         pr_err("%s: ip 0x%lx next 0x%llx\n",
 801                                __func__, kvm_rip_read(vcpu), svm->next_rip);
 802                 kvm_rip_write(vcpu, svm->next_rip);
 803         }
 804         svm_set_interrupt_shadow(vcpu, 0);
 805
 806         return 1;
 807 }
 808
 809 static void svm_queue_exception(struct kvm_vcpu *vcpu)
 810 {
 811         struct vcpu_svm *svm = to_svm(vcpu);
 812         unsigned nr = vcpu->arch.exception.nr;
 813         bool has_error_code = vcpu->arch.exception.has_error_code;
 814         bool reinject = vcpu->arch.exception.injected;
 815         u32 error_code = vcpu->arch.exception.error_code;
 816
 817         /*
 818          * If we are within a nested VM we'd better #VMEXIT and let the guest
 819          * handle the exception
 820          */
 821         if (!reinject &&
 822             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 823                 return;
 824
 825         kvm_deliver_exception_payload(&svm->vcpu);
 826
 827         if (nr == BP_VECTOR && !nrips) {
 828                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 829
 830                 /*
 831                  * For guest debugging where we have to reinject #BP if some
 832                  * INT3 is guest-owned:
 833                  * Emulate nRIP by moving RIP forward. Will fail if injection
 834                  * raises a fault that is not intercepted. Still better than
 835                  * failing in all cases.
 836                  */
 837                 (void)skip_emulated_instruction(&svm->vcpu);
 838                 rip = kvm_rip_read(&svm->vcpu);
 839                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 840                 svm->int3_injected = rip - old_rip;
 841         }
 842
 843         svm->vmcb->control.event_inj = nr
 844                 | SVM_EVTINJ_VALID
 845                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 846                 | SVM_EVTINJ_TYPE_EXEPT;
 847         svm->vmcb->control.event_inj_err = error_code;
 848 }
 849
 850 static void svm_init_erratum_383(void)
 851 {
 852         u32 low, high;
 853         int err;
 854         u64 val;
 855
 856         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 857                 return;
 858
 859         /* Use _safe variants to not break nested virtualization */
 860         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 861         if (err)
 862                 return;
 863
 864         val |= (1ULL << 47);
 865
 866         low  = lower_32_bits(val);
 867         high = upper_32_bits(val);
 868
 869         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 870
 871         erratum_383_found = true;
 872 }
 873
 874 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 875 {
 876         /*
 877          * Guests should see errata 400 and 415 as fixed (assuming that
 878          * HLT and IO instructions are intercepted).
 879          */
 880         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 881         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 882
 883         /*
 884          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 885          * all osvw.status bits inside that length, including bit 0 (which is
 886          * reserved for erratum 298), are valid. However, if host processor's
 887          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 888          * be conservative here and therefore we tell the guest that erratum 298
 889          * is present (because we really don't know).
 890          */
 891         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 892                 vcpu->arch.osvw.status |= 1;
 893 }
 894
 895 static int has_svm(void)
 896 {
 897         const char *msg;
 898
 899         if (!cpu_has_svm(&msg)) {
 900                 printk(KERN_INFO "has_svm: %s\n", msg);
 901                 return 0;
 902         }
 903
 904         return 1;
 905 }
 906
 907 static void svm_hardware_disable(void)
 908 {
 909         /* Make sure we clean up behind us */
 910         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 911                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 912
 913         cpu_svm_disable();
 914
 915         amd_pmu_disable_virt();
 916 }
 917
 918 static int svm_hardware_enable(void)
 919 {
 920
 921         struct svm_cpu_data *sd;
 922         uint64_t efer;
 923         struct desc_struct *gdt;
 924         int me = raw_smp_processor_id();
 925
 926         rdmsrl(MSR_EFER, efer);
 927         if (efer & EFER_SVME)
 928                 return -EBUSY;
 929
 930         if (!has_svm()) {
 931                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 932                 return -EINVAL;
 933         }
 934         sd = per_cpu(svm_data, me);
 935         if (!sd) {
 936                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 937                 return -EINVAL;
 938         }
 939
 940         sd->asid_generation = 1;
 941         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 942         sd->next_asid = sd->max_asid + 1;
 943         sd->min_asid = max_sev_asid + 1;
 944
 945         gdt = get_current_gdt_rw();
 946         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 947
 948         wrmsrl(MSR_EFER, efer | EFER_SVME);
 949
 950         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 951
 952         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 953                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 954                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 955         }
 956
 957
 958         /*
 959          * Get OSVW bits.
 960          *
 961          * Note that it is possible to have a system with mixed processor
 962          * revisions and therefore different OSVW bits. If bits are not the same
 963          * on different processors then choose the worst case (i.e. if erratum
 964          * is present on one processor and not on another then assume that the
 965          * erratum is present everywhere).
 966          */
 967         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 968                 uint64_t len, status = 0;
 969                 int err;
 970
 971                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 972                 if (!err)
 973                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 974                                                       &err);
 975
 976                 if (err)
 977                         osvw_status = osvw_len = 0;
 978                 else {
 979                         if (len < osvw_len)
 980                                 osvw_len = len;
 981                         osvw_status |= status;
 982                         osvw_status &= (1ULL << osvw_len) - 1;
 983                 }
 984         } else
 985                 osvw_status = osvw_len = 0;
 986
 987         svm_init_erratum_383();
 988
 989         amd_pmu_enable_virt();
 990
 991         return 0;
 992 }
 993
 994 static void svm_cpu_uninit(int cpu)
 995 {
 996         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 997
 998         if (!sd)
 999                 return;
1000
1001         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
1002         kfree(sd->sev_vmcbs);
1003         __free_page(sd->save_area);
1004         kfree(sd);
1005 }
1006
1007 static int svm_cpu_init(int cpu)
1008 {
1009         struct svm_cpu_data *sd;
1010
1011         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
1012         if (!sd)
1013                 return -ENOMEM;
1014         sd->cpu = cpu;
1015         sd->save_area = alloc_page(GFP_KERNEL);
1016         if (!sd->save_area)
1017                 goto free_cpu_data;
1018
1019         if (svm_sev_enabled()) {
1020                 sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1,
1021                                               sizeof(void *),
1022                                               GFP_KERNEL);
1023                 if (!sd->sev_vmcbs)
1024                         goto free_save_area;
1025         }
1026
1027         per_cpu(svm_data, cpu) = sd;
1028
1029         return 0;
1030
1031 free_save_area:
1032         __free_page(sd->save_area);
1033 free_cpu_data:
1034         kfree(sd);
1035         return -ENOMEM;
1036
1037 }
1038
1039 static bool valid_msr_intercept(u32 index)
1040 {
1041         int i;
1042
1043         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
1044                 if (direct_access_msrs[i].index == index)
1045                         return true;
1046
1047         return false;
1048 }
1049
1050 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
1051 {
1052         u8 bit_write;
1053         unsigned long tmp;
1054         u32 offset;
1055         u32 *msrpm;
1056
1057         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
1058                                       to_svm(vcpu)->msrpm;
1059
1060         offset    = svm_msrpm_offset(msr);
1061         bit_write = 2 * (msr & 0x0f) + 1;
1062         tmp       = msrpm[offset];
1063
1064         BUG_ON(offset == MSR_INVALID);
1065
1066         return !!test_bit(bit_write,  &tmp);
1067 }
1068
1069 static void set_msr_interception(u32 *msrpm, unsigned msr,
1070                                  int read, int write)
1071 {
1072         u8 bit_read, bit_write;
1073         unsigned long tmp;
1074         u32 offset;
1075
1076         /*
1077          * If this warning triggers extend the direct_access_msrs list at the
1078          * beginning of the file
1079          */
1080         WARN_ON(!valid_msr_intercept(msr));
1081
1082         offset    = svm_msrpm_offset(msr);
1083         bit_read  = 2 * (msr & 0x0f);
1084         bit_write = 2 * (msr & 0x0f) + 1;
1085         tmp       = msrpm[offset];
1086
1087         BUG_ON(offset == MSR_INVALID);
1088
1089         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
1090         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
1091
1092         msrpm[offset] = tmp;
1093 }
1094
1095 static void svm_vcpu_init_msrpm(u32 *msrpm)
1096 {
1097         int i;
1098
1099         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
1100
1101         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1102                 if (!direct_access_msrs[i].always)
1103                         continue;
1104
1105                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1106         }
1107 }
1108
1109 static void add_msr_offset(u32 offset)
1110 {
1111         int i;
1112
1113         for (i = 0; i < MSRPM_OFFSETS; ++i) {
1114
1115                 /* Offset already in list? */
1116                 if (msrpm_offsets[i] == offset)
1117                         return;
1118
1119                 /* Slot used by another offset? */
1120                 if (msrpm_offsets[i] != MSR_INVALID)
1121                         continue;
1122
1123                 /* Add offset to list */
1124                 msrpm_offsets[i] = offset;
1125
1126                 return;
1127         }
1128
1129         /*
1130          * If this BUG triggers the msrpm_offsets table has an overflow. Just
1131          * increase MSRPM_OFFSETS in this case.
1132          */
1133         BUG();
1134 }
1135
1136 static void init_msrpm_offsets(void)
1137 {
1138         int i;
1139
1140         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1141
1142         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1143                 u32 offset;
1144
1145                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1146                 BUG_ON(offset == MSR_INVALID);
1147
1148                 add_msr_offset(offset);
1149         }
1150 }
1151
1152 static void svm_enable_lbrv(struct vcpu_svm *svm)
1153 {
1154         u32 *msrpm = svm->msrpm;
1155
1156         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1157         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1158         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1159         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1160         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1161 }
1162
1163 static void svm_disable_lbrv(struct vcpu_svm *svm)
1164 {
1165         u32 *msrpm = svm->msrpm;
1166
1167         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1168         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1169         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1170         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1171         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1172 }
1173
1174 static void disable_nmi_singlestep(struct vcpu_svm *svm)
1175 {
1176         svm->nmi_singlestep = false;
1177
1178         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1179                 /* Clear our flags if they were not set by the guest */
1180                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1181                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1182                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1183                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1184         }
1185 }
1186
1187 /* Note:
1188  * This hash table is used to map VM_ID to a struct kvm_svm,
1189  * when handling AMD IOMMU GALOG notification to schedule in
1190  * a particular vCPU.
1191  */
1192 #define SVM_VM_DATA_HASH_BITS   8
1193 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1194 static u32 next_vm_id = 0;
1195 static bool next_vm_id_wrapped = 0;
1196 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1197
1198 /* Note:
1199  * This function is called from IOMMU driver to notify
1200  * SVM to schedule in a particular vCPU of a particular VM.
1201  */
1202 static int avic_ga_log_notifier(u32 ga_tag)
1203 {
1204         unsigned long flags;
1205         struct kvm_svm *kvm_svm;
1206         struct kvm_vcpu *vcpu = NULL;
1207         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1208         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1209
1210         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1211
1212         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1213         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
1214                 if (kvm_svm->avic_vm_id != vm_id)
1215                         continue;
1216                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
1217                 break;
1218         }
1219         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1220
1221         /* Note:
1222          * At this point, the IOMMU should have already set the pending
1223          * bit in the vAPIC backing page. So, we just need to schedule
1224          * in the vcpu.
1225          */
1226         if (vcpu)
1227                 kvm_vcpu_wake_up(vcpu);
1228
1229         return 0;
1230 }
1231
1232 static __init int sev_hardware_setup(void)
1233 {
1234         struct sev_user_data_status *status;
1235         int rc;
1236
1237         /* Maximum number of encrypted guests supported simultaneously */
1238         max_sev_asid = cpuid_ecx(0x8000001F);
1239
1240         if (!max_sev_asid)
1241                 return 1;
1242
1243         /* Minimum ASID value that should be used for SEV guest */
1244         min_sev_asid = cpuid_edx(0x8000001F);
1245
1246         /* Initialize SEV ASID bitmaps */
1247         sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1248         if (!sev_asid_bitmap)
1249                 return 1;
1250
1251         sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1252         if (!sev_reclaim_asid_bitmap)
1253                 return 1;
1254
1255         status = kmalloc(sizeof(*status), GFP_KERNEL);
1256         if (!status)
1257                 return 1;
1258
1259         /*
1260          * Check SEV platform status.
1261          *
1262          * PLATFORM_STATUS can be called in any state, if we failed to query
1263          * the PLATFORM status then either PSP firmware does not support SEV
1264          * feature or SEV firmware is dead.
1265          */
1266         rc = sev_platform_status(status, NULL);
1267         if (rc)
1268                 goto err;
1269
1270         pr_info("SEV supported\n");
1271
1272 err:
1273         kfree(status);
1274         return rc;
1275 }
1276
1277 static void grow_ple_window(struct kvm_vcpu *vcpu)
1278 {
1279         struct vcpu_svm *svm = to_svm(vcpu);
1280         struct vmcb_control_area *control = &svm->vmcb->control;
1281         int old = control->pause_filter_count;
1282
1283         control->pause_filter_count = __grow_ple_window(old,
1284                                                         pause_filter_count,
1285                                                         pause_filter_count_grow,
1286                                                         pause_filter_count_max);
1287
1288         if (control->pause_filter_count != old) {
1289                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1290                 trace_kvm_ple_window_update(vcpu->vcpu_id,
1291                                             control->pause_filter_count, old);
1292         }
1293 }
1294
1295 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1296 {
1297         struct vcpu_svm *svm = to_svm(vcpu);
1298         struct vmcb_control_area *control = &svm->vmcb->control;
1299         int old = control->pause_filter_count;
1300
1301         control->pause_filter_count =
1302                                 __shrink_ple_window(old,
1303                                                     pause_filter_count,
1304                                                     pause_filter_count_shrink,
1305                                                     pause_filter_count);
1306         if (control->pause_filter_count != old) {
1307                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1308                 trace_kvm_ple_window_update(vcpu->vcpu_id,
1309                                             control->pause_filter_count, old);
1310         }
1311 }
1312
1313 /*
1314  * The default MMIO mask is a single bit (excluding the present bit),
1315  * which could conflict with the memory encryption bit. Check for
1316  * memory encryption support and override the default MMIO mask if
1317  * memory encryption is enabled.
1318  */
1319 static __init void svm_adjust_mmio_mask(void)
1320 {
1321         unsigned int enc_bit, mask_bit;
1322         u64 msr, mask;
1323
1324         /* If there is no memory encryption support, use existing mask */
1325         if (cpuid_eax(0x80000000) < 0x8000001f)
1326                 return;
1327
1328         /* If memory encryption is not enabled, use existing mask */
1329         rdmsrl(MSR_K8_SYSCFG, msr);
1330         if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT))
1331                 return;
1332
1333         enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
1334         mask_bit = boot_cpu_data.x86_phys_bits;
1335
1336         /* Increment the mask bit if it is the same as the encryption bit */
1337         if (enc_bit == mask_bit)
1338                 mask_bit++;
1339
1340         /*
1341          * If the mask bit location is below 52, then some bits above the
1342          * physical addressing limit will always be reserved, so use the
1343          * rsvd_bits() function to generate the mask. This mask, along with
1344          * the present bit, will be used to generate a page fault with
1345          * PFER.RSV = 1.
1346          *
1347          * If the mask bit location is 52 (or above), then clear the mask.
1348          */
1349         mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
1350
1351         kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
1352 }
1353
1354 static void svm_hardware_teardown(void)
1355 {
1356         int cpu;
1357
1358         if (svm_sev_enabled()) {
1359                 bitmap_free(sev_asid_bitmap);
1360                 bitmap_free(sev_reclaim_asid_bitmap);
1361
1362                 sev_flush_asids();
1363         }
1364
1365         for_each_possible_cpu(cpu)
1366                 svm_cpu_uninit(cpu);
1367
1368         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1369         iopm_base = 0;
1370 }
1371
1372 static __init int svm_hardware_setup(void)
1373 {
1374         int cpu;
1375         struct page *iopm_pages;
1376         void *iopm_va;
1377         int r;
1378
1379         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1380
1381         if (!iopm_pages)
1382                 return -ENOMEM;
1383
1384         iopm_va = page_address(iopm_pages);
1385         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1386         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1387
1388         init_msrpm_offsets();
1389
1390         if (boot_cpu_has(X86_FEATURE_NX))
1391                 kvm_enable_efer_bits(EFER_NX);
1392
1393         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1394                 kvm_enable_efer_bits(EFER_FFXSR);
1395
1396         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1397                 kvm_has_tsc_control = true;
1398                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1399                 kvm_tsc_scaling_ratio_frac_bits = 32;
1400         }
1401
1402         /* Check for pause filtering support */
1403         if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1404                 pause_filter_count = 0;
1405                 pause_filter_thresh = 0;
1406         } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1407                 pause_filter_thresh = 0;
1408         }
1409
1410         if (nested) {
1411                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1412                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1413         }
1414
1415         if (sev) {
1416                 if (boot_cpu_has(X86_FEATURE_SEV) &&
1417                     IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1418                         r = sev_hardware_setup();
1419                         if (r)
1420                                 sev = false;
1421                 } else {
1422                         sev = false;
1423                 }
1424         }
1425
1426         svm_adjust_mmio_mask();
1427
1428         for_each_possible_cpu(cpu) {
1429                 r = svm_cpu_init(cpu);
1430                 if (r)
1431                         goto err;
1432         }
1433
1434         if (!boot_cpu_has(X86_FEATURE_NPT))
1435                 npt_enabled = false;
1436
1437         if (npt_enabled && !npt) {
1438                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1439                 npt_enabled = false;
1440         }
1441
1442         if (npt_enabled) {
1443                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1444                 kvm_enable_tdp();
1445         } else
1446                 kvm_disable_tdp();
1447
1448         if (nrips) {
1449                 if (!boot_cpu_has(X86_FEATURE_NRIPS))
1450                         nrips = false;
1451         }
1452
1453         if (avic) {
1454                 if (!npt_enabled ||
1455                     !boot_cpu_has(X86_FEATURE_AVIC) ||
1456                     !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1457                         avic = false;
1458                 } else {
1459                         pr_info("AVIC enabled\n");
1460
1461                         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1462                 }
1463         }
1464
1465         if (vls) {
1466                 if (!npt_enabled ||
1467                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1468                     !IS_ENABLED(CONFIG_X86_64)) {
1469                         vls = false;
1470                 } else {
1471                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1472                 }
1473         }
1474
1475         if (vgif) {
1476                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1477                         vgif = false;
1478                 else
1479                         pr_info("Virtual GIF supported\n");
1480         }
1481
1482         return 0;
1483
1484 err:
1485         svm_hardware_teardown();
1486         return r;
1487 }
1488
1489 static void init_seg(struct vmcb_seg *seg)
1490 {
1491         seg->selector = 0;
1492         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1493                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1494         seg->limit = 0xffff;
1495         seg->base = 0;
1496 }
1497
1498 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1499 {
1500         seg->selector = 0;
1501         seg->attrib = SVM_SELECTOR_P_MASK | type;
1502         seg->limit = 0xffff;
1503         seg->base = 0;
1504 }
1505
1506 static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
1507 {
1508         struct vcpu_svm *svm = to_svm(vcpu);
1509
1510         if (is_guest_mode(vcpu))
1511                 return svm->nested.hsave->control.tsc_offset;
1512
1513         return vcpu->arch.tsc_offset;
1514 }
1515
1516 static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1517 {
1518         struct vcpu_svm *svm = to_svm(vcpu);
1519         u64 g_tsc_offset = 0;
1520
1521         if (is_guest_mode(vcpu)) {
1522                 /* Write L1's TSC offset.  */
1523                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1524                                svm->nested.hsave->control.tsc_offset;
1525                 svm->nested.hsave->control.tsc_offset = offset;
1526         }
1527
1528         trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1529                                    svm->vmcb->control.tsc_offset - g_tsc_offset,
1530                                    offset);
1531
1532         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1533
1534         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1535         return svm->vmcb->control.tsc_offset;
1536 }
1537
1538 static void avic_init_vmcb(struct vcpu_svm *svm)
1539 {
1540         struct vmcb *vmcb = svm->vmcb;
1541         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
1542         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
1543         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
1544         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
1545
1546         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1547         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1548         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1549         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1550         if (kvm_apicv_activated(svm->vcpu.kvm))
1551                 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1552         else
1553                 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
1554 }
1555
1556 static void init_vmcb(struct vcpu_svm *svm)
1557 {
1558         struct vmcb_control_area *control = &svm->vmcb->control;
1559         struct vmcb_save_area *save = &svm->vmcb->save;
1560
1561         svm->vcpu.arch.hflags = 0;
1562
1563         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1564         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1565         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1566         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1567         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1568         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1569         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1570                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1571
1572         set_dr_intercepts(svm);
1573
1574         set_exception_intercept(svm, PF_VECTOR);
1575         set_exception_intercept(svm, UD_VECTOR);
1576         set_exception_intercept(svm, MC_VECTOR);
1577         set_exception_intercept(svm, AC_VECTOR);
1578         set_exception_intercept(svm, DB_VECTOR);
1579         /*
1580          * Guest access to VMware backdoor ports could legitimately
1581          * trigger #GP because of TSS I/O permission bitmap.
1582          * We intercept those #GP and allow access to them anyway
1583          * as VMware does.
1584          */
1585         if (enable_vmware_backdoor)
1586                 set_exception_intercept(svm, GP_VECTOR);
1587
1588         set_intercept(svm, INTERCEPT_INTR);
1589         set_intercept(svm, INTERCEPT_NMI);
1590         set_intercept(svm, INTERCEPT_SMI);
1591         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1592         set_intercept(svm, INTERCEPT_RDPMC);
1593         set_intercept(svm, INTERCEPT_CPUID);
1594         set_intercept(svm, INTERCEPT_INVD);
1595         set_intercept(svm, INTERCEPT_INVLPG);
1596         set_intercept(svm, INTERCEPT_INVLPGA);
1597         set_intercept(svm, INTERCEPT_IOIO_PROT);
1598         set_intercept(svm, INTERCEPT_MSR_PROT);
1599         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1600         set_intercept(svm, INTERCEPT_SHUTDOWN);
1601         set_intercept(svm, INTERCEPT_VMRUN);
1602         set_intercept(svm, INTERCEPT_VMMCALL);
1603         set_intercept(svm, INTERCEPT_VMLOAD);
1604         set_intercept(svm, INTERCEPT_VMSAVE);
1605         set_intercept(svm, INTERCEPT_STGI);
1606         set_intercept(svm, INTERCEPT_CLGI);
1607         set_intercept(svm, INTERCEPT_SKINIT);
1608         set_intercept(svm, INTERCEPT_WBINVD);
1609         set_intercept(svm, INTERCEPT_XSETBV);
1610         set_intercept(svm, INTERCEPT_RDPRU);
1611         set_intercept(svm, INTERCEPT_RSM);
1612
1613         if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1614                 set_intercept(svm, INTERCEPT_MONITOR);
1615                 set_intercept(svm, INTERCEPT_MWAIT);
1616         }
1617
1618         if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1619                 set_intercept(svm, INTERCEPT_HLT);
1620
1621         control->iopm_base_pa = __sme_set(iopm_base);
1622         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1623         control->int_ctl = V_INTR_MASKING_MASK;
1624
1625         init_seg(&save->es);
1626         init_seg(&save->ss);
1627         init_seg(&save->ds);
1628         init_seg(&save->fs);
1629         init_seg(&save->gs);
1630
1631         save->cs.selector = 0xf000;
1632         save->cs.base = 0xffff0000;
1633         /* Executable/Readable Code Segment */
1634         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1635                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1636         save->cs.limit = 0xffff;
1637
1638         save->gdtr.limit = 0xffff;
1639         save->idtr.limit = 0xffff;
1640
1641         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1642         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1643
1644         svm_set_efer(&svm->vcpu, 0);
1645         save->dr6 = 0xffff0ff0;
1646         kvm_set_rflags(&svm->vcpu, 2);
1647         save->rip = 0x0000fff0;
1648         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1649
1650         /*
1651          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1652          * It also updates the guest-visible cr0 value.
1653          */
1654         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1655         kvm_mmu_reset_context(&svm->vcpu);
1656
1657         save->cr4 = X86_CR4_PAE;
1658         /* rdx = ?? */
1659
1660         if (npt_enabled) {
1661                 /* Setup VMCB for Nested Paging */
1662                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1663                 clr_intercept(svm, INTERCEPT_INVLPG);
1664                 clr_exception_intercept(svm, PF_VECTOR);
1665                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1666                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1667                 save->g_pat = svm->vcpu.arch.pat;
1668                 save->cr3 = 0;
1669                 save->cr4 = 0;
1670         }
1671         svm->asid_generation = 0;
1672
1673         svm->nested.vmcb = 0;
1674         svm->vcpu.arch.hflags = 0;
1675
1676         if (pause_filter_count) {
1677                 control->pause_filter_count = pause_filter_count;
1678                 if (pause_filter_thresh)
1679                         control->pause_filter_thresh = pause_filter_thresh;
1680                 set_intercept(svm, INTERCEPT_PAUSE);
1681         } else {
1682                 clr_intercept(svm, INTERCEPT_PAUSE);
1683         }
1684
1685         if (kvm_vcpu_apicv_active(&svm->vcpu))
1686                 avic_init_vmcb(svm);
1687
1688         /*
1689          * If hardware supports Virtual VMLOAD VMSAVE then enable it
1690          * in VMCB and clear intercepts to avoid #VMEXIT.
1691          */
1692         if (vls) {
1693                 clr_intercept(svm, INTERCEPT_VMLOAD);
1694                 clr_intercept(svm, INTERCEPT_VMSAVE);
1695                 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1696         }
1697
1698         if (vgif) {
1699                 clr_intercept(svm, INTERCEPT_STGI);
1700                 clr_intercept(svm, INTERCEPT_CLGI);
1701                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1702         }
1703
1704         if (sev_guest(svm->vcpu.kvm)) {
1705                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1706                 clr_exception_intercept(svm, UD_VECTOR);
1707         }
1708
1709         mark_all_dirty(svm->vmcb);
1710
1711         enable_gif(svm);
1712
1713 }
1714
1715 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1716                                        unsigned int index)
1717 {
1718         u64 *avic_physical_id_table;
1719         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
1720
1721         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1722                 return NULL;
1723
1724         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
1725
1726         return &avic_physical_id_table[index];
1727 }
1728
1729 /**
1730  * Note:
1731  * AVIC hardware walks the nested page table to check permissions,
1732  * but does not use the SPA address specified in the leaf page
1733  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1734  * field of the VMCB. Therefore, we set up the
1735  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1736  */
1737 static int avic_update_access_page(struct kvm *kvm, bool activate)
1738 {
1739         int ret = 0;
1740
1741         mutex_lock(&kvm->slots_lock);
1742         /*
1743          * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger
1744          * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT
1745          * memory region. So, we need to ensure that kvm->mm == current->mm.
1746          */
1747         if ((kvm->arch.apic_access_page_done == activate) ||
1748             (kvm->mm != current->mm))
1749                 goto out;
1750
1751         ret = __x86_set_memory_region(kvm,
1752                                       APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1753                                       APIC_DEFAULT_PHYS_BASE,
1754                                       activate ? PAGE_SIZE : 0);
1755         if (ret)
1756                 goto out;
1757
1758         kvm->arch.apic_access_page_done = activate;
1759 out:
1760         mutex_unlock(&kvm->slots_lock);
1761         return ret;
1762 }
1763
1764 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1765 {
1766         u64 *entry, new_entry;
1767         int id = vcpu->vcpu_id;
1768         struct vcpu_svm *svm = to_svm(vcpu);
1769
1770         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1771                 return -EINVAL;
1772
1773         if (!svm->vcpu.arch.apic->regs)
1774                 return -EINVAL;
1775
1776         if (kvm_apicv_activated(vcpu->kvm)) {
1777                 int ret;
1778
1779                 ret = avic_update_access_page(vcpu->kvm, true);
1780                 if (ret)
1781                         return ret;
1782         }
1783
1784         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1785
1786         /* Setting AVIC backing page address in the phy APIC ID table */
1787         entry = avic_get_physical_id_entry(vcpu, id);
1788         if (!entry)
1789                 return -EINVAL;
1790
1791         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1792                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1793                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1794         WRITE_ONCE(*entry, new_entry);
1795
1796         svm->avic_physical_id_cache = entry;
1797
1798         return 0;
1799 }
1800
1801 static void sev_asid_free(int asid)
1802 {
1803         struct svm_cpu_data *sd;
1804         int cpu, pos;
1805
1806         mutex_lock(&sev_bitmap_lock);
1807
1808         pos = asid - 1;
1809         __set_bit(pos, sev_reclaim_asid_bitmap);
1810
1811         for_each_possible_cpu(cpu) {
1812                 sd = per_cpu(svm_data, cpu);
1813                 sd->sev_vmcbs[pos] = NULL;
1814         }
1815
1816         mutex_unlock(&sev_bitmap_lock);
1817 }
1818
1819 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1820 {
1821         struct sev_data_decommission *decommission;
1822         struct sev_data_deactivate *data;
1823
1824         if (!handle)
1825                 return;
1826
1827         data = kzalloc(sizeof(*data), GFP_KERNEL);
1828         if (!data)
1829                 return;
1830
1831         /* deactivate handle */
1832         data->handle = handle;
1833
1834         /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */
1835         down_read(&sev_deactivate_lock);
1836         sev_guest_deactivate(data, NULL);
1837         up_read(&sev_deactivate_lock);
1838
1839         kfree(data);
1840
1841         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1842         if (!decommission)
1843                 return;
1844
1845         /* decommission handle */
1846         decommission->handle = handle;
1847         sev_guest_decommission(decommission, NULL);
1848
1849         kfree(decommission);
1850 }
1851
1852 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1853                                     unsigned long ulen, unsigned long *n,
1854                                     int write)
1855 {
1856         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1857         unsigned long npages, npinned, size;
1858         unsigned long locked, lock_limit;
1859         struct page **pages;
1860         unsigned long first, last;
1861
1862         if (ulen == 0 || uaddr + ulen < uaddr)
1863                 return NULL;
1864
1865         /* Calculate number of pages. */
1866         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1867         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1868         npages = (last - first + 1);
1869
1870         locked = sev->pages_locked + npages;
1871         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1872         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1873                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1874                 return NULL;
1875         }
1876
1877         /* Avoid using vmalloc for smaller buffers. */
1878         size = npages * sizeof(struct page *);
1879         if (size > PAGE_SIZE)
1880                 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1881                                   PAGE_KERNEL);
1882         else
1883                 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
1884
1885         if (!pages)
1886                 return NULL;
1887
1888         /* Pin the user virtual address. */
1889         npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages);
1890         if (npinned != npages) {
1891                 pr_err("SEV: Failure locking %lu pages.\n", npages);
1892                 goto err;
1893         }
1894
1895         *n = npages;
1896         sev->pages_locked = locked;
1897
1898         return pages;
1899
1900 err:
1901         if (npinned > 0)
1902                 release_pages(pages, npinned);
1903
1904         kvfree(pages);
1905         return NULL;
1906 }
1907
1908 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1909                              unsigned long npages)
1910 {
1911         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1912
1913         release_pages(pages, npages);
1914         kvfree(pages);
1915         sev->pages_locked -= npages;
1916 }
1917
1918 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1919 {
1920         uint8_t *page_virtual;
1921         unsigned long i;
1922
1923         if (npages == 0 || pages == NULL)
1924                 return;
1925
1926         for (i = 0; i < npages; i++) {
1927                 page_virtual = kmap_atomic(pages[i]);
1928                 clflush_cache_range(page_virtual, PAGE_SIZE);
1929                 kunmap_atomic(page_virtual);
1930         }
1931 }
1932
1933 static void __unregister_enc_region_locked(struct kvm *kvm,
1934                                            struct enc_region *region)
1935 {
1936         /*
1937          * The guest may change the memory encryption attribute from C=0 -> C=1
1938          * or vice versa for this memory range. Lets make sure caches are
1939          * flushed to ensure that guest data gets written into memory with
1940          * correct C-bit.
1941          */
1942         sev_clflush_pages(region->pages, region->npages);
1943
1944         sev_unpin_memory(kvm, region->pages, region->npages);
1945         list_del(&region->list);
1946         kfree(region);
1947 }
1948
1949 static struct kvm *svm_vm_alloc(void)
1950 {
1951         struct kvm_svm *kvm_svm = __vmalloc(sizeof(struct kvm_svm),
1952                                             GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1953                                             PAGE_KERNEL);
1954         return &kvm_svm->kvm;
1955 }
1956
1957 static void svm_vm_free(struct kvm *kvm)
1958 {
1959         vfree(to_kvm_svm(kvm));
1960 }
1961
1962 static void sev_vm_destroy(struct kvm *kvm)
1963 {
1964         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1965         struct list_head *head = &sev->regions_list;
1966         struct list_head *pos, *q;
1967
1968         if (!sev_guest(kvm))
1969                 return;
1970
1971         mutex_lock(&kvm->lock);
1972
1973         /*
1974          * if userspace was terminated before unregistering the memory regions
1975          * then lets unpin all the registered memory.
1976          */
1977         if (!list_empty(head)) {
1978                 list_for_each_safe(pos, q, head) {
1979                         __unregister_enc_region_locked(kvm,
1980                                 list_entry(pos, struct enc_region, list));
1981                 }
1982         }
1983
1984         mutex_unlock(&kvm->lock);
1985
1986         sev_unbind_asid(kvm, sev->handle);
1987         sev_asid_free(sev->asid);
1988 }
1989
1990 static void avic_vm_destroy(struct kvm *kvm)
1991 {
1992         unsigned long flags;
1993         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1994
1995         if (!avic)
1996                 return;
1997
1998         if (kvm_svm->avic_logical_id_table_page)
1999                 __free_page(kvm_svm->avic_logical_id_table_page);
2000         if (kvm_svm->avic_physical_id_table_page)
2001                 __free_page(kvm_svm->avic_physical_id_table_page);
2002
2003         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
2004         hash_del(&kvm_svm->hnode);
2005         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
2006 }
2007
2008 static void svm_vm_destroy(struct kvm *kvm)
2009 {
2010         avic_vm_destroy(kvm);
2011         sev_vm_destroy(kvm);
2012 }
2013
2014 static int avic_vm_init(struct kvm *kvm)
2015 {
2016         unsigned long flags;
2017         int err = -ENOMEM;
2018         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
2019         struct kvm_svm *k2;
2020         struct page *p_page;
2021         struct page *l_page;
2022         u32 vm_id;
2023
2024         if (!avic)
2025                 return 0;
2026
2027         /* Allocating physical APIC ID table (4KB) */
2028         p_page = alloc_page(GFP_KERNEL_ACCOUNT);
2029         if (!p_page)
2030                 goto free_avic;
2031
2032         kvm_svm->avic_physical_id_table_page = p_page;
2033         clear_page(page_address(p_page));
2034
2035         /* Allocating logical APIC ID table (4KB) */
2036         l_page = alloc_page(GFP_KERNEL_ACCOUNT);
2037         if (!l_page)
2038                 goto free_avic;
2039
2040         kvm_svm->avic_logical_id_table_page = l_page;
2041         clear_page(page_address(l_page));
2042
2043         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
2044  again:
2045         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
2046         if (vm_id == 0) { /* id is 1-based, zero is not okay */
2047                 next_vm_id_wrapped = 1;
2048                 goto again;
2049         }
2050         /* Is it still in use? Only possible if wrapped at least once */
2051         if (next_vm_id_wrapped) {
2052                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
2053                         if (k2->avic_vm_id == vm_id)
2054                                 goto again;
2055                 }
2056         }
2057         kvm_svm->avic_vm_id = vm_id;
2058         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
2059         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
2060
2061         return 0;
2062
2063 free_avic:
2064         avic_vm_destroy(kvm);
2065         return err;
2066 }
2067
2068 static int svm_vm_init(struct kvm *kvm)
2069 {
2070         if (avic) {
2071                 int ret = avic_vm_init(kvm);
2072                 if (ret)
2073                         return ret;
2074         }
2075
2076         kvm_apicv_init(kvm, avic);
2077         return 0;
2078 }
2079
2080 static inline int
2081 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
2082 {
2083         int ret = 0;
2084         unsigned long flags;
2085         struct amd_svm_iommu_ir *ir;
2086         struct vcpu_svm *svm = to_svm(vcpu);
2087
2088         if (!kvm_arch_has_assigned_device(vcpu->kvm))
2089                 return 0;
2090
2091         /*
2092          * Here, we go through the per-vcpu ir_list to update all existing
2093          * interrupt remapping table entry targeting this vcpu.
2094          */
2095         spin_lock_irqsave(&svm->ir_list_lock, flags);
2096
2097         if (list_empty(&svm->ir_list))
2098                 goto out;
2099
2100         list_for_each_entry(ir, &svm->ir_list, node) {
2101                 ret = amd_iommu_update_ga(cpu, r, ir->data);
2102                 if (ret)
2103                         break;
2104         }
2105 out:
2106         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
2107         return ret;
2108 }
2109
2110 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2111 {
2112         u64 entry;
2113         /* ID = 0xff (broadcast), ID > 0xff (reserved) */
2114         int h_physical_id = kvm_cpu_get_apicid(cpu);
2115         struct vcpu_svm *svm = to_svm(vcpu);
2116
2117         if (!kvm_vcpu_apicv_active(vcpu))
2118                 return;
2119
2120         /*
2121          * Since the host physical APIC id is 8 bits,
2122          * we can support host APIC ID upto 255.
2123          */
2124         if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
2125                 return;
2126
2127         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2128         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
2129
2130         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
2131         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
2132
2133         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2134         if (svm->avic_is_running)
2135                 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2136
2137         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2138         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
2139                                         svm->avic_is_running);
2140 }
2141
2142 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
2143 {
2144         u64 entry;
2145         struct vcpu_svm *svm = to_svm(vcpu);
2146
2147         if (!kvm_vcpu_apicv_active(vcpu))
2148                 return;
2149
2150         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2151         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
2152                 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
2153
2154         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2155         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2156 }
2157
2158 /**
2159  * This function is called during VCPU halt/unhalt.
2160  */
2161 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
2162 {
2163         struct vcpu_svm *svm = to_svm(vcpu);
2164
2165         svm->avic_is_running = is_run;
2166         if (is_run)
2167                 avic_vcpu_load(vcpu, vcpu->cpu);
2168         else
2169                 avic_vcpu_put(vcpu);
2170 }
2171
2172 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
2173 {
2174         struct vcpu_svm *svm = to_svm(vcpu);
2175         u32 dummy;
2176         u32 eax = 1;
2177
2178         svm->spec_ctrl = 0;
2179         svm->virt_spec_ctrl = 0;
2180
2181         if (!init_event) {
2182                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
2183                                            MSR_IA32_APICBASE_ENABLE;
2184                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
2185                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
2186         }
2187         init_vmcb(svm);
2188
2189         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
2190         kvm_rdx_write(vcpu, eax);
2191
2192         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
2193                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
2194 }
2195
2196 static int avic_init_vcpu(struct vcpu_svm *svm)
2197 {
2198         int ret;
2199         struct kvm_vcpu *vcpu = &svm->vcpu;
2200
2201         if (!avic || !irqchip_in_kernel(vcpu->kvm))
2202                 return 0;
2203
2204         ret = avic_init_backing_page(&svm->vcpu);
2205         if (ret)
2206                 return ret;
2207
2208         INIT_LIST_HEAD(&svm->ir_list);
2209         spin_lock_init(&svm->ir_list_lock);
2210         svm->dfr_reg = APIC_DFR_FLAT;
2211
2212         return ret;
2213 }
2214
2215 static int svm_create_vcpu(struct kvm_vcpu *vcpu)
2216 {
2217         struct vcpu_svm *svm;
2218         struct page *page;
2219         struct page *msrpm_pages;
2220         struct page *hsave_page;
2221         struct page *nested_msrpm_pages;
2222         int err;
2223
2224         BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
2225         svm = to_svm(vcpu);
2226
2227         err = -ENOMEM;
2228         page = alloc_page(GFP_KERNEL_ACCOUNT);
2229         if (!page)
2230                 goto out;
2231
2232         msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
2233         if (!msrpm_pages)
2234                 goto free_page1;
2235
2236         nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
2237         if (!nested_msrpm_pages)
2238                 goto free_page2;
2239
2240         hsave_page = alloc_page(GFP_KERNEL_ACCOUNT);
2241         if (!hsave_page)
2242                 goto free_page3;
2243
2244         err = avic_init_vcpu(svm);
2245         if (err)
2246                 goto free_page4;
2247
2248         /* We initialize this flag to true to make sure that the is_running
2249          * bit would be set the first time the vcpu is loaded.
2250          */
2251         if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm))
2252                 svm->avic_is_running = true;
2253
2254         svm->nested.hsave = page_address(hsave_page);
2255
2256         svm->msrpm = page_address(msrpm_pages);
2257         svm_vcpu_init_msrpm(svm->msrpm);
2258
2259         svm->nested.msrpm = page_address(nested_msrpm_pages);
2260         svm_vcpu_init_msrpm(svm->nested.msrpm);
2261
2262         svm->vmcb = page_address(page);
2263         clear_page(svm->vmcb);
2264         svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2265         svm->asid_generation = 0;
2266         init_vmcb(svm);
2267
2268         svm_init_osvw(vcpu);
2269         vcpu->arch.microcode_version = 0x01000065;
2270
2271         return 0;
2272
2273 free_page4:
2274         __free_page(hsave_page);
2275 free_page3:
2276         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2277 free_page2:
2278         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2279 free_page1:
2280         __free_page(page);
2281 out:
2282         return err;
2283 }
2284
2285 static void svm_clear_current_vmcb(struct vmcb *vmcb)
2286 {
2287         int i;
2288
2289         for_each_online_cpu(i)
2290                 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
2291 }
2292
2293 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2294 {
2295         struct vcpu_svm *svm = to_svm(vcpu);
2296
2297         /*
2298          * The vmcb page can be recycled, causing a false negative in
2299          * svm_vcpu_load(). So, ensure that no logical CPU has this
2300          * vmcb page recorded as its current vmcb.
2301          */
2302         svm_clear_current_vmcb(svm->vmcb);
2303
2304         __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2305         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2306         __free_page(virt_to_page(svm->nested.hsave));
2307         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
2308 }
2309
2310 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2311 {
2312         struct vcpu_svm *svm = to_svm(vcpu);
2313         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2314         int i;
2315
2316         if (unlikely(cpu != vcpu->cpu)) {
2317                 svm->asid_generation = 0;
2318                 mark_all_dirty(svm->vmcb);
2319         }
2320
2321 #ifdef CONFIG_X86_64
2322         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2323 #endif
2324         savesegment(fs, svm->host.fs);
2325         savesegment(gs, svm->host.gs);
2326         svm->host.ldt = kvm_read_ldt();
2327
2328         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2329                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2330
2331         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2332                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2333                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2334                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
2335                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2336                 }
2337         }
2338         /* This assumes that the kernel never uses MSR_TSC_AUX */
2339         if (static_cpu_has(X86_FEATURE_RDTSCP))
2340                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2341
2342         if (sd->current_vmcb != svm->vmcb) {
2343                 sd->current_vmcb = svm->vmcb;
2344                 indirect_branch_prediction_barrier();
2345         }
2346         avic_vcpu_load(vcpu, cpu);
2347 }
2348
2349 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2350 {
2351         struct vcpu_svm *svm = to_svm(vcpu);
2352         int i;
2353
2354         avic_vcpu_put(vcpu);
2355
2356         ++vcpu->stat.host_state_reload;
2357         kvm_load_ldt(svm->host.ldt);
2358 #ifdef CONFIG_X86_64
2359         loadsegment(fs, svm->host.fs);
2360         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2361         load_gs_index(svm->host.gs);
2362 #else
2363 #ifdef CONFIG_X86_32_LAZY_GS
2364         loadsegment(gs, svm->host.gs);
2365 #endif
2366 #endif
2367         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2368                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2369 }
2370
2371 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2372 {
2373         avic_set_running(vcpu, false);
2374 }
2375
2376 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2377 {
2378         if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
2379                 kvm_vcpu_update_apicv(vcpu);
2380         avic_set_running(vcpu, true);
2381 }
2382
2383 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2384 {
2385         struct vcpu_svm *svm = to_svm(vcpu);
2386         unsigned long rflags = svm->vmcb->save.rflags;
2387
2388         if (svm->nmi_singlestep) {
2389                 /* Hide our flags if they were not set by the guest */
2390                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2391                         rflags &= ~X86_EFLAGS_TF;
2392                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2393                         rflags &= ~X86_EFLAGS_RF;
2394         }
2395         return rflags;
2396 }
2397
2398 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2399 {
2400         if (to_svm(vcpu)->nmi_singlestep)
2401                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2402
2403        /*
2404         * Any change of EFLAGS.VM is accompanied by a reload of SS
2405         * (caused by either a task switch or an inter-privilege IRET),
2406         * so we do not need to update the CPL here.
2407         */
2408         to_svm(vcpu)->vmcb->save.rflags = rflags;
2409 }
2410
2411 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2412 {
2413         switch (reg) {
2414         case VCPU_EXREG_PDPTR:
2415                 BUG_ON(!npt_enabled);
2416                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
2417                 break;
2418         default:
2419                 WARN_ON_ONCE(1);
2420         }
2421 }
2422
2423 static void svm_set_vintr(struct vcpu_svm *svm)
2424 {
2425         set_intercept(svm, INTERCEPT_VINTR);
2426 }
2427
2428 static void svm_clear_vintr(struct vcpu_svm *svm)
2429 {
2430         clr_intercept(svm, INTERCEPT_VINTR);
2431 }
2432
2433 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2434 {
2435         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2436
2437         switch (seg) {
2438         case VCPU_SREG_CS: return &save->cs;
2439         case VCPU_SREG_DS: return &save->ds;
2440         case VCPU_SREG_ES: return &save->es;
2441         case VCPU_SREG_FS: return &save->fs;
2442         case VCPU_SREG_GS: return &save->gs;
2443         case VCPU_SREG_SS: return &save->ss;
2444         case VCPU_SREG_TR: return &save->tr;
2445         case VCPU_SREG_LDTR: return &save->ldtr;
2446         }
2447         BUG();
2448         return NULL;
2449 }
2450
2451 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2452 {
2453         struct vmcb_seg *s = svm_seg(vcpu, seg);
2454
2455         return s->base;
2456 }
2457
2458 static void svm_get_segment(struct kvm_vcpu *vcpu,
2459                             struct kvm_segment *var, int seg)
2460 {
2461         struct vmcb_seg *s = svm_seg(vcpu, seg);
2462
2463         var->base = s->base;
2464         var->limit = s->limit;
2465         var->selector = s->selector;
2466         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2467         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2468         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2469         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2470         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2471         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2472         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2473
2474         /*
2475          * AMD CPUs circa 2014 track the G bit for all segments except CS.
2476          * However, the SVM spec states that the G bit is not observed by the
2477          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2478          * So let's synthesize a legal G bit for all segments, this helps
2479          * running KVM nested. It also helps cross-vendor migration, because
2480          * Intel's vmentry has a check on the 'G' bit.
2481          */
2482         var->g = s->limit > 0xfffff;
2483
2484         /*
2485          * AMD's VMCB does not have an explicit unusable field, so emulate it
2486          * for cross vendor migration purposes by "not present"
2487          */
2488         var->unusable = !var->present;
2489
2490         switch (seg) {
2491         case VCPU_SREG_TR:
2492                 /*
2493                  * Work around a bug where the busy flag in the tr selector
2494                  * isn't exposed
2495                  */
2496                 var->type |= 0x2;
2497                 break;
2498         case VCPU_SREG_DS:
2499         case VCPU_SREG_ES:
2500         case VCPU_SREG_FS:
2501         case VCPU_SREG_GS:
2502                 /*
2503                  * The accessed bit must always be set in the segment
2504                  * descriptor cache, although it can be cleared in the
2505                  * descriptor, the cached bit always remains at 1. Since
2506                  * Intel has a check on this, set it here to support
2507                  * cross-vendor migration.
2508                  */
2509                 if (!var->unusable)
2510                         var->type |= 0x1;
2511                 break;
2512         case VCPU_SREG_SS:
2513                 /*
2514                  * On AMD CPUs sometimes the DB bit in the segment
2515                  * descriptor is left as 1, although the whole segment has
2516                  * been made unusable. Clear it here to pass an Intel VMX
2517                  * entry check when cross vendor migrating.
2518                  */
2519                 if (var->unusable)
2520                         var->db = 0;
2521                 /* This is symmetric with svm_set_segment() */
2522                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2523                 break;
2524         }
2525 }
2526
2527 static int svm_get_cpl(struct kvm_vcpu *vcpu)
2528 {
2529         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2530
2531         return save->cpl;
2532 }
2533
2534 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2535 {
2536         struct vcpu_svm *svm = to_svm(vcpu);
2537
2538         dt->size = svm->vmcb->save.idtr.limit;
2539         dt->address = svm->vmcb->save.idtr.base;
2540 }
2541
2542 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2543 {
2544         struct vcpu_svm *svm = to_svm(vcpu);
2545
2546         svm->vmcb->save.idtr.limit = dt->size;
2547         svm->vmcb->save.idtr.base = dt->address ;
2548         mark_dirty(svm->vmcb, VMCB_DT);
2549 }
2550
2551 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2552 {
2553         struct vcpu_svm *svm = to_svm(vcpu);
2554
2555         dt->size = svm->vmcb->save.gdtr.limit;
2556         dt->address = svm->vmcb->save.gdtr.base;
2557 }
2558
2559 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2560 {
2561         struct vcpu_svm *svm = to_svm(vcpu);
2562
2563         svm->vmcb->save.gdtr.limit = dt->size;
2564         svm->vmcb->save.gdtr.base = dt->address ;
2565         mark_dirty(svm->vmcb, VMCB_DT);
2566 }
2567
2568 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2569 {
2570 }
2571
2572 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2573 {
2574 }
2575
2576 static void update_cr0_intercept(struct vcpu_svm *svm)
2577 {
2578         ulong gcr0 = svm->vcpu.arch.cr0;
2579         u64 *hcr0 = &svm->vmcb->save.cr0;
2580
2581         *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2582                 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
2583
2584         mark_dirty(svm->vmcb, VMCB_CR);
2585
2586         if (gcr0 == *hcr0) {
2587                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2588                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2589         } else {
2590                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2591                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2592         }
2593 }
2594
2595 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2596 {
2597         struct vcpu_svm *svm = to_svm(vcpu);
2598
2599 #ifdef CONFIG_X86_64
2600         if (vcpu->arch.efer & EFER_LME) {
2601                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2602                         vcpu->arch.efer |= EFER_LMA;
2603                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
2604                 }
2605
2606                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2607                         vcpu->arch.efer &= ~EFER_LMA;
2608                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2609                 }
2610         }
2611 #endif
2612         vcpu->arch.cr0 = cr0;
2613
2614         if (!npt_enabled)
2615                 cr0 |= X86_CR0_PG | X86_CR0_WP;
2616
2617         /*
2618          * re-enable caching here because the QEMU bios
2619          * does not do it - this results in some delay at
2620          * reboot
2621          */
2622         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2623                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2624         svm->vmcb->save.cr0 = cr0;
2625         mark_dirty(svm->vmcb, VMCB_CR);
2626         update_cr0_intercept(svm);
2627 }
2628
2629 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2630 {
2631         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2632         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2633
2634         if (cr4 & X86_CR4_VMXE)
2635                 return 1;
2636
2637         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2638                 svm_flush_tlb(vcpu, true);
2639
2640         vcpu->arch.cr4 = cr4;
2641         if (!npt_enabled)
2642                 cr4 |= X86_CR4_PAE;
2643         cr4 |= host_cr4_mce;
2644         to_svm(vcpu)->vmcb->save.cr4 = cr4;
2645         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2646         return 0;
2647 }
2648
2649 static void svm_set_segment(struct kvm_vcpu *vcpu,
2650                             struct kvm_segment *var, int seg)
2651 {
2652         struct vcpu_svm *svm = to_svm(vcpu);
2653         struct vmcb_seg *s = svm_seg(vcpu, seg);
2654
2655         s->base = var->base;
2656         s->limit = var->limit;
2657         s->selector = var->selector;
2658         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2659         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2660         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2661         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2662         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2663         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2664         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2665         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2666
2667         /*
2668          * This is always accurate, except if SYSRET returned to a segment
2669          * with SS.DPL != 3.  Intel does not have this quirk, and always
2670          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2671          * would entail passing the CPL to userspace and back.
2672          */
2673         if (seg == VCPU_SREG_SS)
2674                 /* This is symmetric with svm_get_segment() */
2675                 svm->vmcb->save.cpl = (var->dpl & 3);
2676
2677         mark_dirty(svm->vmcb, VMCB_SEG);
2678 }
2679
2680 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2681 {
2682         struct vcpu_svm *svm = to_svm(vcpu);
2683
2684         clr_exception_intercept(svm, BP_VECTOR);
2685
2686         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2687                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2688                         set_exception_intercept(svm, BP_VECTOR);
2689         } else
2690                 vcpu->guest_debug = 0;
2691 }
2692
2693 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2694 {
2695         if (sd->next_asid > sd->max_asid) {
2696                 ++sd->asid_generation;
2697                 sd->next_asid = sd->min_asid;
2698                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2699         }
2700
2701         svm->asid_generation = sd->asid_generation;
2702         svm->vmcb->control.asid = sd->next_asid++;
2703
2704         mark_dirty(svm->vmcb, VMCB_ASID);
2705 }
2706
2707 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2708 {
2709         return to_svm(vcpu)->vmcb->save.dr6;
2710 }
2711
2712 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2713 {
2714         struct vcpu_svm *svm = to_svm(vcpu);
2715
2716         svm->vmcb->save.dr6 = value;
2717         mark_dirty(svm->vmcb, VMCB_DR);
2718 }
2719
2720 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2721 {
2722         struct vcpu_svm *svm = to_svm(vcpu);
2723
2724         get_debugreg(vcpu->arch.db[0], 0);
2725         get_debugreg(vcpu->arch.db[1], 1);
2726         get_debugreg(vcpu->arch.db[2], 2);
2727         get_debugreg(vcpu->arch.db[3], 3);
2728         vcpu->arch.dr6 = svm_get_dr6(vcpu);
2729         vcpu->arch.dr7 = svm->vmcb->save.dr7;
2730
2731         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2732         set_dr_intercepts(svm);
2733 }
2734
2735 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2736 {
2737         struct vcpu_svm *svm = to_svm(vcpu);
2738
2739         svm->vmcb->save.dr7 = value;
2740         mark_dirty(svm->vmcb, VMCB_DR);
2741 }
2742
2743 static int pf_interception(struct vcpu_svm *svm)
2744 {
2745         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2746         u64 error_code = svm->vmcb->control.exit_info_1;
2747
2748         return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2749                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2750                         svm->vmcb->control.insn_bytes : NULL,
2751                         svm->vmcb->control.insn_len);
2752 }
2753
2754 static int npf_interception(struct vcpu_svm *svm)
2755 {
2756         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2757         u64 error_code = svm->vmcb->control.exit_info_1;
2758
2759         trace_kvm_page_fault(fault_address, error_code);
2760         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2761                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2762                         svm->vmcb->control.insn_bytes : NULL,
2763                         svm->vmcb->control.insn_len);
2764 }
2765
2766 static int db_interception(struct vcpu_svm *svm)
2767 {
2768         struct kvm_run *kvm_run = svm->vcpu.run;
2769         struct kvm_vcpu *vcpu = &svm->vcpu;
2770
2771         if (!(svm->vcpu.guest_debug &
2772               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2773                 !svm->nmi_singlestep) {
2774                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2775                 return 1;
2776         }
2777
2778         if (svm->nmi_singlestep) {
2779                 disable_nmi_singlestep(svm);
2780                 /* Make sure we check for pending NMIs upon entry */
2781                 kvm_make_request(KVM_REQ_EVENT, vcpu);
2782         }
2783
2784         if (svm->vcpu.guest_debug &
2785             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2786                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2787                 kvm_run->debug.arch.pc =
2788                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2789                 kvm_run->debug.arch.exception = DB_VECTOR;
2790                 return 0;
2791         }
2792
2793         return 1;
2794 }
2795
2796 static int bp_interception(struct vcpu_svm *svm)
2797 {
2798         struct kvm_run *kvm_run = svm->vcpu.run;
2799
2800         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2801         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2802         kvm_run->debug.arch.exception = BP_VECTOR;
2803         return 0;
2804 }
2805
2806 static int ud_interception(struct vcpu_svm *svm)
2807 {
2808         return handle_ud(&svm->vcpu);
2809 }
2810
2811 static int ac_interception(struct vcpu_svm *svm)
2812 {
2813         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2814         return 1;
2815 }
2816
2817 static int gp_interception(struct vcpu_svm *svm)
2818 {
2819         struct kvm_vcpu *vcpu = &svm->vcpu;
2820         u32 error_code = svm->vmcb->control.exit_info_1;
2821
2822         WARN_ON_ONCE(!enable_vmware_backdoor);
2823
2824         /*
2825          * VMware backdoor emulation on #GP interception only handles IN{S},
2826          * OUT{S}, and RDPMC, none of which generate a non-zero error code.
2827          */
2828         if (error_code) {
2829                 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2830                 return 1;
2831         }
2832         return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
2833 }
2834
2835 static bool is_erratum_383(void)
2836 {
2837         int err, i;
2838         u64 value;
2839
2840         if (!erratum_383_found)
2841                 return false;
2842
2843         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2844         if (err)
2845                 return false;
2846
2847         /* Bit 62 may or may not be set for this mce */
2848         value &= ~(1ULL << 62);
2849
2850         if (value != 0xb600000000010015ULL)
2851                 return false;
2852
2853         /* Clear MCi_STATUS registers */
2854         for (i = 0; i < 6; ++i)
2855                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2856
2857         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2858         if (!err) {
2859                 u32 low, high;
2860
2861                 value &= ~(1ULL << 2);
2862                 low    = lower_32_bits(value);
2863                 high   = upper_32_bits(value);
2864
2865                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2866         }
2867
2868         /* Flush tlb to evict multi-match entries */
2869         __flush_tlb_all();
2870
2871         return true;
2872 }
2873
2874 static void svm_handle_mce(struct vcpu_svm *svm)
2875 {
2876         if (is_erratum_383()) {
2877                 /*
2878                  * Erratum 383 triggered. Guest state is corrupt so kill the
2879                  * guest.
2880                  */
2881                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2882
2883                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2884
2885                 return;
2886         }
2887
2888         /*
2889          * On an #MC intercept the MCE handler is not called automatically in
2890          * the host. So do it by hand here.
2891          */
2892         asm volatile (
2893                 "int $0x12\n");
2894         /* not sure if we ever come back to this point */
2895
2896         return;
2897 }
2898
2899 static int mc_interception(struct vcpu_svm *svm)
2900 {
2901         return 1;
2902 }
2903
2904 static int shutdown_interception(struct vcpu_svm *svm)
2905 {
2906         struct kvm_run *kvm_run = svm->vcpu.run;
2907
2908         /*
2909          * VMCB is undefined after a SHUTDOWN intercept
2910          * so reinitialize it.
2911          */
2912         clear_page(svm->vmcb);
2913         init_vmcb(svm);
2914
2915         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2916         return 0;
2917 }
2918
2919 static int io_interception(struct vcpu_svm *svm)
2920 {
2921         struct kvm_vcpu *vcpu = &svm->vcpu;
2922         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2923         int size, in, string;
2924         unsigned port;
2925
2926         ++svm->vcpu.stat.io_exits;
2927         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2928         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2929         if (string)
2930                 return kvm_emulate_instruction(vcpu, 0);
2931
2932         port = io_info >> 16;
2933         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2934         svm->next_rip = svm->vmcb->control.exit_info_2;
2935
2936         return kvm_fast_pio(&svm->vcpu, size, port, in);
2937 }
2938
2939 static int nmi_interception(struct vcpu_svm *svm)
2940 {
2941         return 1;
2942 }
2943
2944 static int intr_interception(struct vcpu_svm *svm)
2945 {
2946         ++svm->vcpu.stat.irq_exits;
2947         return 1;
2948 }
2949
2950 static int nop_on_interception(struct vcpu_svm *svm)
2951 {
2952         return 1;
2953 }
2954
2955 static int halt_interception(struct vcpu_svm *svm)
2956 {
2957         return kvm_emulate_halt(&svm->vcpu);
2958 }
2959
2960 static int vmmcall_interception(struct vcpu_svm *svm)
2961 {
2962         return kvm_emulate_hypercall(&svm->vcpu);
2963 }
2964
2965 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2966 {
2967         struct vcpu_svm *svm = to_svm(vcpu);
2968
2969         return svm->nested.nested_cr3;
2970 }
2971
2972 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2973 {
2974         struct vcpu_svm *svm = to_svm(vcpu);
2975         u64 cr3 = svm->nested.nested_cr3;
2976         u64 pdpte;
2977         int ret;
2978
2979         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2980                                        offset_in_page(cr3) + index * 8, 8);
2981         if (ret)
2982                 return 0;
2983         return pdpte;
2984 }
2985
2986 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2987                                    unsigned long root)
2988 {
2989         struct vcpu_svm *svm = to_svm(vcpu);
2990
2991         svm->vmcb->control.nested_cr3 = __sme_set(root);
2992         mark_dirty(svm->vmcb, VMCB_NPT);
2993 }
2994
2995 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2996                                        struct x86_exception *fault)
2997 {
2998         struct vcpu_svm *svm = to_svm(vcpu);
2999
3000         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
3001                 /*
3002                  * TODO: track the cause of the nested page fault, and
3003                  * correctly fill in the high bits of exit_info_1.
3004                  */
3005                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
3006                 svm->vmcb->control.exit_code_hi = 0;
3007                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
3008                 svm->vmcb->control.exit_info_2 = fault->address;
3009         }
3010
3011         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
3012         svm->vmcb->control.exit_info_1 |= fault->error_code;
3013
3014         /*
3015          * The present bit is always zero for page structure faults on real
3016          * hardware.
3017          */
3018         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
3019                 svm->vmcb->control.exit_info_1 &= ~1;
3020
3021         nested_svm_vmexit(svm);
3022 }
3023
3024 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
3025 {
3026         WARN_ON(mmu_is_nested(vcpu));
3027
3028         vcpu->arch.mmu = &vcpu->arch.guest_mmu;
3029         kvm_init_shadow_mmu(vcpu);
3030         vcpu->arch.mmu->set_cr3           = nested_svm_set_tdp_cr3;
3031         vcpu->arch.mmu->get_cr3           = nested_svm_get_tdp_cr3;
3032         vcpu->arch.mmu->get_pdptr         = nested_svm_get_tdp_pdptr;
3033         vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
3034         vcpu->arch.mmu->shadow_root_level = get_npt_level(vcpu);
3035         reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu);
3036         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
3037 }
3038
3039 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
3040 {
3041         vcpu->arch.mmu = &vcpu->arch.root_mmu;
3042         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
3043 }
3044
3045 static int nested_svm_check_permissions(struct vcpu_svm *svm)
3046 {
3047         if (!(svm->vcpu.arch.efer & EFER_SVME) ||
3048             !is_paging(&svm->vcpu)) {
3049                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3050                 return 1;
3051         }
3052
3053         if (svm->vmcb->save.cpl) {
3054                 kvm_inject_gp(&svm->vcpu, 0);
3055                 return 1;
3056         }
3057
3058         return 0;
3059 }
3060
3061 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
3062                                       bool has_error_code, u32 error_code)
3063 {
3064         int vmexit;
3065
3066         if (!is_guest_mode(&svm->vcpu))
3067                 return 0;
3068
3069         vmexit = nested_svm_intercept(svm);
3070         if (vmexit != NESTED_EXIT_DONE)
3071                 return 0;
3072
3073         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
3074         svm->vmcb->control.exit_code_hi = 0;
3075         svm->vmcb->control.exit_info_1 = error_code;
3076
3077         /*
3078          * EXITINFO2 is undefined for all exception intercepts other
3079          * than #PF.
3080          */
3081         if (svm->vcpu.arch.exception.nested_apf)
3082                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
3083         else if (svm->vcpu.arch.exception.has_payload)
3084                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload;
3085         else
3086                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
3087
3088         svm->nested.exit_required = true;
3089         return vmexit;
3090 }
3091
3092 /* This function returns true if it is save to enable the irq window */
3093 static inline bool nested_svm_intr(struct vcpu_svm *svm)
3094 {
3095         if (!is_guest_mode(&svm->vcpu))
3096                 return true;
3097
3098         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3099                 return true;
3100
3101         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
3102                 return false;
3103
3104         /*
3105          * if vmexit was already requested (by intercepted exception
3106          * for instance) do not overwrite it with "external interrupt"
3107          * vmexit.
3108          */
3109         if (svm->nested.exit_required)
3110                 return false;
3111
3112         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
3113         svm->vmcb->control.exit_info_1 = 0;
3114         svm->vmcb->control.exit_info_2 = 0;
3115
3116         if (svm->nested.intercept & 1ULL) {
3117                 /*
3118                  * The #vmexit can't be emulated here directly because this
3119                  * code path runs with irqs and preemption disabled. A
3120                  * #vmexit emulation might sleep. Only signal request for
3121                  * the #vmexit here.
3122                  */
3123                 svm->nested.exit_required = true;
3124                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
3125                 return false;
3126         }
3127
3128         return true;
3129 }
3130
3131 /* This function returns true if it is save to enable the nmi window */
3132 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
3133 {
3134         if (!is_guest_mode(&svm->vcpu))
3135                 return true;
3136
3137         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
3138                 return true;
3139
3140         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
3141         svm->nested.exit_required = true;
3142
3143         return false;
3144 }
3145
3146 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
3147 {
3148         unsigned port, size, iopm_len;
3149         u16 val, mask;
3150         u8 start_bit;
3151         u64 gpa;
3152
3153         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
3154                 return NESTED_EXIT_HOST;
3155
3156         port = svm->vmcb->control.exit_info_1 >> 16;
3157         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
3158                 SVM_IOIO_SIZE_SHIFT;
3159         gpa  = svm->nested.vmcb_iopm + (port / 8);
3160         start_bit = port % 8;
3161         iopm_len = (start_bit + size > 8) ? 2 : 1;
3162         mask = (0xf >> (4 - size)) << start_bit;
3163         val = 0;
3164
3165         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
3166                 return NESTED_EXIT_DONE;
3167
3168         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3169 }
3170
3171 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
3172 {
3173         u32 offset, msr, value;
3174         int write, mask;
3175
3176         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3177                 return NESTED_EXIT_HOST;
3178
3179         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
3180         offset = svm_msrpm_offset(msr);
3181         write  = svm->vmcb->control.exit_info_1 & 1;
3182         mask   = 1 << ((2 * (msr & 0xf)) + write);
3183
3184         if (offset == MSR_INVALID)
3185                 return NESTED_EXIT_DONE;
3186
3187         /* Offset is in 32 bit units but need in 8 bit units */
3188         offset *= 4;
3189
3190         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
3191                 return NESTED_EXIT_DONE;
3192
3193         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3194 }
3195
3196 /* DB exceptions for our internal use must not cause vmexit */
3197 static int nested_svm_intercept_db(struct vcpu_svm *svm)
3198 {
3199         unsigned long dr6;
3200
3201         /* if we're not singlestepping, it's not ours */
3202         if (!svm->nmi_singlestep)
3203                 return NESTED_EXIT_DONE;
3204
3205         /* if it's not a singlestep exception, it's not ours */
3206         if (kvm_get_dr(&svm->vcpu, 6, &dr6))
3207                 return NESTED_EXIT_DONE;
3208         if (!(dr6 & DR6_BS))
3209                 return NESTED_EXIT_DONE;
3210
3211         /* if the guest is singlestepping, it should get the vmexit */
3212         if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
3213                 disable_nmi_singlestep(svm);
3214                 return NESTED_EXIT_DONE;
3215         }
3216
3217         /* it's ours, the nested hypervisor must not see this one */
3218         return NESTED_EXIT_HOST;
3219 }
3220
3221 static int nested_svm_exit_special(struct vcpu_svm *svm)
3222 {
3223         u32 exit_code = svm->vmcb->control.exit_code;
3224
3225         switch (exit_code) {
3226         case SVM_EXIT_INTR:
3227         case SVM_EXIT_NMI:
3228         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
3229                 return NESTED_EXIT_HOST;
3230         case SVM_EXIT_NPF:
3231                 /* For now we are always handling NPFs when using them */
3232                 if (npt_enabled)
3233                         return NESTED_EXIT_HOST;
3234                 break;
3235         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
3236                 /* When we're shadowing, trap PFs, but not async PF */
3237                 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
3238                         return NESTED_EXIT_HOST;
3239                 break;
3240         default:
3241                 break;
3242         }
3243
3244         return NESTED_EXIT_CONTINUE;
3245 }
3246
3247 /*
3248  * If this function returns true, this #vmexit was already handled
3249  */
3250 static int nested_svm_intercept(struct vcpu_svm *svm)
3251 {
3252         u32 exit_code = svm->vmcb->control.exit_code;
3253         int vmexit = NESTED_EXIT_HOST;
3254
3255         switch (exit_code) {
3256         case SVM_EXIT_MSR:
3257                 vmexit = nested_svm_exit_handled_msr(svm);
3258                 break;
3259         case SVM_EXIT_IOIO:
3260                 vmexit = nested_svm_intercept_ioio(svm);
3261                 break;
3262         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3263                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3264                 if (svm->nested.intercept_cr & bit)
3265                         vmexit = NESTED_EXIT_DONE;
3266                 break;
3267         }
3268         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3269                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3270                 if (svm->nested.intercept_dr & bit)
3271                         vmexit = NESTED_EXIT_DONE;
3272                 break;
3273         }
3274         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3275                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3276                 if (svm->nested.intercept_exceptions & excp_bits) {
3277                         if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3278                                 vmexit = nested_svm_intercept_db(svm);
3279                         else
3280                                 vmexit = NESTED_EXIT_DONE;
3281                 }
3282                 /* async page fault always cause vmexit */
3283                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3284                          svm->vcpu.arch.exception.nested_apf != 0)
3285                         vmexit = NESTED_EXIT_DONE;
3286                 break;
3287         }
3288         case SVM_EXIT_ERR: {
3289                 vmexit = NESTED_EXIT_DONE;
3290                 break;
3291         }
3292         default: {
3293                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
3294                 if (svm->nested.intercept & exit_bits)
3295                         vmexit = NESTED_EXIT_DONE;
3296         }
3297         }
3298
3299         return vmexit;
3300 }
3301
3302 static int nested_svm_exit_handled(struct vcpu_svm *svm)
3303 {
3304         int vmexit;
3305
3306         vmexit = nested_svm_intercept(svm);
3307
3308         if (vmexit == NESTED_EXIT_DONE)
3309                 nested_svm_vmexit(svm);
3310
3311         return vmexit;
3312 }
3313
3314 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3315 {
3316         struct vmcb_control_area *dst  = &dst_vmcb->control;
3317         struct vmcb_control_area *from = &from_vmcb->control;
3318
3319         dst->intercept_cr         = from->intercept_cr;
3320         dst->intercept_dr         = from->intercept_dr;
3321         dst->intercept_exceptions = from->intercept_exceptions;
3322         dst->intercept            = from->intercept;
3323         dst->iopm_base_pa         = from->iopm_base_pa;
3324         dst->msrpm_base_pa        = from->msrpm_base_pa;
3325         dst->tsc_offset           = from->tsc_offset;
3326         dst->asid                 = from->asid;
3327         dst->tlb_ctl              = from->tlb_ctl;
3328         dst->int_ctl              = from->int_ctl;
3329         dst->int_vector           = from->int_vector;
3330         dst->int_state            = from->int_state;
3331         dst->exit_code            = from->exit_code;
3332         dst->exit_code_hi         = from->exit_code_hi;
3333         dst->exit_info_1          = from->exit_info_1;
3334         dst->exit_info_2          = from->exit_info_2;
3335         dst->exit_int_info        = from->exit_int_info;
3336         dst->exit_int_info_err    = from->exit_int_info_err;
3337         dst->nested_ctl           = from->nested_ctl;
3338         dst->event_inj            = from->event_inj;
3339         dst->event_inj_err        = from->event_inj_err;
3340         dst->nested_cr3           = from->nested_cr3;
3341         dst->virt_ext              = from->virt_ext;
3342         dst->pause_filter_count   = from->pause_filter_count;
3343         dst->pause_filter_thresh  = from->pause_filter_thresh;
3344 }
3345
3346 static int nested_svm_vmexit(struct vcpu_svm *svm)
3347 {
3348         int rc;
3349         struct vmcb *nested_vmcb;
3350         struct vmcb *hsave = svm->nested.hsave;
3351         struct vmcb *vmcb = svm->vmcb;
3352         struct kvm_host_map map;
3353
3354         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3355                                        vmcb->control.exit_info_1,
3356                                        vmcb->control.exit_info_2,
3357                                        vmcb->control.exit_int_info,
3358                                        vmcb->control.exit_int_info_err,
3359                                        KVM_ISA_SVM);
3360
3361         rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map);
3362         if (rc) {
3363                 if (rc == -EINVAL)
3364                         kvm_inject_gp(&svm->vcpu, 0);
3365                 return 1;
3366         }
3367
3368         nested_vmcb = map.hva;
3369
3370         /* Exit Guest-Mode */
3371         leave_guest_mode(&svm->vcpu);
3372         svm->nested.vmcb = 0;
3373
3374         /* Give the current vmcb to the guest */
3375         disable_gif(svm);
3376
3377         nested_vmcb->save.es     = vmcb->save.es;
3378         nested_vmcb->save.cs     = vmcb->save.cs;
3379         nested_vmcb->save.ss     = vmcb->save.ss;
3380         nested_vmcb->save.ds     = vmcb->save.ds;
3381         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
3382         nested_vmcb->save.idtr   = vmcb->save.idtr;
3383         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3384         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3385         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
3386         nested_vmcb->save.cr2    = vmcb->save.cr2;
3387         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3388         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
3389         nested_vmcb->save.rip    = vmcb->save.rip;
3390         nested_vmcb->save.rsp    = vmcb->save.rsp;
3391         nested_vmcb->save.rax    = vmcb->save.rax;
3392         nested_vmcb->save.dr7    = vmcb->save.dr7;
3393         nested_vmcb->save.dr6    = vmcb->save.dr6;
3394         nested_vmcb->save.cpl    = vmcb->save.cpl;
3395
3396         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
3397         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
3398         nested_vmcb->control.int_state         = vmcb->control.int_state;
3399         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
3400         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
3401         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
3402         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
3403         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
3404         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3405
3406         if (svm->nrips_enabled)
3407                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3408
3409         /*
3410          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3411          * to make sure that we do not lose injected events. So check event_inj
3412          * here and copy it to exit_int_info if it is valid.
3413          * Exit_int_info and event_inj can't be both valid because the case
3414          * below only happens on a VMRUN instruction intercept which has
3415          * no valid exit_int_info set.
3416          */
3417         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3418                 struct vmcb_control_area *nc = &nested_vmcb->control;
3419
3420                 nc->exit_int_info     = vmcb->control.event_inj;
3421                 nc->exit_int_info_err = vmcb->control.event_inj_err;
3422         }
3423
3424         nested_vmcb->control.tlb_ctl           = 0;
3425         nested_vmcb->control.event_inj         = 0;
3426         nested_vmcb->control.event_inj_err     = 0;
3427
3428         nested_vmcb->control.pause_filter_count =
3429                 svm->vmcb->control.pause_filter_count;
3430         nested_vmcb->control.pause_filter_thresh =
3431                 svm->vmcb->control.pause_filter_thresh;
3432
3433         /* We always set V_INTR_MASKING and remember the old value in hflags */
3434         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3435                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3436
3437         /* Restore the original control entries */
3438         copy_vmcb_control_area(vmcb, hsave);
3439
3440         svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset;
3441         kvm_clear_exception_queue(&svm->vcpu);
3442         kvm_clear_interrupt_queue(&svm->vcpu);
3443
3444         svm->nested.nested_cr3 = 0;
3445
3446         /* Restore selected save entries */
3447         svm->vmcb->save.es = hsave->save.es;
3448         svm->vmcb->save.cs = hsave->save.cs;
3449         svm->vmcb->save.ss = hsave->save.ss;
3450         svm->vmcb->save.ds = hsave->save.ds;
3451         svm->vmcb->save.gdtr = hsave->save.gdtr;
3452         svm->vmcb->save.idtr = hsave->save.idtr;
3453         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3454         svm_set_efer(&svm->vcpu, hsave->save.efer);
3455         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3456         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3457         if (npt_enabled) {
3458                 svm->vmcb->save.cr3 = hsave->save.cr3;
3459                 svm->vcpu.arch.cr3 = hsave->save.cr3;
3460         } else {
3461                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3462         }
3463         kvm_rax_write(&svm->vcpu, hsave->save.rax);
3464         kvm_rsp_write(&svm->vcpu, hsave->save.rsp);
3465         kvm_rip_write(&svm->vcpu, hsave->save.rip);
3466         svm->vmcb->save.dr7 = 0;
3467         svm->vmcb->save.cpl = 0;
3468         svm->vmcb->control.exit_int_info = 0;
3469
3470         mark_all_dirty(svm->vmcb);
3471
3472         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3473
3474         nested_svm_uninit_mmu_context(&svm->vcpu);
3475         kvm_mmu_reset_context(&svm->vcpu);
3476         kvm_mmu_load(&svm->vcpu);
3477
3478         /*
3479          * Drop what we picked up for L2 via svm_complete_interrupts() so it
3480          * doesn't end up in L1.
3481          */
3482         svm->vcpu.arch.nmi_injected = false;
3483         kvm_clear_exception_queue(&svm->vcpu);
3484         kvm_clear_interrupt_queue(&svm->vcpu);
3485
3486         return 0;
3487 }
3488
3489 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3490 {
3491         /*
3492          * This function merges the msr permission bitmaps of kvm and the
3493          * nested vmcb. It is optimized in that it only merges the parts where
3494          * the kvm msr permission bitmap may contain zero bits
3495          */
3496         int i;
3497
3498         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3499                 return true;
3500
3501         for (i = 0; i < MSRPM_OFFSETS; i++) {
3502                 u32 value, p;
3503                 u64 offset;
3504
3505                 if (msrpm_offsets[i] == 0xffffffff)
3506                         break;
3507
3508                 p      = msrpm_offsets[i];
3509                 offset = svm->nested.vmcb_msrpm + (p * 4);
3510
3511                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3512                         return false;
3513
3514                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3515         }
3516
3517         svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3518
3519         return true;
3520 }
3521
3522 static bool nested_vmcb_checks(struct vmcb *vmcb)
3523 {
3524         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3525                 return false;
3526
3527         if (vmcb->control.asid == 0)
3528                 return false;
3529
3530         if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3531             !npt_enabled)
3532                 return false;
3533
3534         return true;
3535 }
3536
3537 static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
3538                                  struct vmcb *nested_vmcb, struct kvm_host_map *map)
3539 {
3540         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
3541                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3542         else
3543                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3544
3545         if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3546                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3547                 nested_svm_init_mmu_context(&svm->vcpu);
3548         }
3549
3550         /* Load the nested guest state */
3551         svm->vmcb->save.es = nested_vmcb->save.es;
3552         svm->vmcb->save.cs = nested_vmcb->save.cs;
3553         svm->vmcb->save.ss = nested_vmcb->save.ss;
3554         svm->vmcb->save.ds = nested_vmcb->save.ds;
3555         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3556         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3557         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
3558         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3559         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3560         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3561         if (npt_enabled) {
3562                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3563                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3564         } else
3565                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3566
3567         /* Guest paging mode is active - reset mmu */
3568         kvm_mmu_reset_context(&svm->vcpu);
3569
3570         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3571         kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax);
3572         kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp);
3573         kvm_rip_write(&svm->vcpu, nested_vmcb->save.rip);
3574
3575         /* In case we don't even reach vcpu_run, the fields are not updated */
3576         svm->vmcb->save.rax = nested_vmcb->save.rax;
3577         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3578         svm->vmcb->save.rip = nested_vmcb->save.rip;
3579         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3580         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3581         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3582
3583         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3584         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
3585
3586         /* cache intercepts */
3587         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3588         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
3589         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3590         svm->nested.intercept            = nested_vmcb->control.intercept;
3591
3592         svm_flush_tlb(&svm->vcpu, true);
3593         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3594         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3595                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3596         else
3597                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3598
3599         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3600                 /* We only want the cr8 intercept bits of the guest */
3601                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3602                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3603         }
3604
3605         /* We don't want to see VMMCALLs from a nested guest */
3606         clr_intercept(svm, INTERCEPT_VMMCALL);
3607
3608         svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset;
3609         svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset;
3610
3611         svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3612         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3613         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3614         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3615         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3616
3617         svm->vmcb->control.pause_filter_count =
3618                 nested_vmcb->control.pause_filter_count;
3619         svm->vmcb->control.pause_filter_thresh =
3620                 nested_vmcb->control.pause_filter_thresh;
3621
3622         kvm_vcpu_unmap(&svm->vcpu, map, true);
3623
3624         /* Enter Guest-Mode */
3625         enter_guest_mode(&svm->vcpu);
3626
3627         /*
3628          * Merge guest and host intercepts - must be called  with vcpu in
3629          * guest-mode to take affect here
3630          */
3631         recalc_intercepts(svm);
3632
3633         svm->nested.vmcb = vmcb_gpa;
3634
3635         enable_gif(svm);
3636
3637         mark_all_dirty(svm->vmcb);
3638 }
3639
3640 static int nested_svm_vmrun(struct vcpu_svm *svm)
3641 {
3642         int ret;
3643         struct vmcb *nested_vmcb;
3644         struct vmcb *hsave = svm->nested.hsave;
3645         struct vmcb *vmcb = svm->vmcb;
3646         struct kvm_host_map map;
3647         u64 vmcb_gpa;
3648
3649         vmcb_gpa = svm->vmcb->save.rax;
3650
3651         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
3652         if (ret == -EINVAL) {
3653                 kvm_inject_gp(&svm->vcpu, 0);
3654                 return 1;
3655         } else if (ret) {
3656                 return kvm_skip_emulated_instruction(&svm->vcpu);
3657         }
3658
3659         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3660
3661         nested_vmcb = map.hva;
3662
3663         if (!nested_vmcb_checks(nested_vmcb)) {
3664                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
3665                 nested_vmcb->control.exit_code_hi = 0;
3666                 nested_vmcb->control.exit_info_1  = 0;
3667                 nested_vmcb->control.exit_info_2  = 0;
3668
3669                 kvm_vcpu_unmap(&svm->vcpu, &map, true);
3670
3671                 return ret;
3672         }
3673
3674         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3675                                nested_vmcb->save.rip,
3676                                nested_vmcb->control.int_ctl,
3677                                nested_vmcb->control.event_inj,
3678                                nested_vmcb->control.nested_ctl);
3679
3680         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3681                                     nested_vmcb->control.intercept_cr >> 16,
3682                                     nested_vmcb->control.intercept_exceptions,
3683                                     nested_vmcb->control.intercept);
3684
3685         /* Clear internal status */
3686         kvm_clear_exception_queue(&svm->vcpu);
3687         kvm_clear_interrupt_queue(&svm->vcpu);
3688
3689         /*
3690          * Save the old vmcb, so we don't need to pick what we save, but can
3691          * restore everything when a VMEXIT occurs
3692          */
3693         hsave->save.es     = vmcb->save.es;
3694         hsave->save.cs     = vmcb->save.cs;
3695         hsave->save.ss     = vmcb->save.ss;
3696         hsave->save.ds     = vmcb->save.ds;
3697         hsave->save.gdtr   = vmcb->save.gdtr;
3698         hsave->save.idtr   = vmcb->save.idtr;
3699         hsave->save.efer   = svm->vcpu.arch.efer;
3700         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
3701         hsave->save.cr4    = svm->vcpu.arch.cr4;
3702         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3703         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
3704         hsave->save.rsp    = vmcb->save.rsp;
3705         hsave->save.rax    = vmcb->save.rax;
3706         if (npt_enabled)
3707                 hsave->save.cr3    = vmcb->save.cr3;
3708         else
3709                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
3710
3711         copy_vmcb_control_area(hsave, vmcb);
3712
3713         enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);
3714
3715         if (!nested_svm_vmrun_msrpm(svm)) {
3716                 svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
3717                 svm->vmcb->control.exit_code_hi = 0;
3718                 svm->vmcb->control.exit_info_1  = 0;
3719                 svm->vmcb->control.exit_info_2  = 0;
3720
3721                 nested_svm_vmexit(svm);
3722         }
3723
3724         return ret;
3725 }
3726
3727 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3728 {
3729         to_vmcb->save.fs = from_vmcb->save.fs;
3730         to_vmcb->save.gs = from_vmcb->save.gs;
3731         to_vmcb->save.tr = from_vmcb->save.tr;
3732         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3733         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3734         to_vmcb->save.star = from_vmcb->save.star;
3735         to_vmcb->save.lstar = from_vmcb->save.lstar;
3736         to_vmcb->save.cstar = from_vmcb->save.cstar;
3737         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3738         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3739         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3740         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3741 }
3742
3743 static int vmload_interception(struct vcpu_svm *svm)
3744 {
3745         struct vmcb *nested_vmcb;
3746         struct kvm_host_map map;
3747         int ret;
3748
3749         if (nested_svm_check_permissions(svm))
3750                 return 1;
3751
3752         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3753         if (ret) {
3754                 if (ret == -EINVAL)
3755                         kvm_inject_gp(&svm->vcpu, 0);
3756                 return 1;
3757         }
3758
3759         nested_vmcb = map.hva;
3760
3761         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3762
3763         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3764         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3765
3766         return ret;
3767 }
3768
3769 static int vmsave_interception(struct vcpu_svm *svm)
3770 {
3771         struct vmcb *nested_vmcb;
3772         struct kvm_host_map map;
3773         int ret;
3774
3775         if (nested_svm_check_permissions(svm))
3776                 return 1;
3777
3778         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3779         if (ret) {
3780                 if (ret == -EINVAL)
3781                         kvm_inject_gp(&svm->vcpu, 0);
3782                 return 1;
3783         }
3784
3785         nested_vmcb = map.hva;
3786
3787         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3788
3789         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3790         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3791
3792         return ret;
3793 }
3794
3795 static int vmrun_interception(struct vcpu_svm *svm)
3796 {
3797         if (nested_svm_check_permissions(svm))
3798                 return 1;
3799
3800         return nested_svm_vmrun(svm);
3801 }
3802
3803 static int stgi_interception(struct vcpu_svm *svm)
3804 {
3805         int ret;
3806
3807         if (nested_svm_check_permissions(svm))
3808                 return 1;
3809
3810         /*
3811          * If VGIF is enabled, the STGI intercept is only added to
3812          * detect the opening of the SMI/NMI window; remove it now.
3813          */
3814         if (vgif_enabled(svm))
3815                 clr_intercept(svm, INTERCEPT_STGI);
3816
3817         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3818         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3819
3820         enable_gif(svm);
3821
3822         return ret;
3823 }
3824
3825 static int clgi_interception(struct vcpu_svm *svm)
3826 {
3827         int ret;
3828
3829         if (nested_svm_check_permissions(svm))
3830                 return 1;
3831
3832         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3833
3834         disable_gif(svm);
3835
3836         /* After a CLGI no interrupts should come */
3837         if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3838                 svm_clear_vintr(svm);
3839                 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3840                 mark_dirty(svm->vmcb, VMCB_INTR);
3841         }
3842
3843         return ret;
3844 }
3845
3846 static int invlpga_interception(struct vcpu_svm *svm)
3847 {
3848         struct kvm_vcpu *vcpu = &svm->vcpu;
3849
3850         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),
3851                           kvm_rax_read(&svm->vcpu));
3852
3853         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3854         kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
3855
3856         return kvm_skip_emulated_instruction(&svm->vcpu);
3857 }
3858
3859 static int skinit_interception(struct vcpu_svm *svm)
3860 {
3861         trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));
3862
3863         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3864         return 1;
3865 }
3866
3867 static int wbinvd_interception(struct vcpu_svm *svm)
3868 {
3869         return kvm_emulate_wbinvd(&svm->vcpu);
3870 }
3871
3872 static int xsetbv_interception(struct vcpu_svm *svm)
3873 {
3874         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
3875         u32 index = kvm_rcx_read(&svm->vcpu);
3876
3877         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3878                 return kvm_skip_emulated_instruction(&svm->vcpu);
3879         }
3880
3881         return 1;
3882 }
3883
3884 static int rdpru_interception(struct vcpu_svm *svm)
3885 {
3886         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3887         return 1;
3888 }
3889
3890 static int task_switch_interception(struct vcpu_svm *svm)
3891 {
3892         u16 tss_selector;
3893         int reason;
3894         int int_type = svm->vmcb->control.exit_int_info &
3895                 SVM_EXITINTINFO_TYPE_MASK;
3896         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3897         uint32_t type =
3898                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3899         uint32_t idt_v =
3900                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3901         bool has_error_code = false;
3902         u32 error_code = 0;
3903
3904         tss_selector = (u16)svm->vmcb->control.exit_info_1;
3905
3906         if (svm->vmcb->control.exit_info_2 &
3907             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3908                 reason = TASK_SWITCH_IRET;
3909         else if (svm->vmcb->control.exit_info_2 &
3910                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3911                 reason = TASK_SWITCH_JMP;
3912         else if (idt_v)
3913                 reason = TASK_SWITCH_GATE;
3914         else
3915                 reason = TASK_SWITCH_CALL;
3916
3917         if (reason == TASK_SWITCH_GATE) {
3918                 switch (type) {
3919                 case SVM_EXITINTINFO_TYPE_NMI:
3920                         svm->vcpu.arch.nmi_injected = false;
3921                         break;
3922                 case SVM_EXITINTINFO_TYPE_EXEPT:
3923                         if (svm->vmcb->control.exit_info_2 &
3924                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3925                                 has_error_code = true;
3926                                 error_code =
3927                                         (u32)svm->vmcb->control.exit_info_2;
3928                         }
3929                         kvm_clear_exception_queue(&svm->vcpu);
3930                         break;
3931                 case SVM_EXITINTINFO_TYPE_INTR:
3932                         kvm_clear_interrupt_queue(&svm->vcpu);
3933                         break;
3934                 default:
3935                         break;
3936                 }
3937         }
3938
3939         if (reason != TASK_SWITCH_GATE ||
3940             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3941             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3942              (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
3943                 if (!skip_emulated_instruction(&svm->vcpu))
3944                         return 0;
3945         }
3946
3947         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3948                 int_vec = -1;
3949
3950         return kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3951                                has_error_code, error_code);
3952 }
3953
3954 static int cpuid_interception(struct vcpu_svm *svm)
3955 {
3956         return kvm_emulate_cpuid(&svm->vcpu);
3957 }
3958
3959 static int iret_interception(struct vcpu_svm *svm)
3960 {
3961         ++svm->vcpu.stat.nmi_window_exits;
3962         clr_intercept(svm, INTERCEPT_IRET);
3963         svm->vcpu.arch.hflags |= HF_IRET_MASK;
3964         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3965         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3966         return 1;
3967 }
3968
3969 static int invlpg_interception(struct vcpu_svm *svm)
3970 {
3971         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3972                 return kvm_emulate_instruction(&svm->vcpu, 0);
3973
3974         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3975         return kvm_skip_emulated_instruction(&svm->vcpu);
3976 }
3977
3978 static int emulate_on_interception(struct vcpu_svm *svm)
3979 {
3980         return kvm_emulate_instruction(&svm->vcpu, 0);
3981 }
3982
3983 static int rsm_interception(struct vcpu_svm *svm)
3984 {
3985         return kvm_emulate_instruction_from_buffer(&svm->vcpu, rsm_ins_bytes, 2);
3986 }
3987
3988 static int rdpmc_interception(struct vcpu_svm *svm)
3989 {
3990         int err;
3991
3992         if (!nrips)
3993                 return emulate_on_interception(svm);
3994
3995         err = kvm_rdpmc(&svm->vcpu);
3996         return kvm_complete_insn_gp(&svm->vcpu, err);
3997 }
3998
3999 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
4000                                             unsigned long val)
4001 {
4002         unsigned long cr0 = svm->vcpu.arch.cr0;
4003         bool ret = false;
4004         u64 intercept;
4005
4006         intercept = svm->nested.intercept;
4007
4008         if (!is_guest_mode(&svm->vcpu) ||
4009             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
4010                 return false;
4011
4012         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
4013         val &= ~SVM_CR0_SELECTIVE_MASK;
4014
4015         if (cr0 ^ val) {
4016                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4017                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
4018         }
4019
4020         return ret;
4021 }
4022
4023 #define CR_VALID (1ULL << 63)
4024
4025 static int cr_interception(struct vcpu_svm *svm)
4026 {
4027         int reg, cr;
4028         unsigned long val;
4029         int err;
4030
4031         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
4032                 return emulate_on_interception(svm);
4033
4034         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
4035                 return emulate_on_interception(svm);
4036
4037         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4038         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
4039                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
4040         else
4041                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
4042
4043         err = 0;
4044         if (cr >= 16) { /* mov to cr */
4045                 cr -= 16;
4046                 val = kvm_register_read(&svm->vcpu, reg);
4047                 switch (cr) {
4048                 case 0:
4049                         if (!check_selective_cr0_intercepted(svm, val))
4050                                 err = kvm_set_cr0(&svm->vcpu, val);
4051                         else
4052                                 return 1;
4053
4054                         break;
4055                 case 3:
4056                         err = kvm_set_cr3(&svm->vcpu, val);
4057                         break;
4058                 case 4:
4059                         err = kvm_set_cr4(&svm->vcpu, val);
4060                         break;
4061                 case 8:
4062                         err = kvm_set_cr8(&svm->vcpu, val);
4063                         break;
4064                 default:
4065                         WARN(1, "unhandled write to CR%d", cr);
4066                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4067                         return 1;
4068                 }
4069         } else { /* mov from cr */
4070                 switch (cr) {
4071                 case 0:
4072                         val = kvm_read_cr0(&svm->vcpu);
4073                         break;
4074                 case 2:
4075                         val = svm->vcpu.arch.cr2;
4076                         break;
4077                 case 3:
4078                         val = kvm_read_cr3(&svm->vcpu);
4079                         break;
4080                 case 4:
4081                         val = kvm_read_cr4(&svm->vcpu);
4082                         break;
4083                 case 8:
4084                         val = kvm_get_cr8(&svm->vcpu);
4085                         break;
4086                 default:
4087                         WARN(1, "unhandled read from CR%d", cr);
4088                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4089                         return 1;
4090                 }
4091                 kvm_register_write(&svm->vcpu, reg, val);
4092         }
4093         return kvm_complete_insn_gp(&svm->vcpu, err);
4094 }
4095
4096 static int dr_interception(struct vcpu_svm *svm)
4097 {
4098         int reg, dr;
4099         unsigned long val;
4100
4101         if (svm->vcpu.guest_debug == 0) {
4102                 /*
4103                  * No more DR vmexits; force a reload of the debug registers
4104                  * and reenter on this instruction.  The next vmexit will
4105                  * retrieve the full state of the debug registers.
4106                  */
4107                 clr_dr_intercepts(svm);
4108                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
4109                 return 1;
4110         }
4111
4112         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
4113                 return emulate_on_interception(svm);
4114
4115         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4116         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
4117
4118         if (dr >= 16) { /* mov to DRn */
4119                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
4120                         return 1;
4121                 val = kvm_register_read(&svm->vcpu, reg);
4122                 kvm_set_dr(&svm->vcpu, dr - 16, val);
4123         } else {
4124                 if (!kvm_require_dr(&svm->vcpu, dr))
4125                         return 1;
4126                 kvm_get_dr(&svm->vcpu, dr, &val);
4127                 kvm_register_write(&svm->vcpu, reg, val);
4128         }
4129
4130         return kvm_skip_emulated_instruction(&svm->vcpu);
4131 }
4132
4133 static int cr8_write_interception(struct vcpu_svm *svm)
4134 {
4135         struct kvm_run *kvm_run = svm->vcpu.run;
4136         int r;
4137
4138         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
4139         /* instruction emulation calls kvm_set_cr8() */
4140         r = cr_interception(svm);
4141         if (lapic_in_kernel(&svm->vcpu))
4142                 return r;
4143         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
4144                 return r;
4145         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
4146         return 0;
4147 }
4148
4149 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
4150 {
4151         msr->data = 0;
4152
4153         switch (msr->index) {
4154         case MSR_F10H_DECFG:
4155                 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
4156                         msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
4157                 break;
4158         default:
4159                 return 1;
4160         }
4161
4162         return 0;
4163 }
4164
4165 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4166 {
4167         struct vcpu_svm *svm = to_svm(vcpu);
4168
4169         switch (msr_info->index) {
4170         case MSR_STAR:
4171                 msr_info->data = svm->vmcb->save.star;
4172                 break;
4173 #ifdef CONFIG_X86_64
4174         case MSR_LSTAR:
4175                 msr_info->data = svm->vmcb->save.lstar;
4176                 break;
4177         case MSR_CSTAR:
4178                 msr_info->data = svm->vmcb->save.cstar;
4179                 break;
4180         case MSR_KERNEL_GS_BASE:
4181                 msr_info->data = svm->vmcb->save.kernel_gs_base;
4182                 break;
4183         case MSR_SYSCALL_MASK:
4184                 msr_info->data = svm->vmcb->save.sfmask;
4185                 break;
4186 #endif
4187         case MSR_IA32_SYSENTER_CS:
4188                 msr_info->data = svm->vmcb->save.sysenter_cs;
4189                 break;
4190         case MSR_IA32_SYSENTER_EIP:
4191                 msr_info->data = svm->sysenter_eip;
4192                 break;
4193         case MSR_IA32_SYSENTER_ESP:
4194                 msr_info->data = svm->sysenter_esp;
4195                 break;
4196         case MSR_TSC_AUX:
4197                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4198                         return 1;
4199                 msr_info->data = svm->tsc_aux;
4200                 break;
4201         /*
4202          * Nobody will change the following 5 values in the VMCB so we can
4203          * safely return them on rdmsr. They will always be 0 until LBRV is
4204          * implemented.
4205          */
4206         case MSR_IA32_DEBUGCTLMSR:
4207                 msr_info->data = svm->vmcb->save.dbgctl;
4208                 break;
4209         case MSR_IA32_LASTBRANCHFROMIP:
4210                 msr_info->data = svm->vmcb->save.br_from;
4211                 break;
4212         case MSR_IA32_LASTBRANCHTOIP:
4213                 msr_info->data = svm->vmcb->save.br_to;
4214                 break;
4215         case MSR_IA32_LASTINTFROMIP:
4216                 msr_info->data = svm->vmcb->save.last_excp_from;
4217                 break;
4218         case MSR_IA32_LASTINTTOIP:
4219                 msr_info->data = svm->vmcb->save.last_excp_to;
4220                 break;
4221         case MSR_VM_HSAVE_PA:
4222                 msr_info->data = svm->nested.hsave_msr;
4223                 break;
4224         case MSR_VM_CR:
4225                 msr_info->data = svm->nested.vm_cr_msr;
4226                 break;
4227         case MSR_IA32_SPEC_CTRL:
4228                 if (!msr_info->host_initiated &&
4229                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
4230                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) &&
4231                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4232                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4233                         return 1;
4234
4235                 msr_info->data = svm->spec_ctrl;
4236                 break;
4237         case MSR_AMD64_VIRT_SPEC_CTRL:
4238                 if (!msr_info->host_initiated &&
4239                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4240                         return 1;
4241
4242                 msr_info->data = svm->virt_spec_ctrl;
4243                 break;
4244         case MSR_F15H_IC_CFG: {
4245
4246                 int family, model;
4247
4248                 family = guest_cpuid_family(vcpu);
4249                 model  = guest_cpuid_model(vcpu);
4250
4251                 if (family < 0 || model < 0)
4252                         return kvm_get_msr_common(vcpu, msr_info);
4253
4254                 msr_info->data = 0;
4255
4256                 if (family == 0x15 &&
4257                     (model >= 0x2 && model < 0x20))
4258                         msr_info->data = 0x1E;
4259                 }
4260                 break;
4261         case MSR_F10H_DECFG:
4262                 msr_info->data = svm->msr_decfg;
4263                 break;
4264         default:
4265                 return kvm_get_msr_common(vcpu, msr_info);
4266         }
4267         return 0;
4268 }
4269
4270 static int rdmsr_interception(struct vcpu_svm *svm)
4271 {
4272         return kvm_emulate_rdmsr(&svm->vcpu);
4273 }
4274
4275 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
4276 {
4277         struct vcpu_svm *svm = to_svm(vcpu);
4278         int svm_dis, chg_mask;
4279
4280         if (data & ~SVM_VM_CR_VALID_MASK)
4281                 return 1;
4282
4283         chg_mask = SVM_VM_CR_VALID_MASK;
4284
4285         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
4286                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
4287
4288         svm->nested.vm_cr_msr &= ~chg_mask;
4289         svm->nested.vm_cr_msr |= (data & chg_mask);
4290
4291         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
4292
4293         /* check for svm_disable while efer.svme is set */
4294         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
4295                 return 1;
4296
4297         return 0;
4298 }
4299
4300 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
4301 {
4302         struct vcpu_svm *svm = to_svm(vcpu);
4303
4304         u32 ecx = msr->index;
4305         u64 data = msr->data;
4306         switch (ecx) {
4307         case MSR_IA32_CR_PAT:
4308                 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
4309                         return 1;
4310                 vcpu->arch.pat = data;
4311                 svm->vmcb->save.g_pat = data;
4312                 mark_dirty(svm->vmcb, VMCB_NPT);
4313                 break;
4314         case MSR_IA32_SPEC_CTRL:
4315                 if (!msr->host_initiated &&
4316                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
4317                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) &&
4318                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4319                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4320                         return 1;
4321
4322                 if (data & ~kvm_spec_ctrl_valid_bits(vcpu))
4323                         return 1;
4324
4325                 svm->spec_ctrl = data;
4326                 if (!data)
4327                         break;
4328
4329                 /*
4330                  * For non-nested:
4331                  * When it's written (to non-zero) for the first time, pass
4332                  * it through.
4333                  *
4334                  * For nested:
4335                  * The handling of the MSR bitmap for L2 guests is done in
4336                  * nested_svm_vmrun_msrpm.
4337                  * We update the L1 MSR bit as well since it will end up
4338                  * touching the MSR anyway now.
4339                  */
4340                 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
4341                 break;
4342         case MSR_IA32_PRED_CMD:
4343                 if (!msr->host_initiated &&
4344                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
4345                         return 1;
4346
4347                 if (data & ~PRED_CMD_IBPB)
4348                         return 1;
4349                 if (!boot_cpu_has(X86_FEATURE_AMD_IBPB))
4350                         return 1;
4351                 if (!data)
4352                         break;
4353
4354                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4355                 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
4356                 break;
4357         case MSR_AMD64_VIRT_SPEC_CTRL:
4358                 if (!msr->host_initiated &&
4359                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4360                         return 1;
4361
4362                 if (data & ~SPEC_CTRL_SSBD)
4363                         return 1;
4364
4365                 svm->virt_spec_ctrl = data;
4366                 break;
4367         case MSR_STAR:
4368                 svm->vmcb->save.star = data;
4369                 break;
4370 #ifdef CONFIG_X86_64
4371         case MSR_LSTAR:
4372                 svm->vmcb->save.lstar = data;
4373                 break;
4374         case MSR_CSTAR:
4375                 svm->vmcb->save.cstar = data;
4376                 break;
4377         case MSR_KERNEL_GS_BASE:
4378                 svm->vmcb->save.kernel_gs_base = data;
4379                 break;
4380         case MSR_SYSCALL_MASK:
4381                 svm->vmcb->save.sfmask = data;
4382                 break;
4383 #endif
4384         case MSR_IA32_SYSENTER_CS:
4385                 svm->vmcb->save.sysenter_cs = data;
4386                 break;
4387         case MSR_IA32_SYSENTER_EIP:
4388                 svm->sysenter_eip = data;
4389                 svm->vmcb->save.sysenter_eip = data;
4390                 break;
4391         case MSR_IA32_SYSENTER_ESP:
4392                 svm->sysenter_esp = data;
4393                 svm->vmcb->save.sysenter_esp = data;
4394                 break;
4395         case MSR_TSC_AUX:
4396                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4397                         return 1;
4398
4399                 /*
4400                  * This is rare, so we update the MSR here instead of using
4401                  * direct_access_msrs.  Doing that would require a rdmsr in
4402                  * svm_vcpu_put.
4403                  */
4404                 svm->tsc_aux = data;
4405                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
4406                 break;
4407         case MSR_IA32_DEBUGCTLMSR:
4408                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
4409                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
4410                                     __func__, data);
4411                         break;
4412                 }
4413                 if (data & DEBUGCTL_RESERVED_BITS)
4414                         return 1;
4415
4416                 svm->vmcb->save.dbgctl = data;
4417                 mark_dirty(svm->vmcb, VMCB_LBR);
4418                 if (data & (1ULL<<0))
4419                         svm_enable_lbrv(svm);
4420                 else
4421                         svm_disable_lbrv(svm);
4422                 break;
4423         case MSR_VM_HSAVE_PA:
4424                 svm->nested.hsave_msr = data;
4425                 break;
4426         case MSR_VM_CR:
4427                 return svm_set_vm_cr(vcpu, data);
4428         case MSR_VM_IGNNE:
4429                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
4430                 break;
4431         case MSR_F10H_DECFG: {
4432                 struct kvm_msr_entry msr_entry;
4433
4434                 msr_entry.index = msr->index;
4435                 if (svm_get_msr_feature(&msr_entry))
4436                         return 1;
4437
4438                 /* Check the supported bits */
4439                 if (data & ~msr_entry.data)
4440                         return 1;
4441
4442                 /* Don't allow the guest to change a bit, #GP */
4443                 if (!msr->host_initiated && (data ^ msr_entry.data))
4444                         return 1;
4445
4446                 svm->msr_decfg = data;
4447                 break;
4448         }
4449         case MSR_IA32_APICBASE:
4450                 if (kvm_vcpu_apicv_active(vcpu))
4451                         avic_update_vapic_bar(to_svm(vcpu), data);
4452                 /* Fall through */
4453         default:
4454                 return kvm_set_msr_common(vcpu, msr);
4455         }
4456         return 0;
4457 }
4458
4459 static int wrmsr_interception(struct vcpu_svm *svm)
4460 {
4461         return kvm_emulate_wrmsr(&svm->vcpu);
4462 }
4463
4464 static int msr_interception(struct vcpu_svm *svm)
4465 {
4466         if (svm->vmcb->control.exit_info_1)
4467                 return wrmsr_interception(svm);
4468         else
4469                 return rdmsr_interception(svm);
4470 }
4471
4472 static int interrupt_window_interception(struct vcpu_svm *svm)
4473 {
4474         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4475         svm_clear_vintr(svm);
4476
4477         /*
4478          * For AVIC, the only reason to end up here is ExtINTs.
4479          * In this case AVIC was temporarily disabled for
4480          * requesting the IRQ window and we have to re-enable it.
4481          */
4482         svm_toggle_avic_for_irq_window(&svm->vcpu, true);
4483
4484         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
4485         mark_dirty(svm->vmcb, VMCB_INTR);
4486         ++svm->vcpu.stat.irq_window_exits;
4487         return 1;
4488 }
4489
4490 static int pause_interception(struct vcpu_svm *svm)
4491 {
4492         struct kvm_vcpu *vcpu = &svm->vcpu;
4493         bool in_kernel = (svm_get_cpl(vcpu) == 0);
4494
4495         if (pause_filter_thresh)
4496                 grow_ple_window(vcpu);
4497
4498         kvm_vcpu_on_spin(vcpu, in_kernel);
4499         return 1;
4500 }
4501
4502 static int nop_interception(struct vcpu_svm *svm)
4503 {
4504         return kvm_skip_emulated_instruction(&(svm->vcpu));
4505 }
4506
4507 static int monitor_interception(struct vcpu_svm *svm)
4508 {
4509         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
4510         return nop_interception(svm);
4511 }
4512
4513 static int mwait_interception(struct vcpu_svm *svm)
4514 {
4515         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
4516         return nop_interception(svm);
4517 }
4518
4519 enum avic_ipi_failure_cause {
4520         AVIC_IPI_FAILURE_INVALID_INT_TYPE,
4521         AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
4522         AVIC_IPI_FAILURE_INVALID_TARGET,
4523         AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
4524 };
4525
4526 static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
4527 {
4528         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
4529         u32 icrl = svm->vmcb->control.exit_info_1;
4530         u32 id = svm->vmcb->control.exit_info_2 >> 32;
4531         u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
4532         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4533
4534         trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
4535
4536         switch (id) {
4537         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
4538                 /*
4539                  * AVIC hardware handles the generation of
4540                  * IPIs when the specified Message Type is Fixed
4541                  * (also known as fixed delivery mode) and
4542                  * the Trigger Mode is edge-triggered. The hardware
4543                  * also supports self and broadcast delivery modes
4544                  * specified via the Destination Shorthand(DSH)
4545                  * field of the ICRL. Logical and physical APIC ID
4546                  * formats are supported. All other IPI types cause
4547                  * a #VMEXIT, which needs to emulated.
4548                  */
4549                 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
4550                 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
4551                 break;
4552         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
4553                 int i;
4554                 struct kvm_vcpu *vcpu;
4555                 struct kvm *kvm = svm->vcpu.kvm;
4556                 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4557
4558                 /*
4559                  * At this point, we expect that the AVIC HW has already
4560                  * set the appropriate IRR bits on the valid target
4561                  * vcpus. So, we just need to kick the appropriate vcpu.
4562                  */
4563                 kvm_for_each_vcpu(i, vcpu, kvm) {
4564                         bool m = kvm_apic_match_dest(vcpu, apic,
4565                                                      icrl & APIC_SHORT_MASK,
4566                                                      GET_APIC_DEST_FIELD(icrh),
4567                                                      icrl & APIC_DEST_MASK);
4568
4569                         if (m && !avic_vcpu_is_running(vcpu))
4570                                 kvm_vcpu_wake_up(vcpu);
4571                 }
4572                 break;
4573         }
4574         case AVIC_IPI_FAILURE_INVALID_TARGET:
4575                 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
4576                           index, svm->vcpu.vcpu_id, icrh, icrl);
4577                 break;
4578         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
4579                 WARN_ONCE(1, "Invalid backing page\n");
4580                 break;
4581         default:
4582                 pr_err("Unknown IPI interception\n");
4583         }
4584
4585         return 1;
4586 }
4587
4588 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
4589 {
4590         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
4591         int index;
4592         u32 *logical_apic_id_table;
4593         int dlid = GET_APIC_LOGICAL_ID(ldr);
4594
4595         if (!dlid)
4596                 return NULL;
4597
4598         if (flat) { /* flat */
4599                 index = ffs(dlid) - 1;
4600                 if (index > 7)
4601                         return NULL;
4602         } else { /* cluster */
4603                 int cluster = (dlid & 0xf0) >> 4;
4604                 int apic = ffs(dlid & 0x0f) - 1;
4605
4606                 if ((apic < 0) || (apic > 7) ||
4607                     (cluster >= 0xf))
4608                         return NULL;
4609                 index = (cluster << 2) + apic;
4610         }
4611
4612         logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
4613
4614         return &logical_apic_id_table[index];
4615 }
4616
4617 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
4618 {
4619         bool flat;
4620         u32 *entry, new_entry;
4621
4622         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
4623         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
4624         if (!entry)
4625                 return -EINVAL;
4626
4627         new_entry = READ_ONCE(*entry);
4628         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
4629         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
4630         new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4631         WRITE_ONCE(*entry, new_entry);
4632
4633         return 0;
4634 }
4635
4636 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
4637 {
4638         struct vcpu_svm *svm = to_svm(vcpu);
4639         bool flat = svm->dfr_reg == APIC_DFR_FLAT;
4640         u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
4641
4642         if (entry)
4643                 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
4644 }
4645
4646 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
4647 {
4648         int ret = 0;
4649         struct vcpu_svm *svm = to_svm(vcpu);
4650         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
4651         u32 id = kvm_xapic_id(vcpu->arch.apic);
4652
4653         if (ldr == svm->ldr_reg)
4654                 return 0;
4655
4656         avic_invalidate_logical_id_entry(vcpu);
4657
4658         if (ldr)
4659                 ret = avic_ldr_write(vcpu, id, ldr);
4660
4661         if (!ret)
4662                 svm->ldr_reg = ldr;
4663
4664         return ret;
4665 }
4666
4667 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
4668 {
4669         u64 *old, *new;
4670         struct vcpu_svm *svm = to_svm(vcpu);
4671         u32 id = kvm_xapic_id(vcpu->arch.apic);
4672
4673         if (vcpu->vcpu_id == id)
4674                 return 0;
4675
4676         old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
4677         new = avic_get_physical_id_entry(vcpu, id);
4678         if (!new || !old)
4679                 return 1;
4680
4681         /* We need to move physical_id_entry to new offset */
4682         *new = *old;
4683         *old = 0ULL;
4684         to_svm(vcpu)->avic_physical_id_cache = new;
4685
4686         /*
4687          * Also update the guest physical APIC ID in the logical
4688          * APIC ID table entry if already setup the LDR.
4689          */
4690         if (svm->ldr_reg)
4691                 avic_handle_ldr_update(vcpu);
4692
4693         return 0;
4694 }
4695
4696 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
4697 {
4698         struct vcpu_svm *svm = to_svm(vcpu);
4699         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
4700
4701         if (svm->dfr_reg == dfr)
4702                 return;
4703
4704         avic_invalidate_logical_id_entry(vcpu);
4705         svm->dfr_reg = dfr;
4706 }
4707
4708 static int avic_unaccel_trap_write(struct vcpu_svm *svm)
4709 {
4710         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4711         u32 offset = svm->vmcb->control.exit_info_1 &
4712                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4713
4714         switch (offset) {
4715         case APIC_ID:
4716                 if (avic_handle_apic_id_update(&svm->vcpu))
4717                         return 0;
4718                 break;
4719         case APIC_LDR:
4720                 if (avic_handle_ldr_update(&svm->vcpu))
4721                         return 0;
4722                 break;
4723         case APIC_DFR:
4724                 avic_handle_dfr_update(&svm->vcpu);
4725                 break;
4726         default:
4727                 break;
4728         }
4729
4730         kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4731
4732         return 1;
4733 }
4734
4735 static bool is_avic_unaccelerated_access_trap(u32 offset)
4736 {
4737         bool ret = false;
4738
4739         switch (offset) {
4740         case APIC_ID:
4741         case APIC_EOI:
4742         case APIC_RRR:
4743         case APIC_LDR:
4744         case APIC_DFR:
4745         case APIC_SPIV:
4746         case APIC_ESR:
4747         case APIC_ICR:
4748         case APIC_LVTT:
4749         case APIC_LVTTHMR:
4750         case APIC_LVTPC:
4751         case APIC_LVT0:
4752         case APIC_LVT1:
4753         case APIC_LVTERR:
4754         case APIC_TMICT:
4755         case APIC_TDCR:
4756                 ret = true;
4757                 break;
4758         default:
4759                 break;
4760         }
4761         return ret;
4762 }
4763
4764 static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4765 {
4766         int ret = 0;
4767         u32 offset = svm->vmcb->control.exit_info_1 &
4768                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4769         u32 vector = svm->vmcb->control.exit_info_2 &
4770                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4771         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4772                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
4773         bool trap = is_avic_unaccelerated_access_trap(offset);
4774
4775         trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4776                                             trap, write, vector);
4777         if (trap) {
4778                 /* Handling Trap */
4779                 WARN_ONCE(!write, "svm: Handling trap read.\n");
4780                 ret = avic_unaccel_trap_write(svm);
4781         } else {
4782                 /* Handling Fault */
4783                 ret = kvm_emulate_instruction(&svm->vcpu, 0);
4784         }
4785
4786         return ret;
4787 }
4788
4789 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4790         [SVM_EXIT_READ_CR0]                     = cr_interception,
4791         [SVM_EXIT_READ_CR3]                     = cr_interception,
4792         [SVM_EXIT_READ_CR4]                     = cr_interception,
4793         [SVM_EXIT_READ_CR8]                     = cr_interception,
4794         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
4795         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
4796         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
4797         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
4798         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
4799         [SVM_EXIT_READ_DR0]                     = dr_interception,
4800         [SVM_EXIT_READ_DR1]                     = dr_interception,
4801         [SVM_EXIT_READ_DR2]                     = dr_interception,
4802         [SVM_EXIT_READ_DR3]                     = dr_interception,
4803         [SVM_EXIT_READ_DR4]                     = dr_interception,
4804         [SVM_EXIT_READ_DR5]                     = dr_interception,
4805         [SVM_EXIT_READ_DR6]                     = dr_interception,
4806         [SVM_EXIT_READ_DR7]                     = dr_interception,
4807         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
4808         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
4809         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
4810         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
4811         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
4812         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
4813         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
4814         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
4815         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
4816         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
4817         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
4818         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
4819         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
4820         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
4821         [SVM_EXIT_EXCP_BASE + GP_VECTOR]        = gp_interception,
4822         [SVM_EXIT_INTR]                         = intr_interception,
4823         [SVM_EXIT_NMI]                          = nmi_interception,
4824         [SVM_EXIT_SMI]                          = nop_on_interception,
4825         [SVM_EXIT_INIT]                         = nop_on_interception,
4826         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
4827         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
4828         [SVM_EXIT_CPUID]                        = cpuid_interception,
4829         [SVM_EXIT_IRET]                         = iret_interception,
4830         [SVM_EXIT_INVD]                         = emulate_on_interception,
4831         [SVM_EXIT_PAUSE]                        = pause_interception,
4832         [SVM_EXIT_HLT]                          = halt_interception,
4833         [SVM_EXIT_INVLPG]                       = invlpg_interception,
4834         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
4835         [SVM_EXIT_IOIO]                         = io_interception,
4836         [SVM_EXIT_MSR]                          = msr_interception,
4837         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
4838         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
4839         [SVM_EXIT_VMRUN]                        = vmrun_interception,
4840         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
4841         [SVM_EXIT_VMLOAD]                       = vmload_interception,
4842         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
4843         [SVM_EXIT_STGI]                         = stgi_interception,
4844         [SVM_EXIT_CLGI]                         = clgi_interception,
4845         [SVM_EXIT_SKINIT]                       = skinit_interception,
4846         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
4847         [SVM_EXIT_MONITOR]                      = monitor_interception,
4848         [SVM_EXIT_MWAIT]                        = mwait_interception,
4849         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
4850         [SVM_EXIT_RDPRU]                        = rdpru_interception,
4851         [SVM_EXIT_NPF]                          = npf_interception,
4852         [SVM_EXIT_RSM]                          = rsm_interception,
4853         [SVM_EXIT_AVIC_INCOMPLETE_IPI]          = avic_incomplete_ipi_interception,
4854         [SVM_EXIT_AVIC_UNACCELERATED_ACCESS]    = avic_unaccelerated_access_interception,
4855 };
4856
4857 static void dump_vmcb(struct kvm_vcpu *vcpu)
4858 {
4859         struct vcpu_svm *svm = to_svm(vcpu);
4860         struct vmcb_control_area *control = &svm->vmcb->control;
4861         struct vmcb_save_area *save = &svm->vmcb->save;
4862
4863         if (!dump_invalid_vmcb) {
4864                 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
4865                 return;
4866         }
4867
4868         pr_err("VMCB Control Area:\n");
4869         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4870         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4871         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4872         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4873         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4874         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4875         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
4876         pr_err("%-20s%d\n", "pause filter threshold:",
4877                control->pause_filter_thresh);
4878         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4879         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4880         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4881         pr_err("%-20s%d\n", "asid:", control->asid);
4882         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4883         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4884         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4885         pr_err("%-20s%08x\n", "int_state:", control->int_state);
4886         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4887         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4888         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4889         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4890         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4891         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4892         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4893         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4894         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4895         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4896         pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4897         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4898         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4899         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4900         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4901         pr_err("VMCB State Save Area:\n");
4902         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4903                "es:",
4904                save->es.selector, save->es.attrib,
4905                save->es.limit, save->es.base);
4906         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4907                "cs:",
4908                save->cs.selector, save->cs.attrib,
4909                save->cs.limit, save->cs.base);
4910         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4911                "ss:",
4912                save->ss.selector, save->ss.attrib,
4913                save->ss.limit, save->ss.base);
4914         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4915                "ds:",
4916                save->ds.selector, save->ds.attrib,
4917                save->ds.limit, save->ds.base);
4918         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4919                "fs:",
4920                save->fs.selector, save->fs.attrib,
4921                save->fs.limit, save->fs.base);
4922         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4923                "gs:",
4924                save->gs.selector, save->gs.attrib,
4925                save->gs.limit, save->gs.base);
4926         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4927                "gdtr:",
4928                save->gdtr.selector, save->gdtr.attrib,
4929                save->gdtr.limit, save->gdtr.base);
4930         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4931                "ldtr:",
4932                save->ldtr.selector, save->ldtr.attrib,
4933                save->ldtr.limit, save->ldtr.base);
4934         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4935                "idtr:",
4936                save->idtr.selector, save->idtr.attrib,
4937                save->idtr.limit, save->idtr.base);
4938         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4939                "tr:",
4940                save->tr.selector, save->tr.attrib,
4941                save->tr.limit, save->tr.base);
4942         pr_err("cpl:            %d                efer:         %016llx\n",
4943                 save->cpl, save->efer);
4944         pr_err("%-15s %016llx %-13s %016llx\n",
4945                "cr0:", save->cr0, "cr2:", save->cr2);
4946         pr_err("%-15s %016llx %-13s %016llx\n",
4947                "cr3:", save->cr3, "cr4:", save->cr4);
4948         pr_err("%-15s %016llx %-13s %016llx\n",
4949                "dr6:", save->dr6, "dr7:", save->dr7);
4950         pr_err("%-15s %016llx %-13s %016llx\n",
4951                "rip:", save->rip, "rflags:", save->rflags);
4952         pr_err("%-15s %016llx %-13s %016llx\n",
4953                "rsp:", save->rsp, "rax:", save->rax);
4954         pr_err("%-15s %016llx %-13s %016llx\n",
4955                "star:", save->star, "lstar:", save->lstar);
4956         pr_err("%-15s %016llx %-13s %016llx\n",
4957                "cstar:", save->cstar, "sfmask:", save->sfmask);
4958         pr_err("%-15s %016llx %-13s %016llx\n",
4959                "kernel_gs_base:", save->kernel_gs_base,
4960                "sysenter_cs:", save->sysenter_cs);
4961         pr_err("%-15s %016llx %-13s %016llx\n",
4962                "sysenter_esp:", save->sysenter_esp,
4963                "sysenter_eip:", save->sysenter_eip);
4964         pr_err("%-15s %016llx %-13s %016llx\n",
4965                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4966         pr_err("%-15s %016llx %-13s %016llx\n",
4967                "br_from:", save->br_from, "br_to:", save->br_to);
4968         pr_err("%-15s %016llx %-13s %016llx\n",
4969                "excp_from:", save->last_excp_from,
4970                "excp_to:", save->last_excp_to);
4971 }
4972
4973 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4974 {
4975         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4976
4977         *info1 = control->exit_info_1;
4978         *info2 = control->exit_info_2;
4979 }
4980
4981 static int handle_exit(struct kvm_vcpu *vcpu,
4982         enum exit_fastpath_completion exit_fastpath)
4983 {
4984         struct vcpu_svm *svm = to_svm(vcpu);
4985         struct kvm_run *kvm_run = vcpu->run;
4986         u32 exit_code = svm->vmcb->control.exit_code;
4987
4988         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4989
4990         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4991                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4992         if (npt_enabled)
4993                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
4994
4995         if (unlikely(svm->nested.exit_required)) {
4996                 nested_svm_vmexit(svm);
4997                 svm->nested.exit_required = false;
4998
4999                 return 1;
5000         }
5001
5002         if (is_guest_mode(vcpu)) {
5003                 int vmexit;
5004
5005                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
5006                                         svm->vmcb->control.exit_info_1,
5007                                         svm->vmcb->control.exit_info_2,
5008                                         svm->vmcb->control.exit_int_info,
5009                                         svm->vmcb->control.exit_int_info_err,
5010                                         KVM_ISA_SVM);
5011
5012                 vmexit = nested_svm_exit_special(svm);
5013
5014                 if (vmexit == NESTED_EXIT_CONTINUE)
5015                         vmexit = nested_svm_exit_handled(svm);
5016
5017                 if (vmexit == NESTED_EXIT_DONE)
5018                         return 1;
5019         }
5020
5021         svm_complete_interrupts(svm);
5022
5023         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
5024                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5025                 kvm_run->fail_entry.hardware_entry_failure_reason
5026                         = svm->vmcb->control.exit_code;
5027                 dump_vmcb(vcpu);
5028                 return 0;
5029         }
5030
5031         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
5032             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
5033             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
5034             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
5035                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
5036                        "exit_code 0x%x\n",
5037                        __func__, svm->vmcb->control.exit_int_info,
5038                        exit_code);
5039
5040         if (exit_fastpath == EXIT_FASTPATH_SKIP_EMUL_INS) {
5041                 kvm_skip_emulated_instruction(vcpu);
5042                 return 1;
5043         } else if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
5044             || !svm_exit_handlers[exit_code]) {
5045                 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
5046                 dump_vmcb(vcpu);
5047                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5048                 vcpu->run->internal.suberror =
5049                         KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
5050                 vcpu->run->internal.ndata = 1;
5051                 vcpu->run->internal.data[0] = exit_code;
5052                 return 0;
5053         }
5054
5055 #ifdef CONFIG_RETPOLINE
5056         if (exit_code == SVM_EXIT_MSR)
5057                 return msr_interception(svm);
5058         else if (exit_code == SVM_EXIT_VINTR)
5059                 return interrupt_window_interception(svm);
5060         else if (exit_code == SVM_EXIT_INTR)
5061                 return intr_interception(svm);
5062         else if (exit_code == SVM_EXIT_HLT)
5063                 return halt_interception(svm);
5064         else if (exit_code == SVM_EXIT_NPF)
5065                 return npf_interception(svm);
5066 #endif
5067         return svm_exit_handlers[exit_code](svm);
5068 }
5069
5070 static void reload_tss(struct kvm_vcpu *vcpu)
5071 {
5072         int cpu = raw_smp_processor_id();
5073
5074         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5075         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
5076         load_TR_desc();
5077 }
5078
5079 static void pre_sev_run(struct vcpu_svm *svm, int cpu)
5080 {
5081         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5082         int asid = sev_get_asid(svm->vcpu.kvm);
5083
5084         /* Assign the asid allocated with this SEV guest */
5085         svm->vmcb->control.asid = asid;
5086
5087         /*
5088          * Flush guest TLB:
5089          *
5090          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
5091          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
5092          */
5093         if (sd->sev_vmcbs[asid] == svm->vmcb &&
5094             svm->last_cpu == cpu)
5095                 return;
5096
5097         svm->last_cpu = cpu;
5098         sd->sev_vmcbs[asid] = svm->vmcb;
5099         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5100         mark_dirty(svm->vmcb, VMCB_ASID);
5101 }
5102
5103 static void pre_svm_run(struct vcpu_svm *svm)
5104 {
5105         int cpu = raw_smp_processor_id();
5106
5107         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5108
5109         if (sev_guest(svm->vcpu.kvm))
5110                 return pre_sev_run(svm, cpu);
5111
5112         /* FIXME: handle wraparound of asid_generation */
5113         if (svm->asid_generation != sd->asid_generation)
5114                 new_asid(svm, sd);
5115 }
5116
5117 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
5118 {
5119         struct vcpu_svm *svm = to_svm(vcpu);
5120
5121         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
5122         vcpu->arch.hflags |= HF_NMI_MASK;
5123         set_intercept(svm, INTERCEPT_IRET);
5124         ++vcpu->stat.nmi_injections;
5125 }
5126
5127 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
5128 {
5129         struct vmcb_control_area *control;
5130
5131         /* The following fields are ignored when AVIC is enabled */
5132         control = &svm->vmcb->control;
5133         control->int_vector = irq;
5134         control->int_ctl &= ~V_INTR_PRIO_MASK;
5135         control->int_ctl |= V_IRQ_MASK |
5136                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
5137         mark_dirty(svm->vmcb, VMCB_INTR);
5138 }
5139
5140 static void svm_set_irq(struct kvm_vcpu *vcpu)
5141 {
5142         struct vcpu_svm *svm = to_svm(vcpu);
5143
5144         BUG_ON(!(gif_set(svm)));
5145
5146         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
5147         ++vcpu->stat.irq_injections;
5148
5149         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
5150                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
5151 }
5152
5153 static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
5154 {
5155         return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
5156 }
5157
5158 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
5159 {
5160         struct vcpu_svm *svm = to_svm(vcpu);
5161
5162         if (svm_nested_virtualize_tpr(vcpu))
5163                 return;
5164
5165         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5166
5167         if (irr == -1)
5168                 return;
5169
5170         if (tpr >= irr)
5171                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5172 }
5173
5174 static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
5175 {
5176         return;
5177 }
5178
5179 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
5180 {
5181 }
5182
5183 static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
5184 {
5185 }
5186
5187 static void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate)
5188 {
5189         if (!avic || !lapic_in_kernel(vcpu))
5190                 return;
5191
5192         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
5193         kvm_request_apicv_update(vcpu->kvm, activate,
5194                                  APICV_INHIBIT_REASON_IRQWIN);
5195         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5196 }
5197
5198 static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate)
5199 {
5200         int ret = 0;
5201         unsigned long flags;
5202         struct amd_svm_iommu_ir *ir;
5203         struct vcpu_svm *svm = to_svm(vcpu);
5204
5205         if (!kvm_arch_has_assigned_device(vcpu->kvm))
5206                 return 0;
5207
5208         /*
5209          * Here, we go through the per-vcpu ir_list to update all existing
5210          * interrupt remapping table entry targeting this vcpu.
5211          */
5212         spin_lock_irqsave(&svm->ir_list_lock, flags);
5213
5214         if (list_empty(&svm->ir_list))
5215                 goto out;
5216
5217         list_for_each_entry(ir, &svm->ir_list, node) {
5218                 if (activate)
5219                         ret = amd_iommu_activate_guest_mode(ir->data);
5220                 else
5221                         ret = amd_iommu_deactivate_guest_mode(ir->data);
5222                 if (ret)
5223                         break;
5224         }
5225 out:
5226         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5227         return ret;
5228 }
5229
5230 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5231 {
5232         struct vcpu_svm *svm = to_svm(vcpu);
5233         struct vmcb *vmcb = svm->vmcb;
5234         bool activated = kvm_vcpu_apicv_active(vcpu);
5235
5236         if (!avic)
5237                 return;
5238
5239         if (activated) {
5240                 /**
5241                  * During AVIC temporary deactivation, guest could update
5242                  * APIC ID, DFR and LDR registers, which would not be trapped
5243                  * by avic_unaccelerated_access_interception(). In this case,
5244                  * we need to check and update the AVIC logical APIC ID table
5245                  * accordingly before re-activating.
5246                  */
5247                 avic_post_state_restore(vcpu);
5248                 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
5249         } else {
5250                 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
5251         }
5252         mark_dirty(vmcb, VMCB_AVIC);
5253
5254         svm_set_pi_irte_mode(vcpu, activated);
5255 }
5256
5257 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
5258 {
5259         return;
5260 }
5261
5262 static int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
5263 {
5264         if (!vcpu->arch.apicv_active)
5265                 return -1;
5266
5267         kvm_lapic_set_irr(vec, vcpu->arch.apic);
5268         smp_mb__after_atomic();
5269
5270         if (avic_vcpu_is_running(vcpu)) {
5271                 int cpuid = vcpu->cpu;
5272
5273                 if (cpuid != get_cpu())
5274                         wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid));
5275                 put_cpu();
5276         } else
5277                 kvm_vcpu_wake_up(vcpu);
5278
5279         return 0;
5280 }
5281
5282 static bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu)
5283 {
5284         return false;
5285 }
5286
5287 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5288 {
5289         unsigned long flags;
5290         struct amd_svm_iommu_ir *cur;
5291
5292         spin_lock_irqsave(&svm->ir_list_lock, flags);
5293         list_for_each_entry(cur, &svm->ir_list, node) {
5294                 if (cur->data != pi->ir_data)
5295                         continue;
5296                 list_del(&cur->node);
5297                 kfree(cur);
5298                 break;
5299         }
5300         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5301 }
5302
5303 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5304 {
5305         int ret = 0;
5306         unsigned long flags;
5307         struct amd_svm_iommu_ir *ir;
5308
5309         /**
5310          * In some cases, the existing irte is updaed and re-set,
5311          * so we need to check here if it's already been * added
5312          * to the ir_list.
5313          */
5314         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
5315                 struct kvm *kvm = svm->vcpu.kvm;
5316                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
5317                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
5318                 struct vcpu_svm *prev_svm;
5319
5320                 if (!prev_vcpu) {
5321                         ret = -EINVAL;
5322                         goto out;
5323                 }
5324
5325                 prev_svm = to_svm(prev_vcpu);
5326                 svm_ir_list_del(prev_svm, pi);
5327         }
5328
5329         /**
5330          * Allocating new amd_iommu_pi_data, which will get
5331          * add to the per-vcpu ir_list.
5332          */
5333         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
5334         if (!ir) {
5335                 ret = -ENOMEM;
5336                 goto out;
5337         }
5338         ir->data = pi->ir_data;
5339
5340         spin_lock_irqsave(&svm->ir_list_lock, flags);
5341         list_add(&ir->node, &svm->ir_list);
5342         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5343 out:
5344         return ret;
5345 }
5346
5347 /**
5348  * Note:
5349  * The HW cannot support posting multicast/broadcast
5350  * interrupts to a vCPU. So, we still use legacy interrupt
5351  * remapping for these kind of interrupts.
5352  *
5353  * For lowest-priority interrupts, we only support
5354  * those with single CPU as the destination, e.g. user
5355  * configures the interrupts via /proc/irq or uses
5356  * irqbalance to make the interrupts single-CPU.
5357  */
5358 static int
5359 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
5360                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
5361 {
5362         struct kvm_lapic_irq irq;
5363         struct kvm_vcpu *vcpu = NULL;
5364
5365         kvm_set_msi_irq(kvm, e, &irq);
5366
5367         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
5368             !kvm_irq_is_postable(&irq)) {
5369                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5370                          __func__, irq.vector);
5371                 return -1;
5372         }
5373
5374         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
5375                  irq.vector);
5376         *svm = to_svm(vcpu);
5377         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
5378         vcpu_info->vector = irq.vector;
5379
5380         return 0;
5381 }
5382
5383 /*
5384  * svm_update_pi_irte - set IRTE for Posted-Interrupts
5385  *
5386  * @kvm: kvm
5387  * @host_irq: host irq of the interrupt
5388  * @guest_irq: gsi of the interrupt
5389  * @set: set or unset PI
5390  * returns 0 on success, < 0 on failure
5391  */
5392 static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
5393                               uint32_t guest_irq, bool set)
5394 {
5395         struct kvm_kernel_irq_routing_entry *e;
5396         struct kvm_irq_routing_table *irq_rt;
5397         int idx, ret = -EINVAL;
5398
5399         if (!kvm_arch_has_assigned_device(kvm) ||
5400             !irq_remapping_cap(IRQ_POSTING_CAP))
5401                 return 0;
5402
5403         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
5404                  __func__, host_irq, guest_irq, set);
5405
5406         idx = srcu_read_lock(&kvm->irq_srcu);
5407         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
5408         WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
5409
5410         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
5411                 struct vcpu_data vcpu_info;
5412                 struct vcpu_svm *svm = NULL;
5413
5414                 if (e->type != KVM_IRQ_ROUTING_MSI)
5415                         continue;
5416
5417                 /**
5418                  * Here, we setup with legacy mode in the following cases:
5419                  * 1. When cannot target interrupt to a specific vcpu.
5420                  * 2. Unsetting posted interrupt.
5421                  * 3. APIC virtialization is disabled for the vcpu.
5422                  * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
5423                  */
5424                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5425                     kvm_vcpu_apicv_active(&svm->vcpu)) {
5426                         struct amd_iommu_pi_data pi;
5427
5428                         /* Try to enable guest_mode in IRTE */
5429                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
5430                                             AVIC_HPA_MASK);
5431                         pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
5432                                                      svm->vcpu.vcpu_id);
5433                         pi.is_guest_mode = true;
5434                         pi.vcpu_data = &vcpu_info;
5435                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5436
5437                         /**
5438                          * Here, we successfully setting up vcpu affinity in
5439                          * IOMMU guest mode. Now, we need to store the posted
5440                          * interrupt information in a per-vcpu ir_list so that
5441                          * we can reference to them directly when we update vcpu
5442                          * scheduling information in IOMMU irte.
5443                          */
5444                         if (!ret && pi.is_guest_mode)
5445                                 svm_ir_list_add(svm, &pi);
5446                 } else {
5447                         /* Use legacy mode in IRTE */
5448                         struct amd_iommu_pi_data pi;
5449
5450                         /**
5451                          * Here, pi is used to:
5452                          * - Tell IOMMU to use legacy mode for this interrupt.
5453                          * - Retrieve ga_tag of prior interrupt remapping data.
5454                          */
5455                         pi.is_guest_mode = false;
5456                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5457
5458                         /**
5459                          * Check if the posted interrupt was previously
5460                          * setup with the guest_mode by checking if the ga_tag
5461                          * was cached. If so, we need to clean up the per-vcpu
5462                          * ir_list.
5463                          */
5464                         if (!ret && pi.prev_ga_tag) {
5465                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
5466                                 struct kvm_vcpu *vcpu;
5467
5468                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
5469                                 if (vcpu)
5470                                         svm_ir_list_del(to_svm(vcpu), &pi);
5471                         }
5472                 }
5473
5474                 if (!ret && svm) {
5475                         trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
5476                                                  e->gsi, vcpu_info.vector,
5477                                                  vcpu_info.pi_desc_addr, set);
5478                 }
5479
5480                 if (ret < 0) {
5481                         pr_err("%s: failed to update PI IRTE\n", __func__);
5482                         goto out;
5483                 }
5484         }
5485
5486         ret = 0;
5487 out:
5488         srcu_read_unlock(&kvm->irq_srcu, idx);
5489         return ret;
5490 }
5491
5492 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
5493 {
5494         struct vcpu_svm *svm = to_svm(vcpu);
5495         struct vmcb *vmcb = svm->vmcb;
5496         int ret;
5497         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
5498               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
5499         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
5500
5501         return ret;
5502 }
5503
5504 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
5505 {
5506         struct vcpu_svm *svm = to_svm(vcpu);
5507
5508         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
5509 }
5510
5511 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5512 {
5513         struct vcpu_svm *svm = to_svm(vcpu);
5514
5515         if (masked) {
5516                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
5517                 set_intercept(svm, INTERCEPT_IRET);
5518         } else {
5519                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
5520                 clr_intercept(svm, INTERCEPT_IRET);
5521         }
5522 }
5523
5524 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
5525 {
5526         struct vcpu_svm *svm = to_svm(vcpu);
5527         struct vmcb *vmcb = svm->vmcb;
5528         int ret;
5529
5530         if (!gif_set(svm) ||
5531              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
5532                 return 0;
5533
5534         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
5535
5536         if (is_guest_mode(vcpu))
5537                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
5538
5539         return ret;
5540 }
5541
5542 static void enable_irq_window(struct kvm_vcpu *vcpu)
5543 {
5544         struct vcpu_svm *svm = to_svm(vcpu);
5545
5546         /*
5547          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
5548          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
5549          * get that intercept, this function will be called again though and
5550          * we'll get the vintr intercept. However, if the vGIF feature is
5551          * enabled, the STGI interception will not occur. Enable the irq
5552          * window under the assumption that the hardware will set the GIF.
5553          */
5554         if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
5555                 /*
5556                  * IRQ window is not needed when AVIC is enabled,
5557                  * unless we have pending ExtINT since it cannot be injected
5558                  * via AVIC. In such case, we need to temporarily disable AVIC,
5559                  * and fallback to injecting IRQ via V_IRQ.
5560                  */
5561                 svm_toggle_avic_for_irq_window(vcpu, false);
5562                 svm_set_vintr(svm);
5563                 svm_inject_irq(svm, 0x0);
5564         }
5565 }
5566
5567 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5568 {
5569         struct vcpu_svm *svm = to_svm(vcpu);
5570
5571         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
5572             == HF_NMI_MASK)
5573                 return; /* IRET will cause a vm exit */
5574
5575         if (!gif_set(svm)) {
5576                 if (vgif_enabled(svm))
5577                         set_intercept(svm, INTERCEPT_STGI);
5578                 return; /* STGI will cause a vm exit */
5579         }
5580
5581         if (svm->nested.exit_required)
5582                 return; /* we're not going to run the guest yet */
5583
5584         /*
5585          * Something prevents NMI from been injected. Single step over possible
5586          * problem (IRET or exception injection or interrupt shadow)
5587          */
5588         svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
5589         svm->nmi_singlestep = true;
5590         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
5591 }
5592
5593 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
5594 {
5595         return 0;
5596 }
5597
5598 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
5599 {
5600         return 0;
5601 }
5602
5603 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5604 {
5605         struct vcpu_svm *svm = to_svm(vcpu);
5606
5607         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
5608                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5609         else
5610                 svm->asid_generation--;
5611 }
5612
5613 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
5614 {
5615         struct vcpu_svm *svm = to_svm(vcpu);
5616
5617         invlpga(gva, svm->vmcb->control.asid);
5618 }
5619
5620 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
5621 {
5622 }
5623
5624 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
5625 {
5626         struct vcpu_svm *svm = to_svm(vcpu);
5627
5628         if (svm_nested_virtualize_tpr(vcpu))
5629                 return;
5630
5631         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
5632                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
5633                 kvm_set_cr8(vcpu, cr8);
5634         }
5635 }
5636
5637 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
5638 {
5639         struct vcpu_svm *svm = to_svm(vcpu);
5640         u64 cr8;
5641
5642         if (svm_nested_virtualize_tpr(vcpu) ||
5643             kvm_vcpu_apicv_active(vcpu))
5644                 return;
5645
5646         cr8 = kvm_get_cr8(vcpu);
5647         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
5648         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
5649 }
5650
5651 static void svm_complete_interrupts(struct vcpu_svm *svm)
5652 {
5653         u8 vector;
5654         int type;
5655         u32 exitintinfo = svm->vmcb->control.exit_int_info;
5656         unsigned int3_injected = svm->int3_injected;
5657
5658         svm->int3_injected = 0;
5659
5660         /*
5661          * If we've made progress since setting HF_IRET_MASK, we've
5662          * executed an IRET and can allow NMI injection.
5663          */
5664         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
5665             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
5666                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
5667                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5668         }
5669
5670         svm->vcpu.arch.nmi_injected = false;
5671         kvm_clear_exception_queue(&svm->vcpu);
5672         kvm_clear_interrupt_queue(&svm->vcpu);
5673
5674         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
5675                 return;
5676
5677         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5678
5679         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
5680         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
5681
5682         switch (type) {
5683         case SVM_EXITINTINFO_TYPE_NMI:
5684                 svm->vcpu.arch.nmi_injected = true;
5685                 break;
5686         case SVM_EXITINTINFO_TYPE_EXEPT:
5687                 /*
5688                  * In case of software exceptions, do not reinject the vector,
5689                  * but re-execute the instruction instead. Rewind RIP first
5690                  * if we emulated INT3 before.
5691                  */
5692                 if (kvm_exception_is_soft(vector)) {
5693                         if (vector == BP_VECTOR && int3_injected &&
5694                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
5695                                 kvm_rip_write(&svm->vcpu,
5696                                               kvm_rip_read(&svm->vcpu) -
5697                                               int3_injected);
5698                         break;
5699                 }
5700                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
5701                         u32 err = svm->vmcb->control.exit_int_info_err;
5702                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
5703
5704                 } else
5705                         kvm_requeue_exception(&svm->vcpu, vector);
5706                 break;
5707         case SVM_EXITINTINFO_TYPE_INTR:
5708                 kvm_queue_interrupt(&svm->vcpu, vector, false);
5709                 break;
5710         default:
5711                 break;
5712         }
5713 }
5714
5715 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
5716 {
5717         struct vcpu_svm *svm = to_svm(vcpu);
5718         struct vmcb_control_area *control = &svm->vmcb->control;
5719
5720         control->exit_int_info = control->event_inj;
5721         control->exit_int_info_err = control->event_inj_err;
5722         control->event_inj = 0;
5723         svm_complete_interrupts(svm);
5724 }
5725
5726 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
5727 {
5728         struct vcpu_svm *svm = to_svm(vcpu);
5729
5730         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5731         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5732         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5733
5734         /*
5735          * A vmexit emulation is required before the vcpu can be executed
5736          * again.
5737          */
5738         if (unlikely(svm->nested.exit_required))
5739                 return;
5740
5741         /*
5742          * Disable singlestep if we're injecting an interrupt/exception.
5743          * We don't want our modified rflags to be pushed on the stack where
5744          * we might not be able to easily reset them if we disabled NMI
5745          * singlestep later.
5746          */
5747         if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
5748                 /*
5749                  * Event injection happens before external interrupts cause a
5750                  * vmexit and interrupts are disabled here, so smp_send_reschedule
5751                  * is enough to force an immediate vmexit.
5752                  */
5753                 disable_nmi_singlestep(svm);
5754                 smp_send_reschedule(vcpu->cpu);
5755         }
5756
5757         pre_svm_run(svm);
5758
5759         sync_lapic_to_cr8(vcpu);
5760
5761         svm->vmcb->save.cr2 = vcpu->arch.cr2;
5762
5763         clgi();
5764         kvm_load_guest_xsave_state(vcpu);
5765
5766         if (lapic_in_kernel(vcpu) &&
5767                 vcpu->arch.apic->lapic_timer.timer_advance_ns)
5768                 kvm_wait_lapic_expire(vcpu);
5769
5770         /*
5771          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
5772          * it's non-zero. Since vmentry is serialising on affected CPUs, there
5773          * is no need to worry about the conditional branch over the wrmsr
5774          * being speculatively taken.
5775          */
5776         x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
5777
5778         local_irq_enable();
5779
5780         asm volatile (
5781                 "push %%" _ASM_BP "; \n\t"
5782                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
5783                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
5784                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
5785                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
5786                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
5787                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
5788 #ifdef CONFIG_X86_64
5789                 "mov %c[r8](%[svm]),  %%r8  \n\t"
5790                 "mov %c[r9](%[svm]),  %%r9  \n\t"
5791                 "mov %c[r10](%[svm]), %%r10 \n\t"
5792                 "mov %c[r11](%[svm]), %%r11 \n\t"
5793                 "mov %c[r12](%[svm]), %%r12 \n\t"
5794                 "mov %c[r13](%[svm]), %%r13 \n\t"
5795                 "mov %c[r14](%[svm]), %%r14 \n\t"
5796                 "mov %c[r15](%[svm]), %%r15 \n\t"
5797 #endif
5798
5799                 /* Enter guest mode */
5800                 "push %%" _ASM_AX " \n\t"
5801                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
5802                 __ex("vmload %%" _ASM_AX) "\n\t"
5803                 __ex("vmrun %%" _ASM_AX) "\n\t"
5804                 __ex("vmsave %%" _ASM_AX) "\n\t"
5805                 "pop %%" _ASM_AX " \n\t"
5806
5807                 /* Save guest registers, load host registers */
5808                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
5809                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
5810                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
5811                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
5812                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
5813                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
5814 #ifdef CONFIG_X86_64
5815                 "mov %%r8,  %c[r8](%[svm]) \n\t"
5816                 "mov %%r9,  %c[r9](%[svm]) \n\t"
5817                 "mov %%r10, %c[r10](%[svm]) \n\t"
5818                 "mov %%r11, %c[r11](%[svm]) \n\t"
5819                 "mov %%r12, %c[r12](%[svm]) \n\t"
5820                 "mov %%r13, %c[r13](%[svm]) \n\t"
5821                 "mov %%r14, %c[r14](%[svm]) \n\t"
5822                 "mov %%r15, %c[r15](%[svm]) \n\t"
5823                 /*
5824                 * Clear host registers marked as clobbered to prevent
5825                 * speculative use.
5826                 */
5827                 "xor %%r8d, %%r8d \n\t"
5828                 "xor %%r9d, %%r9d \n\t"
5829                 "xor %%r10d, %%r10d \n\t"
5830                 "xor %%r11d, %%r11d \n\t"
5831                 "xor %%r12d, %%r12d \n\t"
5832                 "xor %%r13d, %%r13d \n\t"
5833                 "xor %%r14d, %%r14d \n\t"
5834                 "xor %%r15d, %%r15d \n\t"
5835 #endif
5836                 "xor %%ebx, %%ebx \n\t"
5837                 "xor %%ecx, %%ecx \n\t"
5838                 "xor %%edx, %%edx \n\t"
5839                 "xor %%esi, %%esi \n\t"
5840                 "xor %%edi, %%edi \n\t"
5841                 "pop %%" _ASM_BP
5842                 :
5843                 : [svm]"a"(svm),
5844                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
5845                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
5846                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
5847                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
5848                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
5849                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
5850                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
5851 #ifdef CONFIG_X86_64
5852                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
5853                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
5854                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
5855                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
5856                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
5857                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
5858                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
5859                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
5860 #endif
5861                 : "cc", "memory"
5862 #ifdef CONFIG_X86_64
5863                 , "rbx", "rcx", "rdx", "rsi", "rdi"
5864                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
5865 #else
5866                 , "ebx", "ecx", "edx", "esi", "edi"
5867 #endif
5868                 );
5869
5870         /* Eliminate branch target predictions from guest mode */
5871         vmexit_fill_RSB();
5872
5873 #ifdef CONFIG_X86_64
5874         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
5875 #else
5876         loadsegment(fs, svm->host.fs);
5877 #ifndef CONFIG_X86_32_LAZY_GS
5878         loadsegment(gs, svm->host.gs);
5879 #endif
5880 #endif
5881
5882         /*
5883          * We do not use IBRS in the kernel. If this vCPU has used the
5884          * SPEC_CTRL MSR it may have left it on; save the value and
5885          * turn it off. This is much more efficient than blindly adding
5886          * it to the atomic save/restore list. Especially as the former
5887          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
5888          *
5889          * For non-nested case:
5890          * If the L01 MSR bitmap does not intercept the MSR, then we need to
5891          * save it.
5892          *
5893          * For nested case:
5894          * If the L02 MSR bitmap does not intercept the MSR, then we need to
5895          * save it.
5896          */
5897         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
5898                 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
5899
5900         reload_tss(vcpu);
5901
5902         local_irq_disable();
5903
5904         x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
5905
5906         vcpu->arch.cr2 = svm->vmcb->save.cr2;
5907         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5908         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5909         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5910
5911         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5912                 kvm_before_interrupt(&svm->vcpu);
5913
5914         kvm_load_host_xsave_state(vcpu);
5915         stgi();
5916
5917         /* Any pending NMI will happen here */
5918
5919         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5920                 kvm_after_interrupt(&svm->vcpu);
5921
5922         sync_cr8_to_lapic(vcpu);
5923
5924         svm->next_rip = 0;
5925
5926         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5927
5928         /* if exit due to PF check for async PF */
5929         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5930                 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
5931
5932         if (npt_enabled) {
5933                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5934                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5935         }
5936
5937         /*
5938          * We need to handle MC intercepts here before the vcpu has a chance to
5939          * change the physical cpu
5940          */
5941         if (unlikely(svm->vmcb->control.exit_code ==
5942                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
5943                 svm_handle_mce(svm);
5944
5945         mark_all_clean(svm->vmcb);
5946 }
5947 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
5948
5949 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5950 {
5951         struct vcpu_svm *svm = to_svm(vcpu);
5952
5953         svm->vmcb->save.cr3 = __sme_set(root);
5954         mark_dirty(svm->vmcb, VMCB_CR);
5955 }
5956
5957 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5958 {
5959         struct vcpu_svm *svm = to_svm(vcpu);
5960
5961         svm->vmcb->control.nested_cr3 = __sme_set(root);
5962         mark_dirty(svm->vmcb, VMCB_NPT);
5963
5964         /* Also sync guest cr3 here in case we live migrate */
5965         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5966         mark_dirty(svm->vmcb, VMCB_CR);
5967 }
5968
5969 static int is_disabled(void)
5970 {
5971         u64 vm_cr;
5972
5973         rdmsrl(MSR_VM_CR, vm_cr);
5974         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5975                 return 1;
5976
5977         return 0;
5978 }
5979
5980 static void
5981 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5982 {
5983         /*
5984          * Patch in the VMMCALL instruction:
5985          */
5986         hypercall[0] = 0x0f;
5987         hypercall[1] = 0x01;
5988         hypercall[2] = 0xd9;
5989 }
5990
5991 static int __init svm_check_processor_compat(void)
5992 {
5993         return 0;
5994 }
5995
5996 static bool svm_cpu_has_accelerated_tpr(void)
5997 {
5998         return false;
5999 }
6000
6001 static bool svm_has_emulated_msr(int index)
6002 {
6003         switch (index) {
6004         case MSR_IA32_MCG_EXT_CTL:
6005         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
6006                 return false;
6007         default:
6008                 break;
6009         }
6010
6011         return true;
6012 }
6013
6014 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
6015 {
6016         return 0;
6017 }
6018
6019 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
6020 {
6021         struct vcpu_svm *svm = to_svm(vcpu);
6022
6023         vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
6024                                     boot_cpu_has(X86_FEATURE_XSAVE) &&
6025                                     boot_cpu_has(X86_FEATURE_XSAVES);
6026
6027         /* Update nrips enabled cache */
6028         svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
6029
6030         if (!kvm_vcpu_apicv_active(vcpu))
6031                 return;
6032
6033         guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
6034
6035         /*
6036          * Currently, AVIC does not work with nested virtualization.
6037          * So, we disable AVIC when cpuid for SVM is set in the L1 guest.
6038          */
6039         if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM))
6040                 kvm_request_apicv_update(vcpu->kvm, false,
6041                                          APICV_INHIBIT_REASON_NESTED);
6042 }
6043
6044 #define F feature_bit
6045
6046 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
6047 {
6048         switch (func) {
6049         case 0x1:
6050                 if (avic)
6051                         entry->ecx &= ~F(X2APIC);
6052                 break;
6053         case 0x80000001:
6054                 if (nested)
6055                         entry->ecx |= (1 << 2); /* Set SVM bit */
6056                 break;
6057         case 0x80000008:
6058                 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
6059                      boot_cpu_has(X86_FEATURE_AMD_SSBD))
6060                         entry->ebx |= F(VIRT_SSBD);
6061                 break;
6062         case 0x8000000A:
6063                 entry->eax = 1; /* SVM revision 1 */
6064                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
6065                                    ASID emulation to nested SVM */
6066                 entry->ecx = 0; /* Reserved */
6067                 entry->edx = 0; /* Per default do not support any
6068                                    additional features */
6069
6070                 /* Support next_rip if host supports it */
6071                 if (boot_cpu_has(X86_FEATURE_NRIPS))
6072                         entry->edx |= F(NRIPS);
6073
6074                 /* Support NPT for the guest if enabled */
6075                 if (npt_enabled)
6076                         entry->edx |= F(NPT);
6077
6078         }
6079 }
6080
6081 static int svm_get_lpage_level(void)
6082 {
6083         return PT_PDPE_LEVEL;
6084 }
6085
6086 static bool svm_rdtscp_supported(void)
6087 {
6088         return boot_cpu_has(X86_FEATURE_RDTSCP);
6089 }
6090
6091 static bool svm_invpcid_supported(void)
6092 {
6093         return false;
6094 }
6095
6096 static bool svm_mpx_supported(void)
6097 {
6098         return false;
6099 }
6100
6101 static bool svm_xsaves_supported(void)
6102 {
6103         return boot_cpu_has(X86_FEATURE_XSAVES);
6104 }
6105
6106 static bool svm_umip_emulated(void)
6107 {
6108         return false;
6109 }
6110
6111 static bool svm_pt_supported(void)
6112 {
6113         return false;
6114 }
6115
6116 static bool svm_has_wbinvd_exit(void)
6117 {
6118         return true;
6119 }
6120
6121 static bool svm_pku_supported(void)
6122 {
6123         return false;
6124 }
6125
6126 #define PRE_EX(exit)  { .exit_code = (exit), \
6127                         .stage = X86_ICPT_PRE_EXCEPT, }
6128 #define POST_EX(exit) { .exit_code = (exit), \
6129                         .stage = X86_ICPT_POST_EXCEPT, }
6130 #define POST_MEM(exit) { .exit_code = (exit), \
6131                         .stage = X86_ICPT_POST_MEMACCESS, }
6132
6133 static const struct __x86_intercept {
6134         u32 exit_code;
6135         enum x86_intercept_stage stage;
6136 } x86_intercept_map[] = {
6137         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
6138         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
6139         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
6140         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
6141         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
6142         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
6143         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
6144         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
6145         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
6146         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
6147         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
6148         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
6149         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
6150         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
6151         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
6152         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
6153         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
6154         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
6155         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
6156         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
6157         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
6158         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
6159         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
6160         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
6161         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
6162         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
6163         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
6164         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
6165         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
6166         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
6167         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
6168         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
6169         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
6170         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
6171         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
6172         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
6173         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
6174         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
6175         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
6176         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
6177         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
6178         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
6179         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
6180         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
6181         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
6182         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
6183         [x86_intercept_xsetbv]          = PRE_EX(SVM_EXIT_XSETBV),
6184 };
6185
6186 #undef PRE_EX
6187 #undef POST_EX
6188 #undef POST_MEM
6189
6190 static int svm_check_intercept(struct kvm_vcpu *vcpu,
6191                                struct x86_instruction_info *info,
6192                                enum x86_intercept_stage stage)
6193 {
6194         struct vcpu_svm *svm = to_svm(vcpu);
6195         int vmexit, ret = X86EMUL_CONTINUE;
6196         struct __x86_intercept icpt_info;
6197         struct vmcb *vmcb = svm->vmcb;
6198
6199         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
6200                 goto out;
6201
6202         icpt_info = x86_intercept_map[info->intercept];
6203
6204         if (stage != icpt_info.stage)
6205                 goto out;
6206
6207         switch (icpt_info.exit_code) {
6208         case SVM_EXIT_READ_CR0:
6209                 if (info->intercept == x86_intercept_cr_read)
6210                         icpt_info.exit_code += info->modrm_reg;
6211                 break;
6212         case SVM_EXIT_WRITE_CR0: {
6213                 unsigned long cr0, val;
6214                 u64 intercept;
6215
6216                 if (info->intercept == x86_intercept_cr_write)
6217                         icpt_info.exit_code += info->modrm_reg;
6218
6219                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
6220                     info->intercept == x86_intercept_clts)
6221                         break;
6222
6223                 intercept = svm->nested.intercept;
6224
6225                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
6226                         break;
6227
6228                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
6229                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
6230
6231                 if (info->intercept == x86_intercept_lmsw) {
6232                         cr0 &= 0xfUL;
6233                         val &= 0xfUL;
6234                         /* lmsw can't clear PE - catch this here */
6235                         if (cr0 & X86_CR0_PE)
6236                                 val |= X86_CR0_PE;
6237                 }
6238
6239                 if (cr0 ^ val)
6240                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
6241
6242                 break;
6243         }
6244         case SVM_EXIT_READ_DR0:
6245         case SVM_EXIT_WRITE_DR0:
6246                 icpt_info.exit_code += info->modrm_reg;
6247                 break;
6248         case SVM_EXIT_MSR:
6249                 if (info->intercept == x86_intercept_wrmsr)
6250                         vmcb->control.exit_info_1 = 1;
6251                 else
6252                         vmcb->control.exit_info_1 = 0;
6253                 break;
6254         case SVM_EXIT_PAUSE:
6255                 /*
6256                  * We get this for NOP only, but pause
6257                  * is rep not, check this here
6258                  */
6259                 if (info->rep_prefix != REPE_PREFIX)
6260                         goto out;
6261                 break;
6262         case SVM_EXIT_IOIO: {
6263                 u64 exit_info;
6264                 u32 bytes;
6265
6266                 if (info->intercept == x86_intercept_in ||
6267                     info->intercept == x86_intercept_ins) {
6268                         exit_info = ((info->src_val & 0xffff) << 16) |
6269                                 SVM_IOIO_TYPE_MASK;
6270                         bytes = info->dst_bytes;
6271                 } else {
6272                         exit_info = (info->dst_val & 0xffff) << 16;
6273                         bytes = info->src_bytes;
6274                 }
6275
6276                 if (info->intercept == x86_intercept_outs ||
6277                     info->intercept == x86_intercept_ins)
6278                         exit_info |= SVM_IOIO_STR_MASK;
6279
6280                 if (info->rep_prefix)
6281                         exit_info |= SVM_IOIO_REP_MASK;
6282
6283                 bytes = min(bytes, 4u);
6284
6285                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
6286
6287                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
6288
6289                 vmcb->control.exit_info_1 = exit_info;
6290                 vmcb->control.exit_info_2 = info->next_rip;
6291
6292                 break;
6293         }
6294         default:
6295                 break;
6296         }
6297
6298         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
6299         if (static_cpu_has(X86_FEATURE_NRIPS))
6300                 vmcb->control.next_rip  = info->next_rip;
6301         vmcb->control.exit_code = icpt_info.exit_code;
6302         vmexit = nested_svm_exit_handled(svm);
6303
6304         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
6305                                            : X86EMUL_CONTINUE;
6306
6307 out:
6308         return ret;
6309 }
6310
6311 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu,
6312         enum exit_fastpath_completion *exit_fastpath)
6313 {
6314         if (!is_guest_mode(vcpu) &&
6315                 to_svm(vcpu)->vmcb->control.exit_code == EXIT_REASON_MSR_WRITE)
6316                 *exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
6317 }
6318
6319 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
6320 {
6321         if (pause_filter_thresh)
6322                 shrink_ple_window(vcpu);
6323 }
6324
6325 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
6326 {
6327         if (avic_handle_apic_id_update(vcpu) != 0)
6328                 return;
6329         avic_handle_dfr_update(vcpu);
6330         avic_handle_ldr_update(vcpu);
6331 }
6332
6333 static void svm_setup_mce(struct kvm_vcpu *vcpu)
6334 {
6335         /* [63:9] are reserved. */
6336         vcpu->arch.mcg_cap &= 0x1ff;
6337 }
6338
6339 static int svm_smi_allowed(struct kvm_vcpu *vcpu)
6340 {
6341         struct vcpu_svm *svm = to_svm(vcpu);
6342
6343         /* Per APM Vol.2 15.22.2 "Response to SMI" */
6344         if (!gif_set(svm))
6345                 return 0;
6346
6347         if (is_guest_mode(&svm->vcpu) &&
6348             svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
6349                 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
6350                 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
6351                 svm->nested.exit_required = true;
6352                 return 0;
6353         }
6354
6355         return 1;
6356 }
6357
6358 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
6359 {
6360         struct vcpu_svm *svm = to_svm(vcpu);
6361         int ret;
6362
6363         if (is_guest_mode(vcpu)) {
6364                 /* FED8h - SVM Guest */
6365                 put_smstate(u64, smstate, 0x7ed8, 1);
6366                 /* FEE0h - SVM Guest VMCB Physical Address */
6367                 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
6368
6369                 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
6370                 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
6371                 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
6372
6373                 ret = nested_svm_vmexit(svm);
6374                 if (ret)
6375                         return ret;
6376         }
6377         return 0;
6378 }
6379
6380 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
6381 {
6382         struct vcpu_svm *svm = to_svm(vcpu);
6383         struct vmcb *nested_vmcb;
6384         struct kvm_host_map map;
6385         u64 guest;
6386         u64 vmcb;
6387
6388         guest = GET_SMSTATE(u64, smstate, 0x7ed8);
6389         vmcb = GET_SMSTATE(u64, smstate, 0x7ee0);
6390
6391         if (guest) {
6392                 if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL)
6393                         return 1;
6394                 nested_vmcb = map.hva;
6395                 enter_svm_guest_mode(svm, vmcb, nested_vmcb, &map);
6396         }
6397         return 0;
6398 }
6399
6400 static int enable_smi_window(struct kvm_vcpu *vcpu)
6401 {
6402         struct vcpu_svm *svm = to_svm(vcpu);
6403
6404         if (!gif_set(svm)) {
6405                 if (vgif_enabled(svm))
6406                         set_intercept(svm, INTERCEPT_STGI);
6407                 /* STGI will cause a vm exit */
6408                 return 1;
6409         }
6410         return 0;
6411 }
6412
6413 static int sev_flush_asids(void)
6414 {
6415         int ret, error;
6416
6417         /*
6418          * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail,
6419          * so it must be guarded.
6420          */
6421         down_write(&sev_deactivate_lock);
6422
6423         wbinvd_on_all_cpus();
6424         ret = sev_guest_df_flush(&error);
6425
6426         up_write(&sev_deactivate_lock);
6427
6428         if (ret)
6429                 pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error);
6430
6431         return ret;
6432 }
6433
6434 /* Must be called with the sev_bitmap_lock held */
6435 static bool __sev_recycle_asids(void)
6436 {
6437         int pos;
6438
6439         /* Check if there are any ASIDs to reclaim before performing a flush */
6440         pos = find_next_bit(sev_reclaim_asid_bitmap,
6441                             max_sev_asid, min_sev_asid - 1);
6442         if (pos >= max_sev_asid)
6443                 return false;
6444
6445         if (sev_flush_asids())
6446                 return false;
6447
6448         bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
6449                    max_sev_asid);
6450         bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid);
6451
6452         return true;
6453 }
6454
6455 static int sev_asid_new(void)
6456 {
6457         bool retry = true;
6458         int pos;
6459
6460         mutex_lock(&sev_bitmap_lock);
6461
6462         /*
6463          * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
6464          */
6465 again:
6466         pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
6467         if (pos >= max_sev_asid) {
6468                 if (retry && __sev_recycle_asids()) {
6469                         retry = false;
6470                         goto again;
6471                 }
6472                 mutex_unlock(&sev_bitmap_lock);
6473                 return -EBUSY;
6474         }
6475
6476         __set_bit(pos, sev_asid_bitmap);
6477
6478         mutex_unlock(&sev_bitmap_lock);
6479
6480         return pos + 1;
6481 }
6482
6483 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
6484 {
6485         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6486         int asid, ret;
6487
6488         ret = -EBUSY;
6489         if (unlikely(sev->active))
6490                 return ret;
6491
6492         asid = sev_asid_new();
6493         if (asid < 0)
6494                 return ret;
6495
6496         ret = sev_platform_init(&argp->error);
6497         if (ret)
6498                 goto e_free;
6499
6500         sev->active = true;
6501         sev->asid = asid;
6502         INIT_LIST_HEAD(&sev->regions_list);
6503
6504         return 0;
6505
6506 e_free:
6507         sev_asid_free(asid);
6508         return ret;
6509 }
6510
6511 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
6512 {
6513         struct sev_data_activate *data;
6514         int asid = sev_get_asid(kvm);
6515         int ret;
6516
6517         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6518         if (!data)
6519                 return -ENOMEM;
6520
6521         /* activate ASID on the given handle */
6522         data->handle = handle;
6523         data->asid   = asid;
6524         ret = sev_guest_activate(data, error);
6525         kfree(data);
6526
6527         return ret;
6528 }
6529
6530 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
6531 {
6532         struct fd f;
6533         int ret;
6534
6535         f = fdget(fd);
6536         if (!f.file)
6537                 return -EBADF;
6538
6539         ret = sev_issue_cmd_external_user(f.file, id, data, error);
6540
6541         fdput(f);
6542         return ret;
6543 }
6544
6545 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
6546 {
6547         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6548
6549         return __sev_issue_cmd(sev->fd, id, data, error);
6550 }
6551
6552 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
6553 {
6554         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6555         struct sev_data_launch_start *start;
6556         struct kvm_sev_launch_start params;
6557         void *dh_blob, *session_blob;
6558         int *error = &argp->error;
6559         int ret;
6560
6561         if (!sev_guest(kvm))
6562                 return -ENOTTY;
6563
6564         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6565                 return -EFAULT;
6566
6567         start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT);
6568         if (!start)
6569                 return -ENOMEM;
6570
6571         dh_blob = NULL;
6572         if (params.dh_uaddr) {
6573                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
6574                 if (IS_ERR(dh_blob)) {
6575                         ret = PTR_ERR(dh_blob);
6576                         goto e_free;
6577                 }
6578
6579                 start->dh_cert_address = __sme_set(__pa(dh_blob));
6580                 start->dh_cert_len = params.dh_len;
6581         }
6582
6583         session_blob = NULL;
6584         if (params.session_uaddr) {
6585                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
6586                 if (IS_ERR(session_blob)) {
6587                         ret = PTR_ERR(session_blob);
6588                         goto e_free_dh;
6589                 }
6590
6591                 start->session_address = __sme_set(__pa(session_blob));
6592                 start->session_len = params.session_len;
6593         }
6594
6595         start->handle = params.handle;
6596         start->policy = params.policy;
6597
6598         /* create memory encryption context */
6599         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
6600         if (ret)
6601                 goto e_free_session;
6602
6603         /* Bind ASID to this guest */
6604         ret = sev_bind_asid(kvm, start->handle, error);
6605         if (ret)
6606                 goto e_free_session;
6607
6608         /* return handle to userspace */
6609         params.handle = start->handle;
6610         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
6611                 sev_unbind_asid(kvm, start->handle);
6612                 ret = -EFAULT;
6613                 goto e_free_session;
6614         }
6615
6616         sev->handle = start->handle;
6617         sev->fd = argp->sev_fd;
6618
6619 e_free_session:
6620         kfree(session_blob);
6621 e_free_dh:
6622         kfree(dh_blob);
6623 e_free:
6624         kfree(start);
6625         return ret;
6626 }
6627
6628 static unsigned long get_num_contig_pages(unsigned long idx,
6629                                 struct page **inpages, unsigned long npages)
6630 {
6631         unsigned long paddr, next_paddr;
6632         unsigned long i = idx + 1, pages = 1;
6633
6634         /* find the number of contiguous pages starting from idx */
6635         paddr = __sme_page_pa(inpages[idx]);
6636         while (i < npages) {
6637                 next_paddr = __sme_page_pa(inpages[i++]);
6638                 if ((paddr + PAGE_SIZE) == next_paddr) {
6639                         pages++;
6640                         paddr = next_paddr;
6641                         continue;
6642                 }
6643                 break;
6644         }
6645
6646         return pages;
6647 }
6648
6649 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
6650 {
6651         unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
6652         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6653         struct kvm_sev_launch_update_data params;
6654         struct sev_data_launch_update_data *data;
6655         struct page **inpages;
6656         int ret;
6657
6658         if (!sev_guest(kvm))
6659                 return -ENOTTY;
6660
6661         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6662                 return -EFAULT;
6663
6664         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6665         if (!data)
6666                 return -ENOMEM;
6667
6668         vaddr = params.uaddr;
6669         size = params.len;
6670         vaddr_end = vaddr + size;
6671
6672         /* Lock the user memory. */
6673         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
6674         if (!inpages) {
6675                 ret = -ENOMEM;
6676                 goto e_free;
6677         }
6678
6679         /*
6680          * The LAUNCH_UPDATE command will perform in-place encryption of the
6681          * memory content (i.e it will write the same memory region with C=1).
6682          * It's possible that the cache may contain the data with C=0, i.e.,
6683          * unencrypted so invalidate it first.
6684          */
6685         sev_clflush_pages(inpages, npages);
6686
6687         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
6688                 int offset, len;
6689
6690                 /*
6691                  * If the user buffer is not page-aligned, calculate the offset
6692                  * within the page.
6693                  */
6694                 offset = vaddr & (PAGE_SIZE - 1);
6695
6696                 /* Calculate the number of pages that can be encrypted in one go. */
6697                 pages = get_num_contig_pages(i, inpages, npages);
6698
6699                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
6700
6701                 data->handle = sev->handle;
6702                 data->len = len;
6703                 data->address = __sme_page_pa(inpages[i]) + offset;
6704                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
6705                 if (ret)
6706                         goto e_unpin;
6707
6708                 size -= len;
6709                 next_vaddr = vaddr + len;
6710         }
6711
6712 e_unpin:
6713         /* content of memory is updated, mark pages dirty */
6714         for (i = 0; i < npages; i++) {
6715                 set_page_dirty_lock(inpages[i]);
6716                 mark_page_accessed(inpages[i]);
6717         }
6718         /* unlock the user pages */
6719         sev_unpin_memory(kvm, inpages, npages);
6720 e_free:
6721         kfree(data);
6722         return ret;
6723 }
6724
6725 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
6726 {
6727         void __user *measure = (void __user *)(uintptr_t)argp->data;
6728         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6729         struct sev_data_launch_measure *data;
6730         struct kvm_sev_launch_measure params;
6731         void __user *p = NULL;
6732         void *blob = NULL;
6733         int ret;
6734
6735         if (!sev_guest(kvm))
6736                 return -ENOTTY;
6737
6738         if (copy_from_user(&params, measure, sizeof(params)))
6739                 return -EFAULT;
6740
6741         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6742         if (!data)
6743                 return -ENOMEM;
6744
6745         /* User wants to query the blob length */
6746         if (!params.len)
6747                 goto cmd;
6748
6749         p = (void __user *)(uintptr_t)params.uaddr;
6750         if (p) {
6751                 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
6752                         ret = -EINVAL;
6753                         goto e_free;
6754                 }
6755
6756                 ret = -ENOMEM;
6757                 blob = kmalloc(params.len, GFP_KERNEL);
6758                 if (!blob)
6759                         goto e_free;
6760
6761                 data->address = __psp_pa(blob);
6762                 data->len = params.len;
6763         }
6764
6765 cmd:
6766         data->handle = sev->handle;
6767         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
6768
6769         /*
6770          * If we query the session length, FW responded with expected data.
6771          */
6772         if (!params.len)
6773                 goto done;
6774
6775         if (ret)
6776                 goto e_free_blob;
6777
6778         if (blob) {
6779                 if (copy_to_user(p, blob, params.len))
6780                         ret = -EFAULT;
6781         }
6782
6783 done:
6784         params.len = data->len;
6785         if (copy_to_user(measure, &params, sizeof(params)))
6786                 ret = -EFAULT;
6787 e_free_blob:
6788         kfree(blob);
6789 e_free:
6790         kfree(data);
6791         return ret;
6792 }
6793
6794 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
6795 {
6796         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6797         struct sev_data_launch_finish *data;
6798         int ret;
6799
6800         if (!sev_guest(kvm))
6801                 return -ENOTTY;
6802
6803         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6804         if (!data)
6805                 return -ENOMEM;
6806
6807         data->handle = sev->handle;
6808         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
6809
6810         kfree(data);
6811         return ret;
6812 }
6813
6814 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
6815 {
6816         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6817         struct kvm_sev_guest_status params;
6818         struct sev_data_guest_status *data;
6819         int ret;
6820
6821         if (!sev_guest(kvm))
6822                 return -ENOTTY;
6823
6824         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6825         if (!data)
6826                 return -ENOMEM;
6827
6828         data->handle = sev->handle;
6829         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
6830         if (ret)
6831                 goto e_free;
6832
6833         params.policy = data->policy;
6834         params.state = data->state;
6835         params.handle = data->handle;
6836
6837         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
6838                 ret = -EFAULT;
6839 e_free:
6840         kfree(data);
6841         return ret;
6842 }
6843
6844 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
6845                                unsigned long dst, int size,
6846                                int *error, bool enc)
6847 {
6848         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6849         struct sev_data_dbg *data;
6850         int ret;
6851
6852         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6853         if (!data)
6854                 return -ENOMEM;
6855
6856         data->handle = sev->handle;
6857         data->dst_addr = dst;
6858         data->src_addr = src;
6859         data->len = size;
6860
6861         ret = sev_issue_cmd(kvm,
6862                             enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
6863                             data, error);
6864         kfree(data);
6865         return ret;
6866 }
6867
6868 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
6869                              unsigned long dst_paddr, int sz, int *err)
6870 {
6871         int offset;
6872
6873         /*
6874          * Its safe to read more than we are asked, caller should ensure that
6875          * destination has enough space.
6876          */
6877         src_paddr = round_down(src_paddr, 16);
6878         offset = src_paddr & 15;
6879         sz = round_up(sz + offset, 16);
6880
6881         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
6882 }
6883
6884 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
6885                                   unsigned long __user dst_uaddr,
6886                                   unsigned long dst_paddr,
6887                                   int size, int *err)
6888 {
6889         struct page *tpage = NULL;
6890         int ret, offset;
6891
6892         /* if inputs are not 16-byte then use intermediate buffer */
6893         if (!IS_ALIGNED(dst_paddr, 16) ||
6894             !IS_ALIGNED(paddr,     16) ||
6895             !IS_ALIGNED(size,      16)) {
6896                 tpage = (void *)alloc_page(GFP_KERNEL);
6897                 if (!tpage)
6898                         return -ENOMEM;
6899
6900                 dst_paddr = __sme_page_pa(tpage);
6901         }
6902
6903         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
6904         if (ret)
6905                 goto e_free;
6906
6907         if (tpage) {
6908                 offset = paddr & 15;
6909                 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
6910                                  page_address(tpage) + offset, size))
6911                         ret = -EFAULT;
6912         }
6913
6914 e_free:
6915         if (tpage)
6916                 __free_page(tpage);
6917
6918         return ret;
6919 }
6920
6921 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
6922                                   unsigned long __user vaddr,
6923                                   unsigned long dst_paddr,
6924                                   unsigned long __user dst_vaddr,
6925                                   int size, int *error)
6926 {
6927         struct page *src_tpage = NULL;
6928         struct page *dst_tpage = NULL;
6929         int ret, len = size;
6930
6931         /* If source buffer is not aligned then use an intermediate buffer */
6932         if (!IS_ALIGNED(vaddr, 16)) {
6933                 src_tpage = alloc_page(GFP_KERNEL);
6934                 if (!src_tpage)
6935                         return -ENOMEM;
6936
6937                 if (copy_from_user(page_address(src_tpage),
6938                                 (void __user *)(uintptr_t)vaddr, size)) {
6939                         __free_page(src_tpage);
6940                         return -EFAULT;
6941                 }
6942
6943                 paddr = __sme_page_pa(src_tpage);
6944         }
6945
6946         /*
6947          *  If destination buffer or length is not aligned then do read-modify-write:
6948          *   - decrypt destination in an intermediate buffer
6949          *   - copy the source buffer in an intermediate buffer
6950          *   - use the intermediate buffer as source buffer
6951          */
6952         if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
6953                 int dst_offset;
6954
6955                 dst_tpage = alloc_page(GFP_KERNEL);
6956                 if (!dst_tpage) {
6957                         ret = -ENOMEM;
6958                         goto e_free;
6959                 }
6960
6961                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
6962                                         __sme_page_pa(dst_tpage), size, error);
6963                 if (ret)
6964                         goto e_free;
6965
6966                 /*
6967                  *  If source is kernel buffer then use memcpy() otherwise
6968                  *  copy_from_user().
6969                  */
6970                 dst_offset = dst_paddr & 15;
6971
6972                 if (src_tpage)
6973                         memcpy(page_address(dst_tpage) + dst_offset,
6974                                page_address(src_tpage), size);
6975                 else {
6976                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
6977                                            (void __user *)(uintptr_t)vaddr, size)) {
6978                                 ret = -EFAULT;
6979                                 goto e_free;
6980                         }
6981                 }
6982
6983                 paddr = __sme_page_pa(dst_tpage);
6984                 dst_paddr = round_down(dst_paddr, 16);
6985                 len = round_up(size, 16);
6986         }
6987
6988         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
6989
6990 e_free:
6991         if (src_tpage)
6992                 __free_page(src_tpage);
6993         if (dst_tpage)
6994                 __free_page(dst_tpage);
6995         return ret;
6996 }
6997
6998 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6999 {
7000         unsigned long vaddr, vaddr_end, next_vaddr;
7001         unsigned long dst_vaddr;
7002         struct page **src_p, **dst_p;
7003         struct kvm_sev_dbg debug;
7004         unsigned long n;
7005         unsigned int size;
7006         int ret;
7007
7008         if (!sev_guest(kvm))
7009                 return -ENOTTY;
7010
7011         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
7012                 return -EFAULT;
7013
7014         if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
7015                 return -EINVAL;
7016         if (!debug.dst_uaddr)
7017                 return -EINVAL;
7018
7019         vaddr = debug.src_uaddr;
7020         size = debug.len;
7021         vaddr_end = vaddr + size;
7022         dst_vaddr = debug.dst_uaddr;
7023
7024         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
7025                 int len, s_off, d_off;
7026
7027                 /* lock userspace source and destination page */
7028                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
7029                 if (!src_p)
7030                         return -EFAULT;
7031
7032                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
7033                 if (!dst_p) {
7034                         sev_unpin_memory(kvm, src_p, n);
7035                         return -EFAULT;
7036                 }
7037
7038                 /*
7039                  * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
7040                  * memory content (i.e it will write the same memory region with C=1).
7041                  * It's possible that the cache may contain the data with C=0, i.e.,
7042                  * unencrypted so invalidate it first.
7043                  */
7044                 sev_clflush_pages(src_p, 1);
7045                 sev_clflush_pages(dst_p, 1);
7046
7047                 /*
7048                  * Since user buffer may not be page aligned, calculate the
7049                  * offset within the page.
7050                  */
7051                 s_off = vaddr & ~PAGE_MASK;
7052                 d_off = dst_vaddr & ~PAGE_MASK;
7053                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
7054
7055                 if (dec)
7056                         ret = __sev_dbg_decrypt_user(kvm,
7057                                                      __sme_page_pa(src_p[0]) + s_off,
7058                                                      dst_vaddr,
7059                                                      __sme_page_pa(dst_p[0]) + d_off,
7060                                                      len, &argp->error);
7061                 else
7062                         ret = __sev_dbg_encrypt_user(kvm,
7063                                                      __sme_page_pa(src_p[0]) + s_off,
7064                                                      vaddr,
7065                                                      __sme_page_pa(dst_p[0]) + d_off,
7066                                                      dst_vaddr,
7067                                                      len, &argp->error);
7068
7069                 sev_unpin_memory(kvm, src_p, n);
7070                 sev_unpin_memory(kvm, dst_p, n);
7071
7072                 if (ret)
7073                         goto err;
7074
7075                 next_vaddr = vaddr + len;
7076                 dst_vaddr = dst_vaddr + len;
7077                 size -= len;
7078         }
7079 err:
7080         return ret;
7081 }
7082
7083 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
7084 {
7085         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7086         struct sev_data_launch_secret *data;
7087         struct kvm_sev_launch_secret params;
7088         struct page **pages;
7089         void *blob, *hdr;
7090         unsigned long n;
7091         int ret, offset;
7092
7093         if (!sev_guest(kvm))
7094                 return -ENOTTY;
7095
7096         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
7097                 return -EFAULT;
7098
7099         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
7100         if (!pages)
7101                 return -ENOMEM;
7102
7103         /*
7104          * The secret must be copied into contiguous memory region, lets verify
7105          * that userspace memory pages are contiguous before we issue command.
7106          */
7107         if (get_num_contig_pages(0, pages, n) != n) {
7108                 ret = -EINVAL;
7109                 goto e_unpin_memory;
7110         }
7111
7112         ret = -ENOMEM;
7113         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
7114         if (!data)
7115                 goto e_unpin_memory;
7116
7117         offset = params.guest_uaddr & (PAGE_SIZE - 1);
7118         data->guest_address = __sme_page_pa(pages[0]) + offset;
7119         data->guest_len = params.guest_len;
7120
7121         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
7122         if (IS_ERR(blob)) {
7123                 ret = PTR_ERR(blob);
7124                 goto e_free;
7125         }
7126
7127         data->trans_address = __psp_pa(blob);
7128         data->trans_len = params.trans_len;
7129
7130         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
7131         if (IS_ERR(hdr)) {
7132                 ret = PTR_ERR(hdr);
7133                 goto e_free_blob;
7134         }
7135         data->hdr_address = __psp_pa(hdr);
7136         data->hdr_len = params.hdr_len;
7137
7138         data->handle = sev->handle;
7139         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
7140
7141         kfree(hdr);
7142
7143 e_free_blob:
7144         kfree(blob);
7145 e_free:
7146         kfree(data);
7147 e_unpin_memory:
7148         sev_unpin_memory(kvm, pages, n);
7149         return ret;
7150 }
7151
7152 static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
7153 {
7154         struct kvm_sev_cmd sev_cmd;
7155         int r;
7156
7157         if (!svm_sev_enabled())
7158                 return -ENOTTY;
7159
7160         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
7161                 return -EFAULT;
7162
7163         mutex_lock(&kvm->lock);
7164
7165         switch (sev_cmd.id) {
7166         case KVM_SEV_INIT:
7167                 r = sev_guest_init(kvm, &sev_cmd);
7168                 break;
7169         case KVM_SEV_LAUNCH_START:
7170                 r = sev_launch_start(kvm, &sev_cmd);
7171                 break;
7172         case KVM_SEV_LAUNCH_UPDATE_DATA:
7173                 r = sev_launch_update_data(kvm, &sev_cmd);
7174                 break;
7175         case KVM_SEV_LAUNCH_MEASURE:
7176                 r = sev_launch_measure(kvm, &sev_cmd);
7177                 break;
7178         case KVM_SEV_LAUNCH_FINISH:
7179                 r = sev_launch_finish(kvm, &sev_cmd);
7180                 break;
7181         case KVM_SEV_GUEST_STATUS:
7182                 r = sev_guest_status(kvm, &sev_cmd);
7183                 break;
7184         case KVM_SEV_DBG_DECRYPT:
7185                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
7186                 break;
7187         case KVM_SEV_DBG_ENCRYPT:
7188                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
7189                 break;
7190         case KVM_SEV_LAUNCH_SECRET:
7191                 r = sev_launch_secret(kvm, &sev_cmd);
7192                 break;
7193         default:
7194                 r = -EINVAL;
7195                 goto out;
7196         }
7197
7198         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
7199                 r = -EFAULT;
7200
7201 out:
7202         mutex_unlock(&kvm->lock);
7203         return r;
7204 }
7205
7206 static int svm_register_enc_region(struct kvm *kvm,
7207                                    struct kvm_enc_region *range)
7208 {
7209         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7210         struct enc_region *region;
7211         int ret = 0;
7212
7213         if (!sev_guest(kvm))
7214                 return -ENOTTY;
7215
7216         if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
7217                 return -EINVAL;
7218
7219         region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
7220         if (!region)
7221                 return -ENOMEM;
7222
7223         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
7224         if (!region->pages) {
7225                 ret = -ENOMEM;
7226                 goto e_free;
7227         }
7228
7229         /*
7230          * The guest may change the memory encryption attribute from C=0 -> C=1
7231          * or vice versa for this memory range. Lets make sure caches are
7232          * flushed to ensure that guest data gets written into memory with
7233          * correct C-bit.
7234          */
7235         sev_clflush_pages(region->pages, region->npages);
7236
7237         region->uaddr = range->addr;
7238         region->size = range->size;
7239
7240         mutex_lock(&kvm->lock);
7241         list_add_tail(&region->list, &sev->regions_list);
7242         mutex_unlock(&kvm->lock);
7243
7244         return ret;
7245
7246 e_free:
7247         kfree(region);
7248         return ret;
7249 }
7250
7251 static struct enc_region *
7252 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
7253 {
7254         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7255         struct list_head *head = &sev->regions_list;
7256         struct enc_region *i;
7257
7258         list_for_each_entry(i, head, list) {
7259                 if (i->uaddr == range->addr &&
7260                     i->size == range->size)
7261                         return i;
7262         }
7263
7264         return NULL;
7265 }
7266
7267
7268 static int svm_unregister_enc_region(struct kvm *kvm,
7269                                      struct kvm_enc_region *range)
7270 {
7271         struct enc_region *region;
7272         int ret;
7273
7274         mutex_lock(&kvm->lock);
7275
7276         if (!sev_guest(kvm)) {
7277                 ret = -ENOTTY;
7278                 goto failed;
7279         }
7280
7281         region = find_enc_region(kvm, range);
7282         if (!region) {
7283                 ret = -EINVAL;
7284                 goto failed;
7285         }
7286
7287         __unregister_enc_region_locked(kvm, region);
7288
7289         mutex_unlock(&kvm->lock);
7290         return 0;
7291
7292 failed:
7293         mutex_unlock(&kvm->lock);
7294         return ret;
7295 }
7296
7297 static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu)
7298 {
7299         unsigned long cr4 = kvm_read_cr4(vcpu);
7300         bool smep = cr4 & X86_CR4_SMEP;
7301         bool smap = cr4 & X86_CR4_SMAP;
7302         bool is_user = svm_get_cpl(vcpu) == 3;
7303
7304         /*
7305          * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
7306          *
7307          * Errata:
7308          * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
7309          * possible that CPU microcode implementing DecodeAssist will fail
7310          * to read bytes of instruction which caused #NPF. In this case,
7311          * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
7312          * return 0 instead of the correct guest instruction bytes.
7313          *
7314          * This happens because CPU microcode reading instruction bytes
7315          * uses a special opcode which attempts to read data using CPL=0
7316          * priviledges. The microcode reads CS:RIP and if it hits a SMAP
7317          * fault, it gives up and returns no instruction bytes.
7318          *
7319          * Detection:
7320          * We reach here in case CPU supports DecodeAssist, raised #NPF and
7321          * returned 0 in GuestIntrBytes field of the VMCB.
7322          * First, errata can only be triggered in case vCPU CR4.SMAP=1.
7323          * Second, if vCPU CR4.SMEP=1, errata could only be triggered
7324          * in case vCPU CPL==3 (Because otherwise guest would have triggered
7325          * a SMEP fault instead of #NPF).
7326          * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
7327          * As most guests enable SMAP if they have also enabled SMEP, use above
7328          * logic in order to attempt minimize false-positive of detecting errata
7329          * while still preserving all cases semantic correctness.
7330          *
7331          * Workaround:
7332          * To determine what instruction the guest was executing, the hypervisor
7333          * will have to decode the instruction at the instruction pointer.
7334          *
7335          * In non SEV guest, hypervisor will be able to read the guest
7336          * memory to decode the instruction pointer when insn_len is zero
7337          * so we return true to indicate that decoding is possible.
7338          *
7339          * But in the SEV guest, the guest memory is encrypted with the
7340          * guest specific key and hypervisor will not be able to decode the
7341          * instruction pointer so we will not able to workaround it. Lets
7342          * print the error and request to kill the guest.
7343          */
7344         if (smap && (!smep || is_user)) {
7345                 if (!sev_guest(vcpu->kvm))
7346                         return true;
7347
7348                 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
7349                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7350         }
7351
7352         return false;
7353 }
7354
7355 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
7356 {
7357         struct vcpu_svm *svm = to_svm(vcpu);
7358
7359         /*
7360          * TODO: Last condition latch INIT signals on vCPU when
7361          * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
7362          * To properly emulate the INIT intercept, SVM should implement
7363          * kvm_x86_ops->check_nested_events() and call nested_svm_vmexit()
7364          * there if an INIT signal is pending.
7365          */
7366         return !gif_set(svm) ||
7367                    (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT));
7368 }
7369
7370 static bool svm_check_apicv_inhibit_reasons(ulong bit)
7371 {
7372         ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
7373                           BIT(APICV_INHIBIT_REASON_HYPERV) |
7374                           BIT(APICV_INHIBIT_REASON_NESTED) |
7375                           BIT(APICV_INHIBIT_REASON_IRQWIN) |
7376                           BIT(APICV_INHIBIT_REASON_PIT_REINJ);
7377
7378         return supported & BIT(bit);
7379 }
7380
7381 static void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate)
7382 {
7383         avic_update_access_page(kvm, activate);
7384 }
7385
7386 static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7387         .cpu_has_kvm_support = has_svm,
7388         .disabled_by_bios = is_disabled,
7389         .hardware_setup = svm_hardware_setup,
7390         .hardware_unsetup = svm_hardware_teardown,
7391         .check_processor_compatibility = svm_check_processor_compat,
7392         .hardware_enable = svm_hardware_enable,
7393         .hardware_disable = svm_hardware_disable,
7394         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
7395         .has_emulated_msr = svm_has_emulated_msr,
7396
7397         .vcpu_create = svm_create_vcpu,
7398         .vcpu_free = svm_free_vcpu,
7399         .vcpu_reset = svm_vcpu_reset,
7400
7401         .vm_alloc = svm_vm_alloc,
7402         .vm_free = svm_vm_free,
7403         .vm_init = svm_vm_init,
7404         .vm_destroy = svm_vm_destroy,
7405
7406         .prepare_guest_switch = svm_prepare_guest_switch,
7407         .vcpu_load = svm_vcpu_load,
7408         .vcpu_put = svm_vcpu_put,
7409         .vcpu_blocking = svm_vcpu_blocking,
7410         .vcpu_unblocking = svm_vcpu_unblocking,
7411
7412         .update_bp_intercept = update_bp_intercept,
7413         .get_msr_feature = svm_get_msr_feature,
7414         .get_msr = svm_get_msr,
7415         .set_msr = svm_set_msr,
7416         .get_segment_base = svm_get_segment_base,
7417         .get_segment = svm_get_segment,
7418         .set_segment = svm_set_segment,
7419         .get_cpl = svm_get_cpl,
7420         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
7421         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
7422         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
7423         .set_cr0 = svm_set_cr0,
7424         .set_cr3 = svm_set_cr3,
7425         .set_cr4 = svm_set_cr4,
7426         .set_efer = svm_set_efer,
7427         .get_idt = svm_get_idt,
7428         .set_idt = svm_set_idt,
7429         .get_gdt = svm_get_gdt,
7430         .set_gdt = svm_set_gdt,
7431         .get_dr6 = svm_get_dr6,
7432         .set_dr6 = svm_set_dr6,
7433         .set_dr7 = svm_set_dr7,
7434         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
7435         .cache_reg = svm_cache_reg,
7436         .get_rflags = svm_get_rflags,
7437         .set_rflags = svm_set_rflags,
7438
7439         .tlb_flush = svm_flush_tlb,
7440         .tlb_flush_gva = svm_flush_tlb_gva,
7441
7442         .run = svm_vcpu_run,
7443         .handle_exit = handle_exit,
7444         .skip_emulated_instruction = skip_emulated_instruction,
7445         .update_emulated_instruction = NULL,
7446         .set_interrupt_shadow = svm_set_interrupt_shadow,
7447         .get_interrupt_shadow = svm_get_interrupt_shadow,
7448         .patch_hypercall = svm_patch_hypercall,
7449         .set_irq = svm_set_irq,
7450         .set_nmi = svm_inject_nmi,
7451         .queue_exception = svm_queue_exception,
7452         .cancel_injection = svm_cancel_injection,
7453         .interrupt_allowed = svm_interrupt_allowed,
7454         .nmi_allowed = svm_nmi_allowed,
7455         .get_nmi_mask = svm_get_nmi_mask,
7456         .set_nmi_mask = svm_set_nmi_mask,
7457         .enable_nmi_window = enable_nmi_window,
7458         .enable_irq_window = enable_irq_window,
7459         .update_cr8_intercept = update_cr8_intercept,
7460         .set_virtual_apic_mode = svm_set_virtual_apic_mode,
7461         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
7462         .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
7463         .pre_update_apicv_exec_ctrl = svm_pre_update_apicv_exec_ctrl,
7464         .load_eoi_exitmap = svm_load_eoi_exitmap,
7465         .hwapic_irr_update = svm_hwapic_irr_update,
7466         .hwapic_isr_update = svm_hwapic_isr_update,
7467         .sync_pir_to_irr = kvm_lapic_find_highest_irr,
7468         .apicv_post_state_restore = avic_post_state_restore,
7469
7470         .set_tss_addr = svm_set_tss_addr,
7471         .set_identity_map_addr = svm_set_identity_map_addr,
7472         .get_tdp_level = get_npt_level,
7473         .get_mt_mask = svm_get_mt_mask,
7474
7475         .get_exit_info = svm_get_exit_info,
7476
7477         .get_lpage_level = svm_get_lpage_level,
7478
7479         .cpuid_update = svm_cpuid_update,
7480
7481         .rdtscp_supported = svm_rdtscp_supported,
7482         .invpcid_supported = svm_invpcid_supported,
7483         .mpx_supported = svm_mpx_supported,
7484         .xsaves_supported = svm_xsaves_supported,
7485         .umip_emulated = svm_umip_emulated,
7486         .pt_supported = svm_pt_supported,
7487         .pku_supported = svm_pku_supported,
7488
7489         .set_supported_cpuid = svm_set_supported_cpuid,
7490
7491         .has_wbinvd_exit = svm_has_wbinvd_exit,
7492
7493         .read_l1_tsc_offset = svm_read_l1_tsc_offset,
7494         .write_l1_tsc_offset = svm_write_l1_tsc_offset,
7495
7496         .set_tdp_cr3 = set_tdp_cr3,
7497
7498         .check_intercept = svm_check_intercept,
7499         .handle_exit_irqoff = svm_handle_exit_irqoff,
7500
7501         .request_immediate_exit = __kvm_request_immediate_exit,
7502
7503         .sched_in = svm_sched_in,
7504
7505         .pmu_ops = &amd_pmu_ops,
7506         .deliver_posted_interrupt = svm_deliver_avic_intr,
7507         .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
7508         .update_pi_irte = svm_update_pi_irte,
7509         .setup_mce = svm_setup_mce,
7510
7511         .smi_allowed = svm_smi_allowed,
7512         .pre_enter_smm = svm_pre_enter_smm,
7513         .pre_leave_smm = svm_pre_leave_smm,
7514         .enable_smi_window = enable_smi_window,
7515
7516         .mem_enc_op = svm_mem_enc_op,
7517         .mem_enc_reg_region = svm_register_enc_region,
7518         .mem_enc_unreg_region = svm_unregister_enc_region,
7519
7520         .nested_enable_evmcs = NULL,
7521         .nested_get_evmcs_version = NULL,
7522
7523         .need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
7524
7525         .apic_init_signal_blocked = svm_apic_init_signal_blocked,
7526 };
7527
7528 static int __init svm_init(void)
7529 {
7530         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
7531                         __alignof__(struct vcpu_svm), THIS_MODULE);
7532 }
7533
7534 static void __exit svm_exit(void)
7535 {
7536         kvm_exit();
7537 }
7538
7539 module_init(svm_init)
7540 module_exit(svm_exit)