arch/x86/kvm/svm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM support
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *   Avi Kivity   <avi@qumranet.com>
  13  */
  14
  15 #define pr_fmt(fmt) "SVM: " fmt
  16
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34 #include <linux/amd-iommu.h>
  35 #include <linux/hashtable.h>
  36 #include <linux/frame.h>
  37 #include <linux/psp-sev.h>
  38 #include <linux/file.h>
  39 #include <linux/pagemap.h>
  40 #include <linux/swap.h>
  41
  42 #include <asm/apic.h>
  43 #include <asm/perf_event.h>
  44 #include <asm/tlbflush.h>
  45 #include <asm/desc.h>
  46 #include <asm/debugreg.h>
  47 #include <asm/kvm_para.h>
  48 #include <asm/irq_remapping.h>
  49 #include <asm/spec-ctrl.h>
  50
  51 #include <asm/virtext.h>
  52 #include "trace.h"
  53
  54 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  55
  56 MODULE_AUTHOR("Qumranet");
  57 MODULE_LICENSE("GPL");
  58
  59 static const struct x86_cpu_id svm_cpu_id[] = {
  60         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  61         {}
  62 };
  63 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  64
  65 #define IOPM_ALLOC_ORDER 2
  66 #define MSRPM_ALLOC_ORDER 1
  67
  68 #define SEG_TYPE_LDT 2
  69 #define SEG_TYPE_BUSY_TSS16 3
  70
  71 #define SVM_FEATURE_LBRV           (1 <<  1)
  72 #define SVM_FEATURE_SVML           (1 <<  2)
  73 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  74 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  75 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  76 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  77 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  78
  79 #define SVM_AVIC_DOORBELL       0xc001011b
  80
  81 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  82 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  83 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  84
  85 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  86
  87 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  88 #define TSC_RATIO_MIN           0x0000000000000001ULL
  89 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  90
  91 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) | 0xFFF)
  92
  93 /*
  94  * 0xff is broadcast, so the max index allowed for physical APIC ID
  95  * table is 0xfe.  APIC IDs above 0xff are reserved.
  96  */
  97 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
  98
  99 #define AVIC_UNACCEL_ACCESS_WRITE_MASK          1
 100 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK         0xFF0
 101 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK         0xFFFFFFFF
 102
 103 /* AVIC GATAG is encoded using VM and VCPU IDs */
 104 #define AVIC_VCPU_ID_BITS               8
 105 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
 106
 107 #define AVIC_VM_ID_BITS                 24
 108 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
 109 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
 110
 111 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
 112                                                 (y & AVIC_VCPU_ID_MASK))
 113 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
 114 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
 115
 116 static bool erratum_383_found __read_mostly;
 117
 118 static const u32 host_save_user_msrs[] = {
 119 #ifdef CONFIG_X86_64
 120         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
 121         MSR_FS_BASE,
 122 #endif
 123         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 124         MSR_TSC_AUX,
 125 };
 126
 127 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 128
 129 struct kvm_sev_info {
 130         bool active;            /* SEV enabled guest */
 131         unsigned int asid;      /* ASID used for this guest */
 132         unsigned int handle;    /* SEV firmware handle */
 133         int fd;                 /* SEV device fd */
 134         unsigned long pages_locked; /* Number of pages locked */
 135         struct list_head regions_list;  /* List of registered regions */
 136 };
 137
 138 struct kvm_svm {
 139         struct kvm kvm;
 140
 141         /* Struct members for AVIC */
 142         u32 avic_vm_id;
 143         struct page *avic_logical_id_table_page;
 144         struct page *avic_physical_id_table_page;
 145         struct hlist_node hnode;
 146
 147         struct kvm_sev_info sev_info;
 148 };
 149
 150 struct kvm_vcpu;
 151
 152 struct nested_state {
 153         struct vmcb *hsave;
 154         u64 hsave_msr;
 155         u64 vm_cr_msr;
 156         u64 vmcb;
 157
 158         /* These are the merged vectors */
 159         u32 *msrpm;
 160
 161         /* gpa pointers to the real vectors */
 162         u64 vmcb_msrpm;
 163         u64 vmcb_iopm;
 164
 165         /* A VMEXIT is required but not yet emulated */
 166         bool exit_required;
 167
 168         /* cache for intercepts of the guest */
 169         u32 intercept_cr;
 170         u32 intercept_dr;
 171         u32 intercept_exceptions;
 172         u64 intercept;
 173
 174         /* Nested Paging related state */
 175         u64 nested_cr3;
 176 };
 177
 178 #define MSRPM_OFFSETS   16
 179 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 180
 181 /*
 182  * Set osvw_len to higher value when updated Revision Guides
 183  * are published and we know what the new status bits are
 184  */
 185 static uint64_t osvw_len = 4, osvw_status;
 186
 187 struct vcpu_svm {
 188         struct kvm_vcpu vcpu;
 189         struct vmcb *vmcb;
 190         unsigned long vmcb_pa;
 191         struct svm_cpu_data *svm_data;
 192         uint64_t asid_generation;
 193         uint64_t sysenter_esp;
 194         uint64_t sysenter_eip;
 195         uint64_t tsc_aux;
 196
 197         u64 msr_decfg;
 198
 199         u64 next_rip;
 200
 201         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 202         struct {
 203                 u16 fs;
 204                 u16 gs;
 205                 u16 ldt;
 206                 u64 gs_base;
 207         } host;
 208
 209         u64 spec_ctrl;
 210         /*
 211          * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
 212          * translated into the appropriate L2_CFG bits on the host to
 213          * perform speculative control.
 214          */
 215         u64 virt_spec_ctrl;
 216
 217         u32 *msrpm;
 218
 219         ulong nmi_iret_rip;
 220
 221         struct nested_state nested;
 222
 223         bool nmi_singlestep;
 224         u64 nmi_singlestep_guest_rflags;
 225
 226         unsigned int3_injected;
 227         unsigned long int3_rip;
 228
 229         /* cached guest cpuid flags for faster access */
 230         bool nrips_enabled      : 1;
 231
 232         u32 ldr_reg;
 233         u32 dfr_reg;
 234         struct page *avic_backing_page;
 235         u64 *avic_physical_id_cache;
 236         bool avic_is_running;
 237
 238         /*
 239          * Per-vcpu list of struct amd_svm_iommu_ir:
 240          * This is used mainly to store interrupt remapping information used
 241          * when update the vcpu affinity. This avoids the need to scan for
 242          * IRTE and try to match ga_tag in the IOMMU driver.
 243          */
 244         struct list_head ir_list;
 245         spinlock_t ir_list_lock;
 246
 247         /* which host CPU was used for running this vcpu */
 248         unsigned int last_cpu;
 249 };
 250
 251 /*
 252  * This is a wrapper of struct amd_iommu_ir_data.
 253  */
 254 struct amd_svm_iommu_ir {
 255         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
 256         void *data;             /* Storing pointer to struct amd_ir_data */
 257 };
 258
 259 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
 260 #define AVIC_LOGICAL_ID_ENTRY_VALID_BIT                 31
 261 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
 262
 263 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
 264 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
 265 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
 266 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
 267
 268 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 269 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 270
 271 #define MSR_INVALID                     0xffffffffU
 272
 273 static const struct svm_direct_access_msrs {
 274         u32 index;   /* Index of the MSR */
 275         bool always; /* True if intercept is always on */
 276 } direct_access_msrs[] = {
 277         { .index = MSR_STAR,                            .always = true  },
 278         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 279 #ifdef CONFIG_X86_64
 280         { .index = MSR_GS_BASE,                         .always = true  },
 281         { .index = MSR_FS_BASE,                         .always = true  },
 282         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 283         { .index = MSR_LSTAR,                           .always = true  },
 284         { .index = MSR_CSTAR,                           .always = true  },
 285         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 286 #endif
 287         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
 288         { .index = MSR_IA32_PRED_CMD,                   .always = false },
 289         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 290         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 291         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 292         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 293         { .index = MSR_INVALID,                         .always = false },
 294 };
 295
 296 /* enable NPT for AMD64 and X86 with PAE */
 297 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 298 static bool npt_enabled = true;
 299 #else
 300 static bool npt_enabled;
 301 #endif
 302
 303 /*
 304  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 305  * pause_filter_count: On processors that support Pause filtering(indicated
 306  *      by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
 307  *      count value. On VMRUN this value is loaded into an internal counter.
 308  *      Each time a pause instruction is executed, this counter is decremented
 309  *      until it reaches zero at which time a #VMEXIT is generated if pause
 310  *      intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
 311  *      Intercept Filtering for more details.
 312  *      This also indicate if ple logic enabled.
 313  *
 314  * pause_filter_thresh: In addition, some processor families support advanced
 315  *      pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
 316  *      the amount of time a guest is allowed to execute in a pause loop.
 317  *      In this mode, a 16-bit pause filter threshold field is added in the
 318  *      VMCB. The threshold value is a cycle count that is used to reset the
 319  *      pause counter. As with simple pause filtering, VMRUN loads the pause
 320  *      count value from VMCB into an internal counter. Then, on each pause
 321  *      instruction the hardware checks the elapsed number of cycles since
 322  *      the most recent pause instruction against the pause filter threshold.
 323  *      If the elapsed cycle count is greater than the pause filter threshold,
 324  *      then the internal pause count is reloaded from the VMCB and execution
 325  *      continues. If the elapsed cycle count is less than the pause filter
 326  *      threshold, then the internal pause count is decremented. If the count
 327  *      value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 328  *      triggered. If advanced pause filtering is supported and pause filter
 329  *      threshold field is set to zero, the filter will operate in the simpler,
 330  *      count only mode.
 331  */
 332
 333 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 334 module_param(pause_filter_thresh, ushort, 0444);
 335
 336 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 337 module_param(pause_filter_count, ushort, 0444);
 338
 339 /* Default doubles per-vcpu window every exit. */
 340 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 341 module_param(pause_filter_count_grow, ushort, 0444);
 342
 343 /* Default resets per-vcpu window every exit to pause_filter_count. */
 344 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 345 module_param(pause_filter_count_shrink, ushort, 0444);
 346
 347 /* Default is to compute the maximum so we can never overflow. */
 348 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 349 module_param(pause_filter_count_max, ushort, 0444);
 350
 351 /* allow nested paging (virtualized MMU) for all guests */
 352 static int npt = true;
 353 module_param(npt, int, S_IRUGO);
 354
 355 /* allow nested virtualization in KVM/SVM */
 356 static int nested = true;
 357 module_param(nested, int, S_IRUGO);
 358
 359 /* enable / disable AVIC */
 360 static int avic;
 361 #ifdef CONFIG_X86_LOCAL_APIC
 362 module_param(avic, int, S_IRUGO);
 363 #endif
 364
 365 /* enable/disable Next RIP Save */
 366 static int nrips = true;
 367 module_param(nrips, int, 0444);
 368
 369 /* enable/disable Virtual VMLOAD VMSAVE */
 370 static int vls = true;
 371 module_param(vls, int, 0444);
 372
 373 /* enable/disable Virtual GIF */
 374 static int vgif = true;
 375 module_param(vgif, int, 0444);
 376
 377 /* enable/disable SEV support */
 378 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
 379 module_param(sev, int, 0444);
 380
 381 static bool __read_mostly dump_invalid_vmcb = 0;
 382 module_param(dump_invalid_vmcb, bool, 0644);
 383
 384 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 385
 386 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 387 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 388 static void svm_complete_interrupts(struct vcpu_svm *svm);
 389
 390 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 391 static int nested_svm_intercept(struct vcpu_svm *svm);
 392 static int nested_svm_vmexit(struct vcpu_svm *svm);
 393 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 394                                       bool has_error_code, u32 error_code);
 395
 396 enum {
 397         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 398                             pause filter count */
 399         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 400         VMCB_ASID,       /* ASID */
 401         VMCB_INTR,       /* int_ctl, int_vector */
 402         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 403         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 404         VMCB_DR,         /* DR6, DR7 */
 405         VMCB_DT,         /* GDT, IDT */
 406         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 407         VMCB_CR2,        /* CR2 only */
 408         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 409         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
 410                           * AVIC PHYSICAL_TABLE pointer,
 411                           * AVIC LOGICAL_TABLE pointer
 412                           */
 413         VMCB_DIRTY_MAX,
 414 };
 415
 416 /* TPR and CR2 are always written before VMRUN */
 417 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 418
 419 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
 420
 421 static unsigned int max_sev_asid;
 422 static unsigned int min_sev_asid;
 423 static unsigned long *sev_asid_bitmap;
 424 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
 425
 426 struct enc_region {
 427         struct list_head list;
 428         unsigned long npages;
 429         struct page **pages;
 430         unsigned long uaddr;
 431         unsigned long size;
 432 };
 433
 434
 435 static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
 436 {
 437         return container_of(kvm, struct kvm_svm, kvm);
 438 }
 439
 440 static inline bool svm_sev_enabled(void)
 441 {
 442         return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0;
 443 }
 444
 445 static inline bool sev_guest(struct kvm *kvm)
 446 {
 447 #ifdef CONFIG_KVM_AMD_SEV
 448         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 449
 450         return sev->active;
 451 #else
 452         return false;
 453 #endif
 454 }
 455
 456 static inline int sev_get_asid(struct kvm *kvm)
 457 {
 458         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 459
 460         return sev->asid;
 461 }
 462
 463 static inline void mark_all_dirty(struct vmcb *vmcb)
 464 {
 465         vmcb->control.clean = 0;
 466 }
 467
 468 static inline void mark_all_clean(struct vmcb *vmcb)
 469 {
 470         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 471                                & ~VMCB_ALWAYS_DIRTY_MASK;
 472 }
 473
 474 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 475 {
 476         vmcb->control.clean &= ~(1 << bit);
 477 }
 478
 479 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 480 {
 481         return container_of(vcpu, struct vcpu_svm, vcpu);
 482 }
 483
 484 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
 485 {
 486         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
 487         mark_dirty(svm->vmcb, VMCB_AVIC);
 488 }
 489
 490 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
 491 {
 492         struct vcpu_svm *svm = to_svm(vcpu);
 493         u64 *entry = svm->avic_physical_id_cache;
 494
 495         if (!entry)
 496                 return false;
 497
 498         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 499 }
 500
 501 static void recalc_intercepts(struct vcpu_svm *svm)
 502 {
 503         struct vmcb_control_area *c, *h;
 504         struct nested_state *g;
 505
 506         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 507
 508         if (!is_guest_mode(&svm->vcpu))
 509                 return;
 510
 511         c = &svm->vmcb->control;
 512         h = &svm->nested.hsave->control;
 513         g = &svm->nested;
 514
 515         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 516         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 517         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 518         c->intercept = h->intercept | g->intercept;
 519 }
 520
 521 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 522 {
 523         if (is_guest_mode(&svm->vcpu))
 524                 return svm->nested.hsave;
 525         else
 526                 return svm->vmcb;
 527 }
 528
 529 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 530 {
 531         struct vmcb *vmcb = get_host_vmcb(svm);
 532
 533         vmcb->control.intercept_cr |= (1U << bit);
 534
 535         recalc_intercepts(svm);
 536 }
 537
 538 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 539 {
 540         struct vmcb *vmcb = get_host_vmcb(svm);
 541
 542         vmcb->control.intercept_cr &= ~(1U << bit);
 543
 544         recalc_intercepts(svm);
 545 }
 546
 547 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 548 {
 549         struct vmcb *vmcb = get_host_vmcb(svm);
 550
 551         return vmcb->control.intercept_cr & (1U << bit);
 552 }
 553
 554 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 555 {
 556         struct vmcb *vmcb = get_host_vmcb(svm);
 557
 558         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 559                 | (1 << INTERCEPT_DR1_READ)
 560                 | (1 << INTERCEPT_DR2_READ)
 561                 | (1 << INTERCEPT_DR3_READ)
 562                 | (1 << INTERCEPT_DR4_READ)
 563                 | (1 << INTERCEPT_DR5_READ)
 564                 | (1 << INTERCEPT_DR6_READ)
 565                 | (1 << INTERCEPT_DR7_READ)
 566                 | (1 << INTERCEPT_DR0_WRITE)
 567                 | (1 << INTERCEPT_DR1_WRITE)
 568                 | (1 << INTERCEPT_DR2_WRITE)
 569                 | (1 << INTERCEPT_DR3_WRITE)
 570                 | (1 << INTERCEPT_DR4_WRITE)
 571                 | (1 << INTERCEPT_DR5_WRITE)
 572                 | (1 << INTERCEPT_DR6_WRITE)
 573                 | (1 << INTERCEPT_DR7_WRITE);
 574
 575         recalc_intercepts(svm);
 576 }
 577
 578 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 579 {
 580         struct vmcb *vmcb = get_host_vmcb(svm);
 581
 582         vmcb->control.intercept_dr = 0;
 583
 584         recalc_intercepts(svm);
 585 }
 586
 587 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 588 {
 589         struct vmcb *vmcb = get_host_vmcb(svm);
 590
 591         vmcb->control.intercept_exceptions |= (1U << bit);
 592
 593         recalc_intercepts(svm);
 594 }
 595
 596 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 597 {
 598         struct vmcb *vmcb = get_host_vmcb(svm);
 599
 600         vmcb->control.intercept_exceptions &= ~(1U << bit);
 601
 602         recalc_intercepts(svm);
 603 }
 604
 605 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 606 {
 607         struct vmcb *vmcb = get_host_vmcb(svm);
 608
 609         vmcb->control.intercept |= (1ULL << bit);
 610
 611         recalc_intercepts(svm);
 612 }
 613
 614 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 615 {
 616         struct vmcb *vmcb = get_host_vmcb(svm);
 617
 618         vmcb->control.intercept &= ~(1ULL << bit);
 619
 620         recalc_intercepts(svm);
 621 }
 622
 623 static inline bool vgif_enabled(struct vcpu_svm *svm)
 624 {
 625         return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
 626 }
 627
 628 static inline void enable_gif(struct vcpu_svm *svm)
 629 {
 630         if (vgif_enabled(svm))
 631                 svm->vmcb->control.int_ctl |= V_GIF_MASK;
 632         else
 633                 svm->vcpu.arch.hflags |= HF_GIF_MASK;
 634 }
 635
 636 static inline void disable_gif(struct vcpu_svm *svm)
 637 {
 638         if (vgif_enabled(svm))
 639                 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
 640         else
 641                 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 642 }
 643
 644 static inline bool gif_set(struct vcpu_svm *svm)
 645 {
 646         if (vgif_enabled(svm))
 647                 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
 648         else
 649                 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 650 }
 651
 652 static unsigned long iopm_base;
 653
 654 struct kvm_ldttss_desc {
 655         u16 limit0;
 656         u16 base0;
 657         unsigned base1:8, type:5, dpl:2, p:1;
 658         unsigned limit1:4, zero0:3, g:1, base2:8;
 659         u32 base3;
 660         u32 zero1;
 661 } __attribute__((packed));
 662
 663 struct svm_cpu_data {
 664         int cpu;
 665
 666         u64 asid_generation;
 667         u32 max_asid;
 668         u32 next_asid;
 669         u32 min_asid;
 670         struct kvm_ldttss_desc *tss_desc;
 671
 672         struct page *save_area;
 673         struct vmcb *current_vmcb;
 674
 675         /* index = sev_asid, value = vmcb pointer */
 676         struct vmcb **sev_vmcbs;
 677 };
 678
 679 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 680
 681 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 682
 683 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 684 #define MSRS_RANGE_SIZE 2048
 685 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 686
 687 static u32 svm_msrpm_offset(u32 msr)
 688 {
 689         u32 offset;
 690         int i;
 691
 692         for (i = 0; i < NUM_MSR_MAPS; i++) {
 693                 if (msr < msrpm_ranges[i] ||
 694                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 695                         continue;
 696
 697                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 698                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 699
 700                 /* Now we have the u8 offset - but need the u32 offset */
 701                 return offset / 4;
 702         }
 703
 704         /* MSR not in any range */
 705         return MSR_INVALID;
 706 }
 707
 708 #define MAX_INST_SIZE 15
 709
 710 static inline void clgi(void)
 711 {
 712         asm volatile (__ex("clgi"));
 713 }
 714
 715 static inline void stgi(void)
 716 {
 717         asm volatile (__ex("stgi"));
 718 }
 719
 720 static inline void invlpga(unsigned long addr, u32 asid)
 721 {
 722         asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
 723 }
 724
 725 static int get_npt_level(struct kvm_vcpu *vcpu)
 726 {
 727 #ifdef CONFIG_X86_64
 728         return PT64_ROOT_4LEVEL;
 729 #else
 730         return PT32E_ROOT_LEVEL;
 731 #endif
 732 }
 733
 734 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 735 {
 736         vcpu->arch.efer = efer;
 737         if (!npt_enabled && !(efer & EFER_LMA))
 738                 efer &= ~EFER_LME;
 739
 740         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 741         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 742 }
 743
 744 static int is_external_interrupt(u32 info)
 745 {
 746         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 747         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 748 }
 749
 750 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 751 {
 752         struct vcpu_svm *svm = to_svm(vcpu);
 753         u32 ret = 0;
 754
 755         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 756                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 757         return ret;
 758 }
 759
 760 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 761 {
 762         struct vcpu_svm *svm = to_svm(vcpu);
 763
 764         if (mask == 0)
 765                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 766         else
 767                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 768
 769 }
 770
 771 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
 772 {
 773         struct vcpu_svm *svm = to_svm(vcpu);
 774
 775         if (nrips && svm->vmcb->control.next_rip != 0) {
 776                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 777                 svm->next_rip = svm->vmcb->control.next_rip;
 778         }
 779
 780         if (!svm->next_rip)
 781                 return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP);
 782
 783         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 784                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 785                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 786
 787         kvm_rip_write(vcpu, svm->next_rip);
 788         svm_set_interrupt_shadow(vcpu, 0);
 789
 790         return EMULATE_DONE;
 791 }
 792
 793 static void svm_queue_exception(struct kvm_vcpu *vcpu)
 794 {
 795         struct vcpu_svm *svm = to_svm(vcpu);
 796         unsigned nr = vcpu->arch.exception.nr;
 797         bool has_error_code = vcpu->arch.exception.has_error_code;
 798         bool reinject = vcpu->arch.exception.injected;
 799         u32 error_code = vcpu->arch.exception.error_code;
 800
 801         /*
 802          * If we are within a nested VM we'd better #VMEXIT and let the guest
 803          * handle the exception
 804          */
 805         if (!reinject &&
 806             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 807                 return;
 808
 809         kvm_deliver_exception_payload(&svm->vcpu);
 810
 811         if (nr == BP_VECTOR && !nrips) {
 812                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 813
 814                 /*
 815                  * For guest debugging where we have to reinject #BP if some
 816                  * INT3 is guest-owned:
 817                  * Emulate nRIP by moving RIP forward. Will fail if injection
 818                  * raises a fault that is not intercepted. Still better than
 819                  * failing in all cases.
 820                  */
 821                 (void)skip_emulated_instruction(&svm->vcpu);
 822                 rip = kvm_rip_read(&svm->vcpu);
 823                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 824                 svm->int3_injected = rip - old_rip;
 825         }
 826
 827         svm->vmcb->control.event_inj = nr
 828                 | SVM_EVTINJ_VALID
 829                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 830                 | SVM_EVTINJ_TYPE_EXEPT;
 831         svm->vmcb->control.event_inj_err = error_code;
 832 }
 833
 834 static void svm_init_erratum_383(void)
 835 {
 836         u32 low, high;
 837         int err;
 838         u64 val;
 839
 840         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 841                 return;
 842
 843         /* Use _safe variants to not break nested virtualization */
 844         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 845         if (err)
 846                 return;
 847
 848         val |= (1ULL << 47);
 849
 850         low  = lower_32_bits(val);
 851         high = upper_32_bits(val);
 852
 853         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 854
 855         erratum_383_found = true;
 856 }
 857
 858 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 859 {
 860         /*
 861          * Guests should see errata 400 and 415 as fixed (assuming that
 862          * HLT and IO instructions are intercepted).
 863          */
 864         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 865         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 866
 867         /*
 868          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 869          * all osvw.status bits inside that length, including bit 0 (which is
 870          * reserved for erratum 298), are valid. However, if host processor's
 871          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 872          * be conservative here and therefore we tell the guest that erratum 298
 873          * is present (because we really don't know).
 874          */
 875         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 876                 vcpu->arch.osvw.status |= 1;
 877 }
 878
 879 static int has_svm(void)
 880 {
 881         const char *msg;
 882
 883         if (!cpu_has_svm(&msg)) {
 884                 printk(KERN_INFO "has_svm: %s\n", msg);
 885                 return 0;
 886         }
 887
 888         return 1;
 889 }
 890
 891 static void svm_hardware_disable(void)
 892 {
 893         /* Make sure we clean up behind us */
 894         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 895                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 896
 897         cpu_svm_disable();
 898
 899         amd_pmu_disable_virt();
 900 }
 901
 902 static int svm_hardware_enable(void)
 903 {
 904
 905         struct svm_cpu_data *sd;
 906         uint64_t efer;
 907         struct desc_struct *gdt;
 908         int me = raw_smp_processor_id();
 909
 910         rdmsrl(MSR_EFER, efer);
 911         if (efer & EFER_SVME)
 912                 return -EBUSY;
 913
 914         if (!has_svm()) {
 915                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 916                 return -EINVAL;
 917         }
 918         sd = per_cpu(svm_data, me);
 919         if (!sd) {
 920                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 921                 return -EINVAL;
 922         }
 923
 924         sd->asid_generation = 1;
 925         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 926         sd->next_asid = sd->max_asid + 1;
 927         sd->min_asid = max_sev_asid + 1;
 928
 929         gdt = get_current_gdt_rw();
 930         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 931
 932         wrmsrl(MSR_EFER, efer | EFER_SVME);
 933
 934         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 935
 936         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 937                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 938                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 939         }
 940
 941
 942         /*
 943          * Get OSVW bits.
 944          *
 945          * Note that it is possible to have a system with mixed processor
 946          * revisions and therefore different OSVW bits. If bits are not the same
 947          * on different processors then choose the worst case (i.e. if erratum
 948          * is present on one processor and not on another then assume that the
 949          * erratum is present everywhere).
 950          */
 951         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 952                 uint64_t len, status = 0;
 953                 int err;
 954
 955                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 956                 if (!err)
 957                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 958                                                       &err);
 959
 960                 if (err)
 961                         osvw_status = osvw_len = 0;
 962                 else {
 963                         if (len < osvw_len)
 964                                 osvw_len = len;
 965                         osvw_status |= status;
 966                         osvw_status &= (1ULL << osvw_len) - 1;
 967                 }
 968         } else
 969                 osvw_status = osvw_len = 0;
 970
 971         svm_init_erratum_383();
 972
 973         amd_pmu_enable_virt();
 974
 975         return 0;
 976 }
 977
 978 static void svm_cpu_uninit(int cpu)
 979 {
 980         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 981
 982         if (!sd)
 983                 return;
 984
 985         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 986         kfree(sd->sev_vmcbs);
 987         __free_page(sd->save_area);
 988         kfree(sd);
 989 }
 990
 991 static int svm_cpu_init(int cpu)
 992 {
 993         struct svm_cpu_data *sd;
 994         int r;
 995
 996         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 997         if (!sd)
 998                 return -ENOMEM;
 999         sd->cpu = cpu;
1000         r = -ENOMEM;
1001         sd->save_area = alloc_page(GFP_KERNEL);
1002         if (!sd->save_area)
1003                 goto err_1;
1004
1005         if (svm_sev_enabled()) {
1006                 r = -ENOMEM;
1007                 sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1,
1008                                               sizeof(void *),
1009                                               GFP_KERNEL);
1010                 if (!sd->sev_vmcbs)
1011                         goto err_1;
1012         }
1013
1014         per_cpu(svm_data, cpu) = sd;
1015
1016         return 0;
1017
1018 err_1:
1019         kfree(sd);
1020         return r;
1021
1022 }
1023
1024 static bool valid_msr_intercept(u32 index)
1025 {
1026         int i;
1027
1028         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
1029                 if (direct_access_msrs[i].index == index)
1030                         return true;
1031
1032         return false;
1033 }
1034
1035 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
1036 {
1037         u8 bit_write;
1038         unsigned long tmp;
1039         u32 offset;
1040         u32 *msrpm;
1041
1042         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
1043                                       to_svm(vcpu)->msrpm;
1044
1045         offset    = svm_msrpm_offset(msr);
1046         bit_write = 2 * (msr & 0x0f) + 1;
1047         tmp       = msrpm[offset];
1048
1049         BUG_ON(offset == MSR_INVALID);
1050
1051         return !!test_bit(bit_write,  &tmp);
1052 }
1053
1054 static void set_msr_interception(u32 *msrpm, unsigned msr,
1055                                  int read, int write)
1056 {
1057         u8 bit_read, bit_write;
1058         unsigned long tmp;
1059         u32 offset;
1060
1061         /*
1062          * If this warning triggers extend the direct_access_msrs list at the
1063          * beginning of the file
1064          */
1065         WARN_ON(!valid_msr_intercept(msr));
1066
1067         offset    = svm_msrpm_offset(msr);
1068         bit_read  = 2 * (msr & 0x0f);
1069         bit_write = 2 * (msr & 0x0f) + 1;
1070         tmp       = msrpm[offset];
1071
1072         BUG_ON(offset == MSR_INVALID);
1073
1074         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
1075         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
1076
1077         msrpm[offset] = tmp;
1078 }
1079
1080 static void svm_vcpu_init_msrpm(u32 *msrpm)
1081 {
1082         int i;
1083
1084         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
1085
1086         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1087                 if (!direct_access_msrs[i].always)
1088                         continue;
1089
1090                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1091         }
1092 }
1093
1094 static void add_msr_offset(u32 offset)
1095 {
1096         int i;
1097
1098         for (i = 0; i < MSRPM_OFFSETS; ++i) {
1099
1100                 /* Offset already in list? */
1101                 if (msrpm_offsets[i] == offset)
1102                         return;
1103
1104                 /* Slot used by another offset? */
1105                 if (msrpm_offsets[i] != MSR_INVALID)
1106                         continue;
1107
1108                 /* Add offset to list */
1109                 msrpm_offsets[i] = offset;
1110
1111                 return;
1112         }
1113
1114         /*
1115          * If this BUG triggers the msrpm_offsets table has an overflow. Just
1116          * increase MSRPM_OFFSETS in this case.
1117          */
1118         BUG();
1119 }
1120
1121 static void init_msrpm_offsets(void)
1122 {
1123         int i;
1124
1125         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1126
1127         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1128                 u32 offset;
1129
1130                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1131                 BUG_ON(offset == MSR_INVALID);
1132
1133                 add_msr_offset(offset);
1134         }
1135 }
1136
1137 static void svm_enable_lbrv(struct vcpu_svm *svm)
1138 {
1139         u32 *msrpm = svm->msrpm;
1140
1141         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1142         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1143         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1144         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1145         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1146 }
1147
1148 static void svm_disable_lbrv(struct vcpu_svm *svm)
1149 {
1150         u32 *msrpm = svm->msrpm;
1151
1152         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1153         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1154         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1155         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1156         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1157 }
1158
1159 static void disable_nmi_singlestep(struct vcpu_svm *svm)
1160 {
1161         svm->nmi_singlestep = false;
1162
1163         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1164                 /* Clear our flags if they were not set by the guest */
1165                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1166                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1167                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1168                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1169         }
1170 }
1171
1172 /* Note:
1173  * This hash table is used to map VM_ID to a struct kvm_svm,
1174  * when handling AMD IOMMU GALOG notification to schedule in
1175  * a particular vCPU.
1176  */
1177 #define SVM_VM_DATA_HASH_BITS   8
1178 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1179 static u32 next_vm_id = 0;
1180 static bool next_vm_id_wrapped = 0;
1181 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1182
1183 /* Note:
1184  * This function is called from IOMMU driver to notify
1185  * SVM to schedule in a particular vCPU of a particular VM.
1186  */
1187 static int avic_ga_log_notifier(u32 ga_tag)
1188 {
1189         unsigned long flags;
1190         struct kvm_svm *kvm_svm;
1191         struct kvm_vcpu *vcpu = NULL;
1192         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1193         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1194
1195         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1196
1197         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1198         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
1199                 if (kvm_svm->avic_vm_id != vm_id)
1200                         continue;
1201                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
1202                 break;
1203         }
1204         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1205
1206         /* Note:
1207          * At this point, the IOMMU should have already set the pending
1208          * bit in the vAPIC backing page. So, we just need to schedule
1209          * in the vcpu.
1210          */
1211         if (vcpu)
1212                 kvm_vcpu_wake_up(vcpu);
1213
1214         return 0;
1215 }
1216
1217 static __init int sev_hardware_setup(void)
1218 {
1219         struct sev_user_data_status *status;
1220         int rc;
1221
1222         /* Maximum number of encrypted guests supported simultaneously */
1223         max_sev_asid = cpuid_ecx(0x8000001F);
1224
1225         if (!max_sev_asid)
1226                 return 1;
1227
1228         /* Minimum ASID value that should be used for SEV guest */
1229         min_sev_asid = cpuid_edx(0x8000001F);
1230
1231         /* Initialize SEV ASID bitmap */
1232         sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1233         if (!sev_asid_bitmap)
1234                 return 1;
1235
1236         status = kmalloc(sizeof(*status), GFP_KERNEL);
1237         if (!status)
1238                 return 1;
1239
1240         /*
1241          * Check SEV platform status.
1242          *
1243          * PLATFORM_STATUS can be called in any state, if we failed to query
1244          * the PLATFORM status then either PSP firmware does not support SEV
1245          * feature or SEV firmware is dead.
1246          */
1247         rc = sev_platform_status(status, NULL);
1248         if (rc)
1249                 goto err;
1250
1251         pr_info("SEV supported\n");
1252
1253 err:
1254         kfree(status);
1255         return rc;
1256 }
1257
1258 static void grow_ple_window(struct kvm_vcpu *vcpu)
1259 {
1260         struct vcpu_svm *svm = to_svm(vcpu);
1261         struct vmcb_control_area *control = &svm->vmcb->control;
1262         int old = control->pause_filter_count;
1263
1264         control->pause_filter_count = __grow_ple_window(old,
1265                                                         pause_filter_count,
1266                                                         pause_filter_count_grow,
1267                                                         pause_filter_count_max);
1268
1269         if (control->pause_filter_count != old)
1270                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1271
1272         trace_kvm_ple_window_grow(vcpu->vcpu_id,
1273                                   control->pause_filter_count, old);
1274 }
1275
1276 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1277 {
1278         struct vcpu_svm *svm = to_svm(vcpu);
1279         struct vmcb_control_area *control = &svm->vmcb->control;
1280         int old = control->pause_filter_count;
1281
1282         control->pause_filter_count =
1283                                 __shrink_ple_window(old,
1284                                                     pause_filter_count,
1285                                                     pause_filter_count_shrink,
1286                                                     pause_filter_count);
1287         if (control->pause_filter_count != old)
1288                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1289
1290         trace_kvm_ple_window_shrink(vcpu->vcpu_id,
1291                                     control->pause_filter_count, old);
1292 }
1293
1294 static __init int svm_hardware_setup(void)
1295 {
1296         int cpu;
1297         struct page *iopm_pages;
1298         void *iopm_va;
1299         int r;
1300
1301         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1302
1303         if (!iopm_pages)
1304                 return -ENOMEM;
1305
1306         iopm_va = page_address(iopm_pages);
1307         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1308         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1309
1310         init_msrpm_offsets();
1311
1312         if (boot_cpu_has(X86_FEATURE_NX))
1313                 kvm_enable_efer_bits(EFER_NX);
1314
1315         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1316                 kvm_enable_efer_bits(EFER_FFXSR);
1317
1318         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1319                 kvm_has_tsc_control = true;
1320                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1321                 kvm_tsc_scaling_ratio_frac_bits = 32;
1322         }
1323
1324         /* Check for pause filtering support */
1325         if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1326                 pause_filter_count = 0;
1327                 pause_filter_thresh = 0;
1328         } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1329                 pause_filter_thresh = 0;
1330         }
1331
1332         if (nested) {
1333                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1334                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1335         }
1336
1337         if (sev) {
1338                 if (boot_cpu_has(X86_FEATURE_SEV) &&
1339                     IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1340                         r = sev_hardware_setup();
1341                         if (r)
1342                                 sev = false;
1343                 } else {
1344                         sev = false;
1345                 }
1346         }
1347
1348         for_each_possible_cpu(cpu) {
1349                 r = svm_cpu_init(cpu);
1350                 if (r)
1351                         goto err;
1352         }
1353
1354         if (!boot_cpu_has(X86_FEATURE_NPT))
1355                 npt_enabled = false;
1356
1357         if (npt_enabled && !npt) {
1358                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1359                 npt_enabled = false;
1360         }
1361
1362         if (npt_enabled) {
1363                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1364                 kvm_enable_tdp();
1365         } else
1366                 kvm_disable_tdp();
1367
1368         if (nrips) {
1369                 if (!boot_cpu_has(X86_FEATURE_NRIPS))
1370                         nrips = false;
1371         }
1372
1373         if (avic) {
1374                 if (!npt_enabled ||
1375                     !boot_cpu_has(X86_FEATURE_AVIC) ||
1376                     !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1377                         avic = false;
1378                 } else {
1379                         pr_info("AVIC enabled\n");
1380
1381                         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1382                 }
1383         }
1384
1385         if (vls) {
1386                 if (!npt_enabled ||
1387                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1388                     !IS_ENABLED(CONFIG_X86_64)) {
1389                         vls = false;
1390                 } else {
1391                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1392                 }
1393         }
1394
1395         if (vgif) {
1396                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1397                         vgif = false;
1398                 else
1399                         pr_info("Virtual GIF supported\n");
1400         }
1401
1402         return 0;
1403
1404 err:
1405         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1406         iopm_base = 0;
1407         return r;
1408 }
1409
1410 static __exit void svm_hardware_unsetup(void)
1411 {
1412         int cpu;
1413
1414         if (svm_sev_enabled())
1415                 bitmap_free(sev_asid_bitmap);
1416
1417         for_each_possible_cpu(cpu)
1418                 svm_cpu_uninit(cpu);
1419
1420         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1421         iopm_base = 0;
1422 }
1423
1424 static void init_seg(struct vmcb_seg *seg)
1425 {
1426         seg->selector = 0;
1427         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1428                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1429         seg->limit = 0xffff;
1430         seg->base = 0;
1431 }
1432
1433 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1434 {
1435         seg->selector = 0;
1436         seg->attrib = SVM_SELECTOR_P_MASK | type;
1437         seg->limit = 0xffff;
1438         seg->base = 0;
1439 }
1440
1441 static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
1442 {
1443         struct vcpu_svm *svm = to_svm(vcpu);
1444
1445         if (is_guest_mode(vcpu))
1446                 return svm->nested.hsave->control.tsc_offset;
1447
1448         return vcpu->arch.tsc_offset;
1449 }
1450
1451 static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1452 {
1453         struct vcpu_svm *svm = to_svm(vcpu);
1454         u64 g_tsc_offset = 0;
1455
1456         if (is_guest_mode(vcpu)) {
1457                 /* Write L1's TSC offset.  */
1458                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1459                                svm->nested.hsave->control.tsc_offset;
1460                 svm->nested.hsave->control.tsc_offset = offset;
1461         }
1462
1463         trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1464                                    svm->vmcb->control.tsc_offset - g_tsc_offset,
1465                                    offset);
1466
1467         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1468
1469         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1470         return svm->vmcb->control.tsc_offset;
1471 }
1472
1473 static void avic_init_vmcb(struct vcpu_svm *svm)
1474 {
1475         struct vmcb *vmcb = svm->vmcb;
1476         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
1477         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
1478         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
1479         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
1480
1481         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1482         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1483         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1484         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1485         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1486 }
1487
1488 static void init_vmcb(struct vcpu_svm *svm)
1489 {
1490         struct vmcb_control_area *control = &svm->vmcb->control;
1491         struct vmcb_save_area *save = &svm->vmcb->save;
1492
1493         svm->vcpu.arch.hflags = 0;
1494
1495         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1496         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1497         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1498         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1499         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1500         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1501         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1502                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1503
1504         set_dr_intercepts(svm);
1505
1506         set_exception_intercept(svm, PF_VECTOR);
1507         set_exception_intercept(svm, UD_VECTOR);
1508         set_exception_intercept(svm, MC_VECTOR);
1509         set_exception_intercept(svm, AC_VECTOR);
1510         set_exception_intercept(svm, DB_VECTOR);
1511         /*
1512          * Guest access to VMware backdoor ports could legitimately
1513          * trigger #GP because of TSS I/O permission bitmap.
1514          * We intercept those #GP and allow access to them anyway
1515          * as VMware does.
1516          */
1517         if (enable_vmware_backdoor)
1518                 set_exception_intercept(svm, GP_VECTOR);
1519
1520         set_intercept(svm, INTERCEPT_INTR);
1521         set_intercept(svm, INTERCEPT_NMI);
1522         set_intercept(svm, INTERCEPT_SMI);
1523         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1524         set_intercept(svm, INTERCEPT_RDPMC);
1525         set_intercept(svm, INTERCEPT_CPUID);
1526         set_intercept(svm, INTERCEPT_INVD);
1527         set_intercept(svm, INTERCEPT_INVLPG);
1528         set_intercept(svm, INTERCEPT_INVLPGA);
1529         set_intercept(svm, INTERCEPT_IOIO_PROT);
1530         set_intercept(svm, INTERCEPT_MSR_PROT);
1531         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1532         set_intercept(svm, INTERCEPT_SHUTDOWN);
1533         set_intercept(svm, INTERCEPT_VMRUN);
1534         set_intercept(svm, INTERCEPT_VMMCALL);
1535         set_intercept(svm, INTERCEPT_VMLOAD);
1536         set_intercept(svm, INTERCEPT_VMSAVE);
1537         set_intercept(svm, INTERCEPT_STGI);
1538         set_intercept(svm, INTERCEPT_CLGI);
1539         set_intercept(svm, INTERCEPT_SKINIT);
1540         set_intercept(svm, INTERCEPT_WBINVD);
1541         set_intercept(svm, INTERCEPT_XSETBV);
1542         set_intercept(svm, INTERCEPT_RSM);
1543
1544         if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1545                 set_intercept(svm, INTERCEPT_MONITOR);
1546                 set_intercept(svm, INTERCEPT_MWAIT);
1547         }
1548
1549         if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1550                 set_intercept(svm, INTERCEPT_HLT);
1551
1552         control->iopm_base_pa = __sme_set(iopm_base);
1553         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1554         control->int_ctl = V_INTR_MASKING_MASK;
1555
1556         init_seg(&save->es);
1557         init_seg(&save->ss);
1558         init_seg(&save->ds);
1559         init_seg(&save->fs);
1560         init_seg(&save->gs);
1561
1562         save->cs.selector = 0xf000;
1563         save->cs.base = 0xffff0000;
1564         /* Executable/Readable Code Segment */
1565         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1566                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1567         save->cs.limit = 0xffff;
1568
1569         save->gdtr.limit = 0xffff;
1570         save->idtr.limit = 0xffff;
1571
1572         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1573         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1574
1575         svm_set_efer(&svm->vcpu, 0);
1576         save->dr6 = 0xffff0ff0;
1577         kvm_set_rflags(&svm->vcpu, 2);
1578         save->rip = 0x0000fff0;
1579         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1580
1581         /*
1582          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1583          * It also updates the guest-visible cr0 value.
1584          */
1585         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1586         kvm_mmu_reset_context(&svm->vcpu);
1587
1588         save->cr4 = X86_CR4_PAE;
1589         /* rdx = ?? */
1590
1591         if (npt_enabled) {
1592                 /* Setup VMCB for Nested Paging */
1593                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1594                 clr_intercept(svm, INTERCEPT_INVLPG);
1595                 clr_exception_intercept(svm, PF_VECTOR);
1596                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1597                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1598                 save->g_pat = svm->vcpu.arch.pat;
1599                 save->cr3 = 0;
1600                 save->cr4 = 0;
1601         }
1602         svm->asid_generation = 0;
1603
1604         svm->nested.vmcb = 0;
1605         svm->vcpu.arch.hflags = 0;
1606
1607         if (pause_filter_count) {
1608                 control->pause_filter_count = pause_filter_count;
1609                 if (pause_filter_thresh)
1610                         control->pause_filter_thresh = pause_filter_thresh;
1611                 set_intercept(svm, INTERCEPT_PAUSE);
1612         } else {
1613                 clr_intercept(svm, INTERCEPT_PAUSE);
1614         }
1615
1616         if (kvm_vcpu_apicv_active(&svm->vcpu))
1617                 avic_init_vmcb(svm);
1618
1619         /*
1620          * If hardware supports Virtual VMLOAD VMSAVE then enable it
1621          * in VMCB and clear intercepts to avoid #VMEXIT.
1622          */
1623         if (vls) {
1624                 clr_intercept(svm, INTERCEPT_VMLOAD);
1625                 clr_intercept(svm, INTERCEPT_VMSAVE);
1626                 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1627         }
1628
1629         if (vgif) {
1630                 clr_intercept(svm, INTERCEPT_STGI);
1631                 clr_intercept(svm, INTERCEPT_CLGI);
1632                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1633         }
1634
1635         if (sev_guest(svm->vcpu.kvm)) {
1636                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1637                 clr_exception_intercept(svm, UD_VECTOR);
1638         }
1639
1640         mark_all_dirty(svm->vmcb);
1641
1642         enable_gif(svm);
1643
1644 }
1645
1646 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1647                                        unsigned int index)
1648 {
1649         u64 *avic_physical_id_table;
1650         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
1651
1652         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1653                 return NULL;
1654
1655         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
1656
1657         return &avic_physical_id_table[index];
1658 }
1659
1660 /**
1661  * Note:
1662  * AVIC hardware walks the nested page table to check permissions,
1663  * but does not use the SPA address specified in the leaf page
1664  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1665  * field of the VMCB. Therefore, we set up the
1666  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1667  */
1668 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1669 {
1670         struct kvm *kvm = vcpu->kvm;
1671         int ret = 0;
1672
1673         mutex_lock(&kvm->slots_lock);
1674         if (kvm->arch.apic_access_page_done)
1675                 goto out;
1676
1677         ret = __x86_set_memory_region(kvm,
1678                                       APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1679                                       APIC_DEFAULT_PHYS_BASE,
1680                                       PAGE_SIZE);
1681         if (ret)
1682                 goto out;
1683
1684         kvm->arch.apic_access_page_done = true;
1685 out:
1686         mutex_unlock(&kvm->slots_lock);
1687         return ret;
1688 }
1689
1690 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1691 {
1692         int ret;
1693         u64 *entry, new_entry;
1694         int id = vcpu->vcpu_id;
1695         struct vcpu_svm *svm = to_svm(vcpu);
1696
1697         ret = avic_init_access_page(vcpu);
1698         if (ret)
1699                 return ret;
1700
1701         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1702                 return -EINVAL;
1703
1704         if (!svm->vcpu.arch.apic->regs)
1705                 return -EINVAL;
1706
1707         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1708
1709         /* Setting AVIC backing page address in the phy APIC ID table */
1710         entry = avic_get_physical_id_entry(vcpu, id);
1711         if (!entry)
1712                 return -EINVAL;
1713
1714         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1715                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1716                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1717         WRITE_ONCE(*entry, new_entry);
1718
1719         svm->avic_physical_id_cache = entry;
1720
1721         return 0;
1722 }
1723
1724 static void __sev_asid_free(int asid)
1725 {
1726         struct svm_cpu_data *sd;
1727         int cpu, pos;
1728
1729         pos = asid - 1;
1730         clear_bit(pos, sev_asid_bitmap);
1731
1732         for_each_possible_cpu(cpu) {
1733                 sd = per_cpu(svm_data, cpu);
1734                 sd->sev_vmcbs[pos] = NULL;
1735         }
1736 }
1737
1738 static void sev_asid_free(struct kvm *kvm)
1739 {
1740         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1741
1742         __sev_asid_free(sev->asid);
1743 }
1744
1745 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1746 {
1747         struct sev_data_decommission *decommission;
1748         struct sev_data_deactivate *data;
1749
1750         if (!handle)
1751                 return;
1752
1753         data = kzalloc(sizeof(*data), GFP_KERNEL);
1754         if (!data)
1755                 return;
1756
1757         /* deactivate handle */
1758         data->handle = handle;
1759         sev_guest_deactivate(data, NULL);
1760
1761         wbinvd_on_all_cpus();
1762         sev_guest_df_flush(NULL);
1763         kfree(data);
1764
1765         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1766         if (!decommission)
1767                 return;
1768
1769         /* decommission handle */
1770         decommission->handle = handle;
1771         sev_guest_decommission(decommission, NULL);
1772
1773         kfree(decommission);
1774 }
1775
1776 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1777                                     unsigned long ulen, unsigned long *n,
1778                                     int write)
1779 {
1780         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1781         unsigned long npages, npinned, size;
1782         unsigned long locked, lock_limit;
1783         struct page **pages;
1784         unsigned long first, last;
1785
1786         if (ulen == 0 || uaddr + ulen < uaddr)
1787                 return NULL;
1788
1789         /* Calculate number of pages. */
1790         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1791         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1792         npages = (last - first + 1);
1793
1794         locked = sev->pages_locked + npages;
1795         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1796         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1797                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1798                 return NULL;
1799         }
1800
1801         /* Avoid using vmalloc for smaller buffers. */
1802         size = npages * sizeof(struct page *);
1803         if (size > PAGE_SIZE)
1804                 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1805                                   PAGE_KERNEL);
1806         else
1807                 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
1808
1809         if (!pages)
1810                 return NULL;
1811
1812         /* Pin the user virtual address. */
1813         npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages);
1814         if (npinned != npages) {
1815                 pr_err("SEV: Failure locking %lu pages.\n", npages);
1816                 goto err;
1817         }
1818
1819         *n = npages;
1820         sev->pages_locked = locked;
1821
1822         return pages;
1823
1824 err:
1825         if (npinned > 0)
1826                 release_pages(pages, npinned);
1827
1828         kvfree(pages);
1829         return NULL;
1830 }
1831
1832 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1833                              unsigned long npages)
1834 {
1835         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1836
1837         release_pages(pages, npages);
1838         kvfree(pages);
1839         sev->pages_locked -= npages;
1840 }
1841
1842 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1843 {
1844         uint8_t *page_virtual;
1845         unsigned long i;
1846
1847         if (npages == 0 || pages == NULL)
1848                 return;
1849
1850         for (i = 0; i < npages; i++) {
1851                 page_virtual = kmap_atomic(pages[i]);
1852                 clflush_cache_range(page_virtual, PAGE_SIZE);
1853                 kunmap_atomic(page_virtual);
1854         }
1855 }
1856
1857 static void __unregister_enc_region_locked(struct kvm *kvm,
1858                                            struct enc_region *region)
1859 {
1860         /*
1861          * The guest may change the memory encryption attribute from C=0 -> C=1
1862          * or vice versa for this memory range. Lets make sure caches are
1863          * flushed to ensure that guest data gets written into memory with
1864          * correct C-bit.
1865          */
1866         sev_clflush_pages(region->pages, region->npages);
1867
1868         sev_unpin_memory(kvm, region->pages, region->npages);
1869         list_del(&region->list);
1870         kfree(region);
1871 }
1872
1873 static struct kvm *svm_vm_alloc(void)
1874 {
1875         struct kvm_svm *kvm_svm = __vmalloc(sizeof(struct kvm_svm),
1876                                             GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1877                                             PAGE_KERNEL);
1878         return &kvm_svm->kvm;
1879 }
1880
1881 static void svm_vm_free(struct kvm *kvm)
1882 {
1883         vfree(to_kvm_svm(kvm));
1884 }
1885
1886 static void sev_vm_destroy(struct kvm *kvm)
1887 {
1888         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1889         struct list_head *head = &sev->regions_list;
1890         struct list_head *pos, *q;
1891
1892         if (!sev_guest(kvm))
1893                 return;
1894
1895         mutex_lock(&kvm->lock);
1896
1897         /*
1898          * if userspace was terminated before unregistering the memory regions
1899          * then lets unpin all the registered memory.
1900          */
1901         if (!list_empty(head)) {
1902                 list_for_each_safe(pos, q, head) {
1903                         __unregister_enc_region_locked(kvm,
1904                                 list_entry(pos, struct enc_region, list));
1905                 }
1906         }
1907
1908         mutex_unlock(&kvm->lock);
1909
1910         sev_unbind_asid(kvm, sev->handle);
1911         sev_asid_free(kvm);
1912 }
1913
1914 static void avic_vm_destroy(struct kvm *kvm)
1915 {
1916         unsigned long flags;
1917         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1918
1919         if (!avic)
1920                 return;
1921
1922         if (kvm_svm->avic_logical_id_table_page)
1923                 __free_page(kvm_svm->avic_logical_id_table_page);
1924         if (kvm_svm->avic_physical_id_table_page)
1925                 __free_page(kvm_svm->avic_physical_id_table_page);
1926
1927         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1928         hash_del(&kvm_svm->hnode);
1929         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1930 }
1931
1932 static void svm_vm_destroy(struct kvm *kvm)
1933 {
1934         avic_vm_destroy(kvm);
1935         sev_vm_destroy(kvm);
1936 }
1937
1938 static int avic_vm_init(struct kvm *kvm)
1939 {
1940         unsigned long flags;
1941         int err = -ENOMEM;
1942         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1943         struct kvm_svm *k2;
1944         struct page *p_page;
1945         struct page *l_page;
1946         u32 vm_id;
1947
1948         if (!avic)
1949                 return 0;
1950
1951         /* Allocating physical APIC ID table (4KB) */
1952         p_page = alloc_page(GFP_KERNEL_ACCOUNT);
1953         if (!p_page)
1954                 goto free_avic;
1955
1956         kvm_svm->avic_physical_id_table_page = p_page;
1957         clear_page(page_address(p_page));
1958
1959         /* Allocating logical APIC ID table (4KB) */
1960         l_page = alloc_page(GFP_KERNEL_ACCOUNT);
1961         if (!l_page)
1962                 goto free_avic;
1963
1964         kvm_svm->avic_logical_id_table_page = l_page;
1965         clear_page(page_address(l_page));
1966
1967         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1968  again:
1969         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
1970         if (vm_id == 0) { /* id is 1-based, zero is not okay */
1971                 next_vm_id_wrapped = 1;
1972                 goto again;
1973         }
1974         /* Is it still in use? Only possible if wrapped at least once */
1975         if (next_vm_id_wrapped) {
1976                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
1977                         if (k2->avic_vm_id == vm_id)
1978                                 goto again;
1979                 }
1980         }
1981         kvm_svm->avic_vm_id = vm_id;
1982         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
1983         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1984
1985         return 0;
1986
1987 free_avic:
1988         avic_vm_destroy(kvm);
1989         return err;
1990 }
1991
1992 static inline int
1993 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1994 {
1995         int ret = 0;
1996         unsigned long flags;
1997         struct amd_svm_iommu_ir *ir;
1998         struct vcpu_svm *svm = to_svm(vcpu);
1999
2000         if (!kvm_arch_has_assigned_device(vcpu->kvm))
2001                 return 0;
2002
2003         /*
2004          * Here, we go through the per-vcpu ir_list to update all existing
2005          * interrupt remapping table entry targeting this vcpu.
2006          */
2007         spin_lock_irqsave(&svm->ir_list_lock, flags);
2008
2009         if (list_empty(&svm->ir_list))
2010                 goto out;
2011
2012         list_for_each_entry(ir, &svm->ir_list, node) {
2013                 ret = amd_iommu_update_ga(cpu, r, ir->data);
2014                 if (ret)
2015                         break;
2016         }
2017 out:
2018         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
2019         return ret;
2020 }
2021
2022 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2023 {
2024         u64 entry;
2025         /* ID = 0xff (broadcast), ID > 0xff (reserved) */
2026         int h_physical_id = kvm_cpu_get_apicid(cpu);
2027         struct vcpu_svm *svm = to_svm(vcpu);
2028
2029         if (!kvm_vcpu_apicv_active(vcpu))
2030                 return;
2031
2032         /*
2033          * Since the host physical APIC id is 8 bits,
2034          * we can support host APIC ID upto 255.
2035          */
2036         if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
2037                 return;
2038
2039         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2040         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
2041
2042         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
2043         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
2044
2045         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2046         if (svm->avic_is_running)
2047                 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2048
2049         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2050         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
2051                                         svm->avic_is_running);
2052 }
2053
2054 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
2055 {
2056         u64 entry;
2057         struct vcpu_svm *svm = to_svm(vcpu);
2058
2059         if (!kvm_vcpu_apicv_active(vcpu))
2060                 return;
2061
2062         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2063         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
2064                 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
2065
2066         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2067         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2068 }
2069
2070 /**
2071  * This function is called during VCPU halt/unhalt.
2072  */
2073 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
2074 {
2075         struct vcpu_svm *svm = to_svm(vcpu);
2076
2077         svm->avic_is_running = is_run;
2078         if (is_run)
2079                 avic_vcpu_load(vcpu, vcpu->cpu);
2080         else
2081                 avic_vcpu_put(vcpu);
2082 }
2083
2084 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
2085 {
2086         struct vcpu_svm *svm = to_svm(vcpu);
2087         u32 dummy;
2088         u32 eax = 1;
2089
2090         vcpu->arch.microcode_version = 0x01000065;
2091         svm->spec_ctrl = 0;
2092         svm->virt_spec_ctrl = 0;
2093
2094         if (!init_event) {
2095                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
2096                                            MSR_IA32_APICBASE_ENABLE;
2097                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
2098                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
2099         }
2100         init_vmcb(svm);
2101
2102         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
2103         kvm_rdx_write(vcpu, eax);
2104
2105         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
2106                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
2107 }
2108
2109 static int avic_init_vcpu(struct vcpu_svm *svm)
2110 {
2111         int ret;
2112
2113         if (!kvm_vcpu_apicv_active(&svm->vcpu))
2114                 return 0;
2115
2116         ret = avic_init_backing_page(&svm->vcpu);
2117         if (ret)
2118                 return ret;
2119
2120         INIT_LIST_HEAD(&svm->ir_list);
2121         spin_lock_init(&svm->ir_list_lock);
2122         svm->dfr_reg = APIC_DFR_FLAT;
2123
2124         return ret;
2125 }
2126
2127 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
2128 {
2129         struct vcpu_svm *svm;
2130         struct page *page;
2131         struct page *msrpm_pages;
2132         struct page *hsave_page;
2133         struct page *nested_msrpm_pages;
2134         int err;
2135
2136         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
2137         if (!svm) {
2138                 err = -ENOMEM;
2139                 goto out;
2140         }
2141
2142         svm->vcpu.arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
2143                                                      GFP_KERNEL_ACCOUNT);
2144         if (!svm->vcpu.arch.user_fpu) {
2145                 printk(KERN_ERR "kvm: failed to allocate kvm userspace's fpu\n");
2146                 err = -ENOMEM;
2147                 goto free_partial_svm;
2148         }
2149
2150         svm->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
2151                                                      GFP_KERNEL_ACCOUNT);
2152         if (!svm->vcpu.arch.guest_fpu) {
2153                 printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n");
2154                 err = -ENOMEM;
2155                 goto free_user_fpu;
2156         }
2157
2158         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
2159         if (err)
2160                 goto free_svm;
2161
2162         err = -ENOMEM;
2163         page = alloc_page(GFP_KERNEL_ACCOUNT);
2164         if (!page)
2165                 goto uninit;
2166
2167         msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
2168         if (!msrpm_pages)
2169                 goto free_page1;
2170
2171         nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
2172         if (!nested_msrpm_pages)
2173                 goto free_page2;
2174
2175         hsave_page = alloc_page(GFP_KERNEL_ACCOUNT);
2176         if (!hsave_page)
2177                 goto free_page3;
2178
2179         err = avic_init_vcpu(svm);
2180         if (err)
2181                 goto free_page4;
2182
2183         /* We initialize this flag to true to make sure that the is_running
2184          * bit would be set the first time the vcpu is loaded.
2185          */
2186         svm->avic_is_running = true;
2187
2188         svm->nested.hsave = page_address(hsave_page);
2189
2190         svm->msrpm = page_address(msrpm_pages);
2191         svm_vcpu_init_msrpm(svm->msrpm);
2192
2193         svm->nested.msrpm = page_address(nested_msrpm_pages);
2194         svm_vcpu_init_msrpm(svm->nested.msrpm);
2195
2196         svm->vmcb = page_address(page);
2197         clear_page(svm->vmcb);
2198         svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2199         svm->asid_generation = 0;
2200         init_vmcb(svm);
2201
2202         svm_init_osvw(&svm->vcpu);
2203
2204         return &svm->vcpu;
2205
2206 free_page4:
2207         __free_page(hsave_page);
2208 free_page3:
2209         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2210 free_page2:
2211         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2212 free_page1:
2213         __free_page(page);
2214 uninit:
2215         kvm_vcpu_uninit(&svm->vcpu);
2216 free_svm:
2217         kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.guest_fpu);
2218 free_user_fpu:
2219         kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.user_fpu);
2220 free_partial_svm:
2221         kmem_cache_free(kvm_vcpu_cache, svm);
2222 out:
2223         return ERR_PTR(err);
2224 }
2225
2226 static void svm_clear_current_vmcb(struct vmcb *vmcb)
2227 {
2228         int i;
2229
2230         for_each_online_cpu(i)
2231                 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
2232 }
2233
2234 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2235 {
2236         struct vcpu_svm *svm = to_svm(vcpu);
2237
2238         /*
2239          * The vmcb page can be recycled, causing a false negative in
2240          * svm_vcpu_load(). So, ensure that no logical CPU has this
2241          * vmcb page recorded as its current vmcb.
2242          */
2243         svm_clear_current_vmcb(svm->vmcb);
2244
2245         __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2246         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2247         __free_page(virt_to_page(svm->nested.hsave));
2248         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
2249         kvm_vcpu_uninit(vcpu);
2250         kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.user_fpu);
2251         kmem_cache_free(x86_fpu_cache, svm->vcpu.arch.guest_fpu);
2252         kmem_cache_free(kvm_vcpu_cache, svm);
2253 }
2254
2255 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2256 {
2257         struct vcpu_svm *svm = to_svm(vcpu);
2258         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2259         int i;
2260
2261         if (unlikely(cpu != vcpu->cpu)) {
2262                 svm->asid_generation = 0;
2263                 mark_all_dirty(svm->vmcb);
2264         }
2265
2266 #ifdef CONFIG_X86_64
2267         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2268 #endif
2269         savesegment(fs, svm->host.fs);
2270         savesegment(gs, svm->host.gs);
2271         svm->host.ldt = kvm_read_ldt();
2272
2273         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2274                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2275
2276         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2277                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2278                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2279                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
2280                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2281                 }
2282         }
2283         /* This assumes that the kernel never uses MSR_TSC_AUX */
2284         if (static_cpu_has(X86_FEATURE_RDTSCP))
2285                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2286
2287         if (sd->current_vmcb != svm->vmcb) {
2288                 sd->current_vmcb = svm->vmcb;
2289                 indirect_branch_prediction_barrier();
2290         }
2291         avic_vcpu_load(vcpu, cpu);
2292 }
2293
2294 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2295 {
2296         struct vcpu_svm *svm = to_svm(vcpu);
2297         int i;
2298
2299         avic_vcpu_put(vcpu);
2300
2301         ++vcpu->stat.host_state_reload;
2302         kvm_load_ldt(svm->host.ldt);
2303 #ifdef CONFIG_X86_64
2304         loadsegment(fs, svm->host.fs);
2305         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2306         load_gs_index(svm->host.gs);
2307 #else
2308 #ifdef CONFIG_X86_32_LAZY_GS
2309         loadsegment(gs, svm->host.gs);
2310 #endif
2311 #endif
2312         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2313                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2314 }
2315
2316 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2317 {
2318         avic_set_running(vcpu, false);
2319 }
2320
2321 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2322 {
2323         avic_set_running(vcpu, true);
2324 }
2325
2326 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2327 {
2328         struct vcpu_svm *svm = to_svm(vcpu);
2329         unsigned long rflags = svm->vmcb->save.rflags;
2330
2331         if (svm->nmi_singlestep) {
2332                 /* Hide our flags if they were not set by the guest */
2333                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2334                         rflags &= ~X86_EFLAGS_TF;
2335                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2336                         rflags &= ~X86_EFLAGS_RF;
2337         }
2338         return rflags;
2339 }
2340
2341 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2342 {
2343         if (to_svm(vcpu)->nmi_singlestep)
2344                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2345
2346        /*
2347         * Any change of EFLAGS.VM is accompanied by a reload of SS
2348         * (caused by either a task switch or an inter-privilege IRET),
2349         * so we do not need to update the CPL here.
2350         */
2351         to_svm(vcpu)->vmcb->save.rflags = rflags;
2352 }
2353
2354 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2355 {
2356         switch (reg) {
2357         case VCPU_EXREG_PDPTR:
2358                 BUG_ON(!npt_enabled);
2359                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
2360                 break;
2361         default:
2362                 BUG();
2363         }
2364 }
2365
2366 static void svm_set_vintr(struct vcpu_svm *svm)
2367 {
2368         set_intercept(svm, INTERCEPT_VINTR);
2369 }
2370
2371 static void svm_clear_vintr(struct vcpu_svm *svm)
2372 {
2373         clr_intercept(svm, INTERCEPT_VINTR);
2374 }
2375
2376 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2377 {
2378         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2379
2380         switch (seg) {
2381         case VCPU_SREG_CS: return &save->cs;
2382         case VCPU_SREG_DS: return &save->ds;
2383         case VCPU_SREG_ES: return &save->es;
2384         case VCPU_SREG_FS: return &save->fs;
2385         case VCPU_SREG_GS: return &save->gs;
2386         case VCPU_SREG_SS: return &save->ss;
2387         case VCPU_SREG_TR: return &save->tr;
2388         case VCPU_SREG_LDTR: return &save->ldtr;
2389         }
2390         BUG();
2391         return NULL;
2392 }
2393
2394 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2395 {
2396         struct vmcb_seg *s = svm_seg(vcpu, seg);
2397
2398         return s->base;
2399 }
2400
2401 static void svm_get_segment(struct kvm_vcpu *vcpu,
2402                             struct kvm_segment *var, int seg)
2403 {
2404         struct vmcb_seg *s = svm_seg(vcpu, seg);
2405
2406         var->base = s->base;
2407         var->limit = s->limit;
2408         var->selector = s->selector;
2409         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2410         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2411         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2412         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2413         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2414         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2415         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2416
2417         /*
2418          * AMD CPUs circa 2014 track the G bit for all segments except CS.
2419          * However, the SVM spec states that the G bit is not observed by the
2420          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2421          * So let's synthesize a legal G bit for all segments, this helps
2422          * running KVM nested. It also helps cross-vendor migration, because
2423          * Intel's vmentry has a check on the 'G' bit.
2424          */
2425         var->g = s->limit > 0xfffff;
2426
2427         /*
2428          * AMD's VMCB does not have an explicit unusable field, so emulate it
2429          * for cross vendor migration purposes by "not present"
2430          */
2431         var->unusable = !var->present;
2432
2433         switch (seg) {
2434         case VCPU_SREG_TR:
2435                 /*
2436                  * Work around a bug where the busy flag in the tr selector
2437                  * isn't exposed
2438                  */
2439                 var->type |= 0x2;
2440                 break;
2441         case VCPU_SREG_DS:
2442         case VCPU_SREG_ES:
2443         case VCPU_SREG_FS:
2444         case VCPU_SREG_GS:
2445                 /*
2446                  * The accessed bit must always be set in the segment
2447                  * descriptor cache, although it can be cleared in the
2448                  * descriptor, the cached bit always remains at 1. Since
2449                  * Intel has a check on this, set it here to support
2450                  * cross-vendor migration.
2451                  */
2452                 if (!var->unusable)
2453                         var->type |= 0x1;
2454                 break;
2455         case VCPU_SREG_SS:
2456                 /*
2457                  * On AMD CPUs sometimes the DB bit in the segment
2458                  * descriptor is left as 1, although the whole segment has
2459                  * been made unusable. Clear it here to pass an Intel VMX
2460                  * entry check when cross vendor migrating.
2461                  */
2462                 if (var->unusable)
2463                         var->db = 0;
2464                 /* This is symmetric with svm_set_segment() */
2465                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2466                 break;
2467         }
2468 }
2469
2470 static int svm_get_cpl(struct kvm_vcpu *vcpu)
2471 {
2472         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2473
2474         return save->cpl;
2475 }
2476
2477 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2478 {
2479         struct vcpu_svm *svm = to_svm(vcpu);
2480
2481         dt->size = svm->vmcb->save.idtr.limit;
2482         dt->address = svm->vmcb->save.idtr.base;
2483 }
2484
2485 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2486 {
2487         struct vcpu_svm *svm = to_svm(vcpu);
2488
2489         svm->vmcb->save.idtr.limit = dt->size;
2490         svm->vmcb->save.idtr.base = dt->address ;
2491         mark_dirty(svm->vmcb, VMCB_DT);
2492 }
2493
2494 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2495 {
2496         struct vcpu_svm *svm = to_svm(vcpu);
2497
2498         dt->size = svm->vmcb->save.gdtr.limit;
2499         dt->address = svm->vmcb->save.gdtr.base;
2500 }
2501
2502 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2503 {
2504         struct vcpu_svm *svm = to_svm(vcpu);
2505
2506         svm->vmcb->save.gdtr.limit = dt->size;
2507         svm->vmcb->save.gdtr.base = dt->address ;
2508         mark_dirty(svm->vmcb, VMCB_DT);
2509 }
2510
2511 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2512 {
2513 }
2514
2515 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
2516 {
2517 }
2518
2519 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2520 {
2521 }
2522
2523 static void update_cr0_intercept(struct vcpu_svm *svm)
2524 {
2525         ulong gcr0 = svm->vcpu.arch.cr0;
2526         u64 *hcr0 = &svm->vmcb->save.cr0;
2527
2528         *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2529                 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
2530
2531         mark_dirty(svm->vmcb, VMCB_CR);
2532
2533         if (gcr0 == *hcr0) {
2534                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2535                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2536         } else {
2537                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2538                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2539         }
2540 }
2541
2542 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2543 {
2544         struct vcpu_svm *svm = to_svm(vcpu);
2545
2546 #ifdef CONFIG_X86_64
2547         if (vcpu->arch.efer & EFER_LME) {
2548                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2549                         vcpu->arch.efer |= EFER_LMA;
2550                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
2551                 }
2552
2553                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2554                         vcpu->arch.efer &= ~EFER_LMA;
2555                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2556                 }
2557         }
2558 #endif
2559         vcpu->arch.cr0 = cr0;
2560
2561         if (!npt_enabled)
2562                 cr0 |= X86_CR0_PG | X86_CR0_WP;
2563
2564         /*
2565          * re-enable caching here because the QEMU bios
2566          * does not do it - this results in some delay at
2567          * reboot
2568          */
2569         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2570                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2571         svm->vmcb->save.cr0 = cr0;
2572         mark_dirty(svm->vmcb, VMCB_CR);
2573         update_cr0_intercept(svm);
2574 }
2575
2576 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2577 {
2578         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2579         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2580
2581         if (cr4 & X86_CR4_VMXE)
2582                 return 1;
2583
2584         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2585                 svm_flush_tlb(vcpu, true);
2586
2587         vcpu->arch.cr4 = cr4;
2588         if (!npt_enabled)
2589                 cr4 |= X86_CR4_PAE;
2590         cr4 |= host_cr4_mce;
2591         to_svm(vcpu)->vmcb->save.cr4 = cr4;
2592         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2593         return 0;
2594 }
2595
2596 static void svm_set_segment(struct kvm_vcpu *vcpu,
2597                             struct kvm_segment *var, int seg)
2598 {
2599         struct vcpu_svm *svm = to_svm(vcpu);
2600         struct vmcb_seg *s = svm_seg(vcpu, seg);
2601
2602         s->base = var->base;
2603         s->limit = var->limit;
2604         s->selector = var->selector;
2605         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2606         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2607         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2608         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2609         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2610         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2611         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2612         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2613
2614         /*
2615          * This is always accurate, except if SYSRET returned to a segment
2616          * with SS.DPL != 3.  Intel does not have this quirk, and always
2617          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2618          * would entail passing the CPL to userspace and back.
2619          */
2620         if (seg == VCPU_SREG_SS)
2621                 /* This is symmetric with svm_get_segment() */
2622                 svm->vmcb->save.cpl = (var->dpl & 3);
2623
2624         mark_dirty(svm->vmcb, VMCB_SEG);
2625 }
2626
2627 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2628 {
2629         struct vcpu_svm *svm = to_svm(vcpu);
2630
2631         clr_exception_intercept(svm, BP_VECTOR);
2632
2633         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2634                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2635                         set_exception_intercept(svm, BP_VECTOR);
2636         } else
2637                 vcpu->guest_debug = 0;
2638 }
2639
2640 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2641 {
2642         if (sd->next_asid > sd->max_asid) {
2643                 ++sd->asid_generation;
2644                 sd->next_asid = sd->min_asid;
2645                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2646         }
2647
2648         svm->asid_generation = sd->asid_generation;
2649         svm->vmcb->control.asid = sd->next_asid++;
2650
2651         mark_dirty(svm->vmcb, VMCB_ASID);
2652 }
2653
2654 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2655 {
2656         return to_svm(vcpu)->vmcb->save.dr6;
2657 }
2658
2659 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2660 {
2661         struct vcpu_svm *svm = to_svm(vcpu);
2662
2663         svm->vmcb->save.dr6 = value;
2664         mark_dirty(svm->vmcb, VMCB_DR);
2665 }
2666
2667 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2668 {
2669         struct vcpu_svm *svm = to_svm(vcpu);
2670
2671         get_debugreg(vcpu->arch.db[0], 0);
2672         get_debugreg(vcpu->arch.db[1], 1);
2673         get_debugreg(vcpu->arch.db[2], 2);
2674         get_debugreg(vcpu->arch.db[3], 3);
2675         vcpu->arch.dr6 = svm_get_dr6(vcpu);
2676         vcpu->arch.dr7 = svm->vmcb->save.dr7;
2677
2678         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2679         set_dr_intercepts(svm);
2680 }
2681
2682 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2683 {
2684         struct vcpu_svm *svm = to_svm(vcpu);
2685
2686         svm->vmcb->save.dr7 = value;
2687         mark_dirty(svm->vmcb, VMCB_DR);
2688 }
2689
2690 static int pf_interception(struct vcpu_svm *svm)
2691 {
2692         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2693         u64 error_code = svm->vmcb->control.exit_info_1;
2694
2695         return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2696                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2697                         svm->vmcb->control.insn_bytes : NULL,
2698                         svm->vmcb->control.insn_len);
2699 }
2700
2701 static int npf_interception(struct vcpu_svm *svm)
2702 {
2703         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2704         u64 error_code = svm->vmcb->control.exit_info_1;
2705
2706         trace_kvm_page_fault(fault_address, error_code);
2707         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2708                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2709                         svm->vmcb->control.insn_bytes : NULL,
2710                         svm->vmcb->control.insn_len);
2711 }
2712
2713 static int db_interception(struct vcpu_svm *svm)
2714 {
2715         struct kvm_run *kvm_run = svm->vcpu.run;
2716         struct kvm_vcpu *vcpu = &svm->vcpu;
2717
2718         if (!(svm->vcpu.guest_debug &
2719               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2720                 !svm->nmi_singlestep) {
2721                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2722                 return 1;
2723         }
2724
2725         if (svm->nmi_singlestep) {
2726                 disable_nmi_singlestep(svm);
2727                 /* Make sure we check for pending NMIs upon entry */
2728                 kvm_make_request(KVM_REQ_EVENT, vcpu);
2729         }
2730
2731         if (svm->vcpu.guest_debug &
2732             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2733                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2734                 kvm_run->debug.arch.pc =
2735                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2736                 kvm_run->debug.arch.exception = DB_VECTOR;
2737                 return 0;
2738         }
2739
2740         return 1;
2741 }
2742
2743 static int bp_interception(struct vcpu_svm *svm)
2744 {
2745         struct kvm_run *kvm_run = svm->vcpu.run;
2746
2747         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2748         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2749         kvm_run->debug.arch.exception = BP_VECTOR;
2750         return 0;
2751 }
2752
2753 static int ud_interception(struct vcpu_svm *svm)
2754 {
2755         return handle_ud(&svm->vcpu);
2756 }
2757
2758 static int ac_interception(struct vcpu_svm *svm)
2759 {
2760         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2761         return 1;
2762 }
2763
2764 static int gp_interception(struct vcpu_svm *svm)
2765 {
2766         struct kvm_vcpu *vcpu = &svm->vcpu;
2767         u32 error_code = svm->vmcb->control.exit_info_1;
2768         int er;
2769
2770         WARN_ON_ONCE(!enable_vmware_backdoor);
2771
2772         er = kvm_emulate_instruction(vcpu,
2773                 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
2774         if (er == EMULATE_USER_EXIT)
2775                 return 0;
2776         else if (er != EMULATE_DONE)
2777                 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2778         return 1;
2779 }
2780
2781 static bool is_erratum_383(void)
2782 {
2783         int err, i;
2784         u64 value;
2785
2786         if (!erratum_383_found)
2787                 return false;
2788
2789         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2790         if (err)
2791                 return false;
2792
2793         /* Bit 62 may or may not be set for this mce */
2794         value &= ~(1ULL << 62);
2795
2796         if (value != 0xb600000000010015ULL)
2797                 return false;
2798
2799         /* Clear MCi_STATUS registers */
2800         for (i = 0; i < 6; ++i)
2801                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2802
2803         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2804         if (!err) {
2805                 u32 low, high;
2806
2807                 value &= ~(1ULL << 2);
2808                 low    = lower_32_bits(value);
2809                 high   = upper_32_bits(value);
2810
2811                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2812         }
2813
2814         /* Flush tlb to evict multi-match entries */
2815         __flush_tlb_all();
2816
2817         return true;
2818 }
2819
2820 static void svm_handle_mce(struct vcpu_svm *svm)
2821 {
2822         if (is_erratum_383()) {
2823                 /*
2824                  * Erratum 383 triggered. Guest state is corrupt so kill the
2825                  * guest.
2826                  */
2827                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2828
2829                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2830
2831                 return;
2832         }
2833
2834         /*
2835          * On an #MC intercept the MCE handler is not called automatically in
2836          * the host. So do it by hand here.
2837          */
2838         asm volatile (
2839                 "int $0x12\n");
2840         /* not sure if we ever come back to this point */
2841
2842         return;
2843 }
2844
2845 static int mc_interception(struct vcpu_svm *svm)
2846 {
2847         return 1;
2848 }
2849
2850 static int shutdown_interception(struct vcpu_svm *svm)
2851 {
2852         struct kvm_run *kvm_run = svm->vcpu.run;
2853
2854         /*
2855          * VMCB is undefined after a SHUTDOWN intercept
2856          * so reinitialize it.
2857          */
2858         clear_page(svm->vmcb);
2859         init_vmcb(svm);
2860
2861         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2862         return 0;
2863 }
2864
2865 static int io_interception(struct vcpu_svm *svm)
2866 {
2867         struct kvm_vcpu *vcpu = &svm->vcpu;
2868         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2869         int size, in, string;
2870         unsigned port;
2871
2872         ++svm->vcpu.stat.io_exits;
2873         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2874         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2875         if (string)
2876                 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
2877
2878         port = io_info >> 16;
2879         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2880         svm->next_rip = svm->vmcb->control.exit_info_2;
2881
2882         return kvm_fast_pio(&svm->vcpu, size, port, in);
2883 }
2884
2885 static int nmi_interception(struct vcpu_svm *svm)
2886 {
2887         return 1;
2888 }
2889
2890 static int intr_interception(struct vcpu_svm *svm)
2891 {
2892         ++svm->vcpu.stat.irq_exits;
2893         return 1;
2894 }
2895
2896 static int nop_on_interception(struct vcpu_svm *svm)
2897 {
2898         return 1;
2899 }
2900
2901 static int halt_interception(struct vcpu_svm *svm)
2902 {
2903         return kvm_emulate_halt(&svm->vcpu);
2904 }
2905
2906 static int vmmcall_interception(struct vcpu_svm *svm)
2907 {
2908         return kvm_emulate_hypercall(&svm->vcpu);
2909 }
2910
2911 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2912 {
2913         struct vcpu_svm *svm = to_svm(vcpu);
2914
2915         return svm->nested.nested_cr3;
2916 }
2917
2918 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2919 {
2920         struct vcpu_svm *svm = to_svm(vcpu);
2921         u64 cr3 = svm->nested.nested_cr3;
2922         u64 pdpte;
2923         int ret;
2924
2925         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2926                                        offset_in_page(cr3) + index * 8, 8);
2927         if (ret)
2928                 return 0;
2929         return pdpte;
2930 }
2931
2932 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2933                                    unsigned long root)
2934 {
2935         struct vcpu_svm *svm = to_svm(vcpu);
2936
2937         svm->vmcb->control.nested_cr3 = __sme_set(root);
2938         mark_dirty(svm->vmcb, VMCB_NPT);
2939 }
2940
2941 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2942                                        struct x86_exception *fault)
2943 {
2944         struct vcpu_svm *svm = to_svm(vcpu);
2945
2946         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2947                 /*
2948                  * TODO: track the cause of the nested page fault, and
2949                  * correctly fill in the high bits of exit_info_1.
2950                  */
2951                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2952                 svm->vmcb->control.exit_code_hi = 0;
2953                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2954                 svm->vmcb->control.exit_info_2 = fault->address;
2955         }
2956
2957         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2958         svm->vmcb->control.exit_info_1 |= fault->error_code;
2959
2960         /*
2961          * The present bit is always zero for page structure faults on real
2962          * hardware.
2963          */
2964         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2965                 svm->vmcb->control.exit_info_1 &= ~1;
2966
2967         nested_svm_vmexit(svm);
2968 }
2969
2970 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2971 {
2972         WARN_ON(mmu_is_nested(vcpu));
2973
2974         vcpu->arch.mmu = &vcpu->arch.guest_mmu;
2975         kvm_init_shadow_mmu(vcpu);
2976         vcpu->arch.mmu->set_cr3           = nested_svm_set_tdp_cr3;
2977         vcpu->arch.mmu->get_cr3           = nested_svm_get_tdp_cr3;
2978         vcpu->arch.mmu->get_pdptr         = nested_svm_get_tdp_pdptr;
2979         vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
2980         vcpu->arch.mmu->shadow_root_level = get_npt_level(vcpu);
2981         reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu);
2982         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
2983 }
2984
2985 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2986 {
2987         vcpu->arch.mmu = &vcpu->arch.root_mmu;
2988         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
2989 }
2990
2991 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2992 {
2993         if (!(svm->vcpu.arch.efer & EFER_SVME) ||
2994             !is_paging(&svm->vcpu)) {
2995                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2996                 return 1;
2997         }
2998
2999         if (svm->vmcb->save.cpl) {
3000                 kvm_inject_gp(&svm->vcpu, 0);
3001                 return 1;
3002         }
3003
3004         return 0;
3005 }
3006
3007 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
3008                                       bool has_error_code, u32 error_code)
3009 {
3010         int vmexit;
3011
3012         if (!is_guest_mode(&svm->vcpu))
3013                 return 0;
3014
3015         vmexit = nested_svm_intercept(svm);
3016         if (vmexit != NESTED_EXIT_DONE)
3017                 return 0;
3018
3019         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
3020         svm->vmcb->control.exit_code_hi = 0;
3021         svm->vmcb->control.exit_info_1 = error_code;
3022
3023         /*
3024          * EXITINFO2 is undefined for all exception intercepts other
3025          * than #PF.
3026          */
3027         if (svm->vcpu.arch.exception.nested_apf)
3028                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
3029         else if (svm->vcpu.arch.exception.has_payload)
3030                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload;
3031         else
3032                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
3033
3034         svm->nested.exit_required = true;
3035         return vmexit;
3036 }
3037
3038 /* This function returns true if it is save to enable the irq window */
3039 static inline bool nested_svm_intr(struct vcpu_svm *svm)
3040 {
3041         if (!is_guest_mode(&svm->vcpu))
3042                 return true;
3043
3044         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3045                 return true;
3046
3047         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
3048                 return false;
3049
3050         /*
3051          * if vmexit was already requested (by intercepted exception
3052          * for instance) do not overwrite it with "external interrupt"
3053          * vmexit.
3054          */
3055         if (svm->nested.exit_required)
3056                 return false;
3057
3058         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
3059         svm->vmcb->control.exit_info_1 = 0;
3060         svm->vmcb->control.exit_info_2 = 0;
3061
3062         if (svm->nested.intercept & 1ULL) {
3063                 /*
3064                  * The #vmexit can't be emulated here directly because this
3065                  * code path runs with irqs and preemption disabled. A
3066                  * #vmexit emulation might sleep. Only signal request for
3067                  * the #vmexit here.
3068                  */
3069                 svm->nested.exit_required = true;
3070                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
3071                 return false;
3072         }
3073
3074         return true;
3075 }
3076
3077 /* This function returns true if it is save to enable the nmi window */
3078 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
3079 {
3080         if (!is_guest_mode(&svm->vcpu))
3081                 return true;
3082
3083         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
3084                 return true;
3085
3086         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
3087         svm->nested.exit_required = true;
3088
3089         return false;
3090 }
3091
3092 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
3093 {
3094         unsigned port, size, iopm_len;
3095         u16 val, mask;
3096         u8 start_bit;
3097         u64 gpa;
3098
3099         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
3100                 return NESTED_EXIT_HOST;
3101
3102         port = svm->vmcb->control.exit_info_1 >> 16;
3103         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
3104                 SVM_IOIO_SIZE_SHIFT;
3105         gpa  = svm->nested.vmcb_iopm + (port / 8);
3106         start_bit = port % 8;
3107         iopm_len = (start_bit + size > 8) ? 2 : 1;
3108         mask = (0xf >> (4 - size)) << start_bit;
3109         val = 0;
3110
3111         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
3112                 return NESTED_EXIT_DONE;
3113
3114         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3115 }
3116
3117 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
3118 {
3119         u32 offset, msr, value;
3120         int write, mask;
3121
3122         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3123                 return NESTED_EXIT_HOST;
3124
3125         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
3126         offset = svm_msrpm_offset(msr);
3127         write  = svm->vmcb->control.exit_info_1 & 1;
3128         mask   = 1 << ((2 * (msr & 0xf)) + write);
3129
3130         if (offset == MSR_INVALID)
3131                 return NESTED_EXIT_DONE;
3132
3133         /* Offset is in 32 bit units but need in 8 bit units */
3134         offset *= 4;
3135
3136         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
3137                 return NESTED_EXIT_DONE;
3138
3139         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3140 }
3141
3142 /* DB exceptions for our internal use must not cause vmexit */
3143 static int nested_svm_intercept_db(struct vcpu_svm *svm)
3144 {
3145         unsigned long dr6;
3146
3147         /* if we're not singlestepping, it's not ours */
3148         if (!svm->nmi_singlestep)
3149                 return NESTED_EXIT_DONE;
3150
3151         /* if it's not a singlestep exception, it's not ours */
3152         if (kvm_get_dr(&svm->vcpu, 6, &dr6))
3153                 return NESTED_EXIT_DONE;
3154         if (!(dr6 & DR6_BS))
3155                 return NESTED_EXIT_DONE;
3156
3157         /* if the guest is singlestepping, it should get the vmexit */
3158         if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
3159                 disable_nmi_singlestep(svm);
3160                 return NESTED_EXIT_DONE;
3161         }
3162
3163         /* it's ours, the nested hypervisor must not see this one */
3164         return NESTED_EXIT_HOST;
3165 }
3166
3167 static int nested_svm_exit_special(struct vcpu_svm *svm)
3168 {
3169         u32 exit_code = svm->vmcb->control.exit_code;
3170
3171         switch (exit_code) {
3172         case SVM_EXIT_INTR:
3173         case SVM_EXIT_NMI:
3174         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
3175                 return NESTED_EXIT_HOST;
3176         case SVM_EXIT_NPF:
3177                 /* For now we are always handling NPFs when using them */
3178                 if (npt_enabled)
3179                         return NESTED_EXIT_HOST;
3180                 break;
3181         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
3182                 /* When we're shadowing, trap PFs, but not async PF */
3183                 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
3184                         return NESTED_EXIT_HOST;
3185                 break;
3186         default:
3187                 break;
3188         }
3189
3190         return NESTED_EXIT_CONTINUE;
3191 }
3192
3193 /*
3194  * If this function returns true, this #vmexit was already handled
3195  */
3196 static int nested_svm_intercept(struct vcpu_svm *svm)
3197 {
3198         u32 exit_code = svm->vmcb->control.exit_code;
3199         int vmexit = NESTED_EXIT_HOST;
3200
3201         switch (exit_code) {
3202         case SVM_EXIT_MSR:
3203                 vmexit = nested_svm_exit_handled_msr(svm);
3204                 break;
3205         case SVM_EXIT_IOIO:
3206                 vmexit = nested_svm_intercept_ioio(svm);
3207                 break;
3208         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3209                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3210                 if (svm->nested.intercept_cr & bit)
3211                         vmexit = NESTED_EXIT_DONE;
3212                 break;
3213         }
3214         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3215                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3216                 if (svm->nested.intercept_dr & bit)
3217                         vmexit = NESTED_EXIT_DONE;
3218                 break;
3219         }
3220         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3221                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3222                 if (svm->nested.intercept_exceptions & excp_bits) {
3223                         if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3224                                 vmexit = nested_svm_intercept_db(svm);
3225                         else
3226                                 vmexit = NESTED_EXIT_DONE;
3227                 }
3228                 /* async page fault always cause vmexit */
3229                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3230                          svm->vcpu.arch.exception.nested_apf != 0)
3231                         vmexit = NESTED_EXIT_DONE;
3232                 break;
3233         }
3234         case SVM_EXIT_ERR: {
3235                 vmexit = NESTED_EXIT_DONE;
3236                 break;
3237         }
3238         default: {
3239                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
3240                 if (svm->nested.intercept & exit_bits)
3241                         vmexit = NESTED_EXIT_DONE;
3242         }
3243         }
3244
3245         return vmexit;
3246 }
3247
3248 static int nested_svm_exit_handled(struct vcpu_svm *svm)
3249 {
3250         int vmexit;
3251
3252         vmexit = nested_svm_intercept(svm);
3253
3254         if (vmexit == NESTED_EXIT_DONE)
3255                 nested_svm_vmexit(svm);
3256
3257         return vmexit;
3258 }
3259
3260 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3261 {
3262         struct vmcb_control_area *dst  = &dst_vmcb->control;
3263         struct vmcb_control_area *from = &from_vmcb->control;
3264
3265         dst->intercept_cr         = from->intercept_cr;
3266         dst->intercept_dr         = from->intercept_dr;
3267         dst->intercept_exceptions = from->intercept_exceptions;
3268         dst->intercept            = from->intercept;
3269         dst->iopm_base_pa         = from->iopm_base_pa;
3270         dst->msrpm_base_pa        = from->msrpm_base_pa;
3271         dst->tsc_offset           = from->tsc_offset;
3272         dst->asid                 = from->asid;
3273         dst->tlb_ctl              = from->tlb_ctl;
3274         dst->int_ctl              = from->int_ctl;
3275         dst->int_vector           = from->int_vector;
3276         dst->int_state            = from->int_state;
3277         dst->exit_code            = from->exit_code;
3278         dst->exit_code_hi         = from->exit_code_hi;
3279         dst->exit_info_1          = from->exit_info_1;
3280         dst->exit_info_2          = from->exit_info_2;
3281         dst->exit_int_info        = from->exit_int_info;
3282         dst->exit_int_info_err    = from->exit_int_info_err;
3283         dst->nested_ctl           = from->nested_ctl;
3284         dst->event_inj            = from->event_inj;
3285         dst->event_inj_err        = from->event_inj_err;
3286         dst->nested_cr3           = from->nested_cr3;
3287         dst->virt_ext              = from->virt_ext;
3288         dst->pause_filter_count   = from->pause_filter_count;
3289         dst->pause_filter_thresh  = from->pause_filter_thresh;
3290 }
3291
3292 static int nested_svm_vmexit(struct vcpu_svm *svm)
3293 {
3294         int rc;
3295         struct vmcb *nested_vmcb;
3296         struct vmcb *hsave = svm->nested.hsave;
3297         struct vmcb *vmcb = svm->vmcb;
3298         struct kvm_host_map map;
3299
3300         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3301                                        vmcb->control.exit_info_1,
3302                                        vmcb->control.exit_info_2,
3303                                        vmcb->control.exit_int_info,
3304                                        vmcb->control.exit_int_info_err,
3305                                        KVM_ISA_SVM);
3306
3307         rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map);
3308         if (rc) {
3309                 if (rc == -EINVAL)
3310                         kvm_inject_gp(&svm->vcpu, 0);
3311                 return 1;
3312         }
3313
3314         nested_vmcb = map.hva;
3315
3316         /* Exit Guest-Mode */
3317         leave_guest_mode(&svm->vcpu);
3318         svm->nested.vmcb = 0;
3319
3320         /* Give the current vmcb to the guest */
3321         disable_gif(svm);
3322
3323         nested_vmcb->save.es     = vmcb->save.es;
3324         nested_vmcb->save.cs     = vmcb->save.cs;
3325         nested_vmcb->save.ss     = vmcb->save.ss;
3326         nested_vmcb->save.ds     = vmcb->save.ds;
3327         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
3328         nested_vmcb->save.idtr   = vmcb->save.idtr;
3329         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3330         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3331         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
3332         nested_vmcb->save.cr2    = vmcb->save.cr2;
3333         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3334         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
3335         nested_vmcb->save.rip    = vmcb->save.rip;
3336         nested_vmcb->save.rsp    = vmcb->save.rsp;
3337         nested_vmcb->save.rax    = vmcb->save.rax;
3338         nested_vmcb->save.dr7    = vmcb->save.dr7;
3339         nested_vmcb->save.dr6    = vmcb->save.dr6;
3340         nested_vmcb->save.cpl    = vmcb->save.cpl;
3341
3342         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
3343         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
3344         nested_vmcb->control.int_state         = vmcb->control.int_state;
3345         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
3346         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
3347         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
3348         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
3349         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
3350         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3351
3352         if (svm->nrips_enabled)
3353                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3354
3355         /*
3356          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3357          * to make sure that we do not lose injected events. So check event_inj
3358          * here and copy it to exit_int_info if it is valid.
3359          * Exit_int_info and event_inj can't be both valid because the case
3360          * below only happens on a VMRUN instruction intercept which has
3361          * no valid exit_int_info set.
3362          */
3363         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3364                 struct vmcb_control_area *nc = &nested_vmcb->control;
3365
3366                 nc->exit_int_info     = vmcb->control.event_inj;
3367                 nc->exit_int_info_err = vmcb->control.event_inj_err;
3368         }
3369
3370         nested_vmcb->control.tlb_ctl           = 0;
3371         nested_vmcb->control.event_inj         = 0;
3372         nested_vmcb->control.event_inj_err     = 0;
3373
3374         nested_vmcb->control.pause_filter_count =
3375                 svm->vmcb->control.pause_filter_count;
3376         nested_vmcb->control.pause_filter_thresh =
3377                 svm->vmcb->control.pause_filter_thresh;
3378
3379         /* We always set V_INTR_MASKING and remember the old value in hflags */
3380         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3381                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3382
3383         /* Restore the original control entries */
3384         copy_vmcb_control_area(vmcb, hsave);
3385
3386         svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset;
3387         kvm_clear_exception_queue(&svm->vcpu);
3388         kvm_clear_interrupt_queue(&svm->vcpu);
3389
3390         svm->nested.nested_cr3 = 0;
3391
3392         /* Restore selected save entries */
3393         svm->vmcb->save.es = hsave->save.es;
3394         svm->vmcb->save.cs = hsave->save.cs;
3395         svm->vmcb->save.ss = hsave->save.ss;
3396         svm->vmcb->save.ds = hsave->save.ds;
3397         svm->vmcb->save.gdtr = hsave->save.gdtr;
3398         svm->vmcb->save.idtr = hsave->save.idtr;
3399         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3400         svm_set_efer(&svm->vcpu, hsave->save.efer);
3401         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3402         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3403         if (npt_enabled) {
3404                 svm->vmcb->save.cr3 = hsave->save.cr3;
3405                 svm->vcpu.arch.cr3 = hsave->save.cr3;
3406         } else {
3407                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3408         }
3409         kvm_rax_write(&svm->vcpu, hsave->save.rax);
3410         kvm_rsp_write(&svm->vcpu, hsave->save.rsp);
3411         kvm_rip_write(&svm->vcpu, hsave->save.rip);
3412         svm->vmcb->save.dr7 = 0;
3413         svm->vmcb->save.cpl = 0;
3414         svm->vmcb->control.exit_int_info = 0;
3415
3416         mark_all_dirty(svm->vmcb);
3417
3418         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3419
3420         nested_svm_uninit_mmu_context(&svm->vcpu);
3421         kvm_mmu_reset_context(&svm->vcpu);
3422         kvm_mmu_load(&svm->vcpu);
3423
3424         /*
3425          * Drop what we picked up for L2 via svm_complete_interrupts() so it
3426          * doesn't end up in L1.
3427          */
3428         svm->vcpu.arch.nmi_injected = false;
3429         kvm_clear_exception_queue(&svm->vcpu);
3430         kvm_clear_interrupt_queue(&svm->vcpu);
3431
3432         return 0;
3433 }
3434
3435 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3436 {
3437         /*
3438          * This function merges the msr permission bitmaps of kvm and the
3439          * nested vmcb. It is optimized in that it only merges the parts where
3440          * the kvm msr permission bitmap may contain zero bits
3441          */
3442         int i;
3443
3444         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3445                 return true;
3446
3447         for (i = 0; i < MSRPM_OFFSETS; i++) {
3448                 u32 value, p;
3449                 u64 offset;
3450
3451                 if (msrpm_offsets[i] == 0xffffffff)
3452                         break;
3453
3454                 p      = msrpm_offsets[i];
3455                 offset = svm->nested.vmcb_msrpm + (p * 4);
3456
3457                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3458                         return false;
3459
3460                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3461         }
3462
3463         svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3464
3465         return true;
3466 }
3467
3468 static bool nested_vmcb_checks(struct vmcb *vmcb)
3469 {
3470         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3471                 return false;
3472
3473         if (vmcb->control.asid == 0)
3474                 return false;
3475
3476         if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3477             !npt_enabled)
3478                 return false;
3479
3480         return true;
3481 }
3482
3483 static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
3484                                  struct vmcb *nested_vmcb, struct kvm_host_map *map)
3485 {
3486         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
3487                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3488         else
3489                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3490
3491         if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3492                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3493                 nested_svm_init_mmu_context(&svm->vcpu);
3494         }
3495
3496         /* Load the nested guest state */
3497         svm->vmcb->save.es = nested_vmcb->save.es;
3498         svm->vmcb->save.cs = nested_vmcb->save.cs;
3499         svm->vmcb->save.ss = nested_vmcb->save.ss;
3500         svm->vmcb->save.ds = nested_vmcb->save.ds;
3501         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3502         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3503         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
3504         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3505         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3506         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3507         if (npt_enabled) {
3508                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3509                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3510         } else
3511                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3512
3513         /* Guest paging mode is active - reset mmu */
3514         kvm_mmu_reset_context(&svm->vcpu);
3515
3516         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3517         kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax);
3518         kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp);
3519         kvm_rip_write(&svm->vcpu, nested_vmcb->save.rip);
3520
3521         /* In case we don't even reach vcpu_run, the fields are not updated */
3522         svm->vmcb->save.rax = nested_vmcb->save.rax;
3523         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3524         svm->vmcb->save.rip = nested_vmcb->save.rip;
3525         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3526         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3527         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3528
3529         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3530         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
3531
3532         /* cache intercepts */
3533         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3534         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
3535         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3536         svm->nested.intercept            = nested_vmcb->control.intercept;
3537
3538         svm_flush_tlb(&svm->vcpu, true);
3539         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3540         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3541                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3542         else
3543                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3544
3545         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3546                 /* We only want the cr8 intercept bits of the guest */
3547                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3548                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3549         }
3550
3551         /* We don't want to see VMMCALLs from a nested guest */
3552         clr_intercept(svm, INTERCEPT_VMMCALL);
3553
3554         svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset;
3555         svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset;
3556
3557         svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3558         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3559         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3560         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3561         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3562
3563         svm->vmcb->control.pause_filter_count =
3564                 nested_vmcb->control.pause_filter_count;
3565         svm->vmcb->control.pause_filter_thresh =
3566                 nested_vmcb->control.pause_filter_thresh;
3567
3568         kvm_vcpu_unmap(&svm->vcpu, map, true);
3569
3570         /* Enter Guest-Mode */
3571         enter_guest_mode(&svm->vcpu);
3572
3573         /*
3574          * Merge guest and host intercepts - must be called  with vcpu in
3575          * guest-mode to take affect here
3576          */
3577         recalc_intercepts(svm);
3578
3579         svm->nested.vmcb = vmcb_gpa;
3580
3581         enable_gif(svm);
3582
3583         mark_all_dirty(svm->vmcb);
3584 }
3585
3586 static bool nested_svm_vmrun(struct vcpu_svm *svm)
3587 {
3588         int rc;
3589         struct vmcb *nested_vmcb;
3590         struct vmcb *hsave = svm->nested.hsave;
3591         struct vmcb *vmcb = svm->vmcb;
3592         struct kvm_host_map map;
3593         u64 vmcb_gpa;
3594
3595         vmcb_gpa = svm->vmcb->save.rax;
3596
3597         rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
3598         if (rc) {
3599                 if (rc == -EINVAL)
3600                         kvm_inject_gp(&svm->vcpu, 0);
3601                 return false;
3602         }
3603
3604         nested_vmcb = map.hva;
3605
3606         if (!nested_vmcb_checks(nested_vmcb)) {
3607                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
3608                 nested_vmcb->control.exit_code_hi = 0;
3609                 nested_vmcb->control.exit_info_1  = 0;
3610                 nested_vmcb->control.exit_info_2  = 0;
3611
3612                 kvm_vcpu_unmap(&svm->vcpu, &map, true);
3613
3614                 return false;
3615         }
3616
3617         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3618                                nested_vmcb->save.rip,
3619                                nested_vmcb->control.int_ctl,
3620                                nested_vmcb->control.event_inj,
3621                                nested_vmcb->control.nested_ctl);
3622
3623         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3624                                     nested_vmcb->control.intercept_cr >> 16,
3625                                     nested_vmcb->control.intercept_exceptions,
3626                                     nested_vmcb->control.intercept);
3627
3628         /* Clear internal status */
3629         kvm_clear_exception_queue(&svm->vcpu);
3630         kvm_clear_interrupt_queue(&svm->vcpu);
3631
3632         /*
3633          * Save the old vmcb, so we don't need to pick what we save, but can
3634          * restore everything when a VMEXIT occurs
3635          */
3636         hsave->save.es     = vmcb->save.es;
3637         hsave->save.cs     = vmcb->save.cs;
3638         hsave->save.ss     = vmcb->save.ss;
3639         hsave->save.ds     = vmcb->save.ds;
3640         hsave->save.gdtr   = vmcb->save.gdtr;
3641         hsave->save.idtr   = vmcb->save.idtr;
3642         hsave->save.efer   = svm->vcpu.arch.efer;
3643         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
3644         hsave->save.cr4    = svm->vcpu.arch.cr4;
3645         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3646         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
3647         hsave->save.rsp    = vmcb->save.rsp;
3648         hsave->save.rax    = vmcb->save.rax;
3649         if (npt_enabled)
3650                 hsave->save.cr3    = vmcb->save.cr3;
3651         else
3652                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
3653
3654         copy_vmcb_control_area(hsave, vmcb);
3655
3656         enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);
3657
3658         return true;
3659 }
3660
3661 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3662 {
3663         to_vmcb->save.fs = from_vmcb->save.fs;
3664         to_vmcb->save.gs = from_vmcb->save.gs;
3665         to_vmcb->save.tr = from_vmcb->save.tr;
3666         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3667         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3668         to_vmcb->save.star = from_vmcb->save.star;
3669         to_vmcb->save.lstar = from_vmcb->save.lstar;
3670         to_vmcb->save.cstar = from_vmcb->save.cstar;
3671         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3672         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3673         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3674         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3675 }
3676
3677 static int vmload_interception(struct vcpu_svm *svm)
3678 {
3679         struct vmcb *nested_vmcb;
3680         struct kvm_host_map map;
3681         int ret;
3682
3683         if (nested_svm_check_permissions(svm))
3684                 return 1;
3685
3686         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3687         if (ret) {
3688                 if (ret == -EINVAL)
3689                         kvm_inject_gp(&svm->vcpu, 0);
3690                 return 1;
3691         }
3692
3693         nested_vmcb = map.hva;
3694
3695         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3696
3697         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3698         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3699
3700         return ret;
3701 }
3702
3703 static int vmsave_interception(struct vcpu_svm *svm)
3704 {
3705         struct vmcb *nested_vmcb;
3706         struct kvm_host_map map;
3707         int ret;
3708
3709         if (nested_svm_check_permissions(svm))
3710                 return 1;
3711
3712         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3713         if (ret) {
3714                 if (ret == -EINVAL)
3715                         kvm_inject_gp(&svm->vcpu, 0);
3716                 return 1;
3717         }
3718
3719         nested_vmcb = map.hva;
3720
3721         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3722
3723         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3724         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3725
3726         return ret;
3727 }
3728
3729 static int vmrun_interception(struct vcpu_svm *svm)
3730 {
3731         if (nested_svm_check_permissions(svm))
3732                 return 1;
3733
3734         /* Save rip after vmrun instruction */
3735         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3736
3737         if (!nested_svm_vmrun(svm))
3738                 return 1;
3739
3740         if (!nested_svm_vmrun_msrpm(svm))
3741                 goto failed;
3742
3743         return 1;
3744
3745 failed:
3746
3747         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
3748         svm->vmcb->control.exit_code_hi = 0;
3749         svm->vmcb->control.exit_info_1  = 0;
3750         svm->vmcb->control.exit_info_2  = 0;
3751
3752         nested_svm_vmexit(svm);
3753
3754         return 1;
3755 }
3756
3757 static int stgi_interception(struct vcpu_svm *svm)
3758 {
3759         int ret;
3760
3761         if (nested_svm_check_permissions(svm))
3762                 return 1;
3763
3764         /*
3765          * If VGIF is enabled, the STGI intercept is only added to
3766          * detect the opening of the SMI/NMI window; remove it now.
3767          */
3768         if (vgif_enabled(svm))
3769                 clr_intercept(svm, INTERCEPT_STGI);
3770
3771         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3772         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3773
3774         enable_gif(svm);
3775
3776         return ret;
3777 }
3778
3779 static int clgi_interception(struct vcpu_svm *svm)
3780 {
3781         int ret;
3782
3783         if (nested_svm_check_permissions(svm))
3784                 return 1;
3785
3786         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3787
3788         disable_gif(svm);
3789
3790         /* After a CLGI no interrupts should come */
3791         if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3792                 svm_clear_vintr(svm);
3793                 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3794                 mark_dirty(svm->vmcb, VMCB_INTR);
3795         }
3796
3797         return ret;
3798 }
3799
3800 static int invlpga_interception(struct vcpu_svm *svm)
3801 {
3802         struct kvm_vcpu *vcpu = &svm->vcpu;
3803
3804         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),
3805                           kvm_rax_read(&svm->vcpu));
3806
3807         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3808         kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
3809
3810         return kvm_skip_emulated_instruction(&svm->vcpu);
3811 }
3812
3813 static int skinit_interception(struct vcpu_svm *svm)
3814 {
3815         trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));
3816
3817         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3818         return 1;
3819 }
3820
3821 static int wbinvd_interception(struct vcpu_svm *svm)
3822 {
3823         return kvm_emulate_wbinvd(&svm->vcpu);
3824 }
3825
3826 static int xsetbv_interception(struct vcpu_svm *svm)
3827 {
3828         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
3829         u32 index = kvm_rcx_read(&svm->vcpu);
3830
3831         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3832                 return kvm_skip_emulated_instruction(&svm->vcpu);
3833         }
3834
3835         return 1;
3836 }
3837
3838 static int task_switch_interception(struct vcpu_svm *svm)
3839 {
3840         u16 tss_selector;
3841         int reason;
3842         int int_type = svm->vmcb->control.exit_int_info &
3843                 SVM_EXITINTINFO_TYPE_MASK;
3844         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3845         uint32_t type =
3846                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3847         uint32_t idt_v =
3848                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3849         bool has_error_code = false;
3850         u32 error_code = 0;
3851
3852         tss_selector = (u16)svm->vmcb->control.exit_info_1;
3853
3854         if (svm->vmcb->control.exit_info_2 &
3855             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3856                 reason = TASK_SWITCH_IRET;
3857         else if (svm->vmcb->control.exit_info_2 &
3858                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3859                 reason = TASK_SWITCH_JMP;
3860         else if (idt_v)
3861                 reason = TASK_SWITCH_GATE;
3862         else
3863                 reason = TASK_SWITCH_CALL;
3864
3865         if (reason == TASK_SWITCH_GATE) {
3866                 switch (type) {
3867                 case SVM_EXITINTINFO_TYPE_NMI:
3868                         svm->vcpu.arch.nmi_injected = false;
3869                         break;
3870                 case SVM_EXITINTINFO_TYPE_EXEPT:
3871                         if (svm->vmcb->control.exit_info_2 &
3872                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3873                                 has_error_code = true;
3874                                 error_code =
3875                                         (u32)svm->vmcb->control.exit_info_2;
3876                         }
3877                         kvm_clear_exception_queue(&svm->vcpu);
3878                         break;
3879                 case SVM_EXITINTINFO_TYPE_INTR:
3880                         kvm_clear_interrupt_queue(&svm->vcpu);
3881                         break;
3882                 default:
3883                         break;
3884                 }
3885         }
3886
3887         if (reason != TASK_SWITCH_GATE ||
3888             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3889             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3890              (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
3891                 if (skip_emulated_instruction(&svm->vcpu) != EMULATE_DONE)
3892                         goto fail;
3893         }
3894
3895         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3896                 int_vec = -1;
3897
3898         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3899                                 has_error_code, error_code) == EMULATE_FAIL)
3900                 goto fail;
3901
3902         return 1;
3903
3904 fail:
3905         svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3906         svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3907         svm->vcpu.run->internal.ndata = 0;
3908         return 0;
3909 }
3910
3911 static int cpuid_interception(struct vcpu_svm *svm)
3912 {
3913         return kvm_emulate_cpuid(&svm->vcpu);
3914 }
3915
3916 static int iret_interception(struct vcpu_svm *svm)
3917 {
3918         ++svm->vcpu.stat.nmi_window_exits;
3919         clr_intercept(svm, INTERCEPT_IRET);
3920         svm->vcpu.arch.hflags |= HF_IRET_MASK;
3921         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3922         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3923         return 1;
3924 }
3925
3926 static int invlpg_interception(struct vcpu_svm *svm)
3927 {
3928         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3929                 return kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3930
3931         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3932         return kvm_skip_emulated_instruction(&svm->vcpu);
3933 }
3934
3935 static int emulate_on_interception(struct vcpu_svm *svm)
3936 {
3937         return kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3938 }
3939
3940 static int rsm_interception(struct vcpu_svm *svm)
3941 {
3942         return kvm_emulate_instruction_from_buffer(&svm->vcpu,
3943                                         rsm_ins_bytes, 2) == EMULATE_DONE;
3944 }
3945
3946 static int rdpmc_interception(struct vcpu_svm *svm)
3947 {
3948         int err;
3949
3950         if (!nrips)
3951                 return emulate_on_interception(svm);
3952
3953         err = kvm_rdpmc(&svm->vcpu);
3954         return kvm_complete_insn_gp(&svm->vcpu, err);
3955 }
3956
3957 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3958                                             unsigned long val)
3959 {
3960         unsigned long cr0 = svm->vcpu.arch.cr0;
3961         bool ret = false;
3962         u64 intercept;
3963
3964         intercept = svm->nested.intercept;
3965
3966         if (!is_guest_mode(&svm->vcpu) ||
3967             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3968                 return false;
3969
3970         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3971         val &= ~SVM_CR0_SELECTIVE_MASK;
3972
3973         if (cr0 ^ val) {
3974                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3975                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3976         }
3977
3978         return ret;
3979 }
3980
3981 #define CR_VALID (1ULL << 63)
3982
3983 static int cr_interception(struct vcpu_svm *svm)
3984 {
3985         int reg, cr;
3986         unsigned long val;
3987         int err;
3988
3989         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3990                 return emulate_on_interception(svm);
3991
3992         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3993                 return emulate_on_interception(svm);
3994
3995         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3996         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3997                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3998         else
3999                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
4000
4001         err = 0;
4002         if (cr >= 16) { /* mov to cr */
4003                 cr -= 16;
4004                 val = kvm_register_read(&svm->vcpu, reg);
4005                 switch (cr) {
4006                 case 0:
4007                         if (!check_selective_cr0_intercepted(svm, val))
4008                                 err = kvm_set_cr0(&svm->vcpu, val);
4009                         else
4010                                 return 1;
4011
4012                         break;
4013                 case 3:
4014                         err = kvm_set_cr3(&svm->vcpu, val);
4015                         break;
4016                 case 4:
4017                         err = kvm_set_cr4(&svm->vcpu, val);
4018                         break;
4019                 case 8:
4020                         err = kvm_set_cr8(&svm->vcpu, val);
4021                         break;
4022                 default:
4023                         WARN(1, "unhandled write to CR%d", cr);
4024                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4025                         return 1;
4026                 }
4027         } else { /* mov from cr */
4028                 switch (cr) {
4029                 case 0:
4030                         val = kvm_read_cr0(&svm->vcpu);
4031                         break;
4032                 case 2:
4033                         val = svm->vcpu.arch.cr2;
4034                         break;
4035                 case 3:
4036                         val = kvm_read_cr3(&svm->vcpu);
4037                         break;
4038                 case 4:
4039                         val = kvm_read_cr4(&svm->vcpu);
4040                         break;
4041                 case 8:
4042                         val = kvm_get_cr8(&svm->vcpu);
4043                         break;
4044                 default:
4045                         WARN(1, "unhandled read from CR%d", cr);
4046                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4047                         return 1;
4048                 }
4049                 kvm_register_write(&svm->vcpu, reg, val);
4050         }
4051         return kvm_complete_insn_gp(&svm->vcpu, err);
4052 }
4053
4054 static int dr_interception(struct vcpu_svm *svm)
4055 {
4056         int reg, dr;
4057         unsigned long val;
4058
4059         if (svm->vcpu.guest_debug == 0) {
4060                 /*
4061                  * No more DR vmexits; force a reload of the debug registers
4062                  * and reenter on this instruction.  The next vmexit will
4063                  * retrieve the full state of the debug registers.
4064                  */
4065                 clr_dr_intercepts(svm);
4066                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
4067                 return 1;
4068         }
4069
4070         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
4071                 return emulate_on_interception(svm);
4072
4073         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4074         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
4075
4076         if (dr >= 16) { /* mov to DRn */
4077                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
4078                         return 1;
4079                 val = kvm_register_read(&svm->vcpu, reg);
4080                 kvm_set_dr(&svm->vcpu, dr - 16, val);
4081         } else {
4082                 if (!kvm_require_dr(&svm->vcpu, dr))
4083                         return 1;
4084                 kvm_get_dr(&svm->vcpu, dr, &val);
4085                 kvm_register_write(&svm->vcpu, reg, val);
4086         }
4087
4088         return kvm_skip_emulated_instruction(&svm->vcpu);
4089 }
4090
4091 static int cr8_write_interception(struct vcpu_svm *svm)
4092 {
4093         struct kvm_run *kvm_run = svm->vcpu.run;
4094         int r;
4095
4096         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
4097         /* instruction emulation calls kvm_set_cr8() */
4098         r = cr_interception(svm);
4099         if (lapic_in_kernel(&svm->vcpu))
4100                 return r;
4101         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
4102                 return r;
4103         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
4104         return 0;
4105 }
4106
4107 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
4108 {
4109         msr->data = 0;
4110
4111         switch (msr->index) {
4112         case MSR_F10H_DECFG:
4113                 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
4114                         msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
4115                 break;
4116         default:
4117                 return 1;
4118         }
4119
4120         return 0;
4121 }
4122
4123 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4124 {
4125         struct vcpu_svm *svm = to_svm(vcpu);
4126
4127         switch (msr_info->index) {
4128         case MSR_STAR:
4129                 msr_info->data = svm->vmcb->save.star;
4130                 break;
4131 #ifdef CONFIG_X86_64
4132         case MSR_LSTAR:
4133                 msr_info->data = svm->vmcb->save.lstar;
4134                 break;
4135         case MSR_CSTAR:
4136                 msr_info->data = svm->vmcb->save.cstar;
4137                 break;
4138         case MSR_KERNEL_GS_BASE:
4139                 msr_info->data = svm->vmcb->save.kernel_gs_base;
4140                 break;
4141         case MSR_SYSCALL_MASK:
4142                 msr_info->data = svm->vmcb->save.sfmask;
4143                 break;
4144 #endif
4145         case MSR_IA32_SYSENTER_CS:
4146                 msr_info->data = svm->vmcb->save.sysenter_cs;
4147                 break;
4148         case MSR_IA32_SYSENTER_EIP:
4149                 msr_info->data = svm->sysenter_eip;
4150                 break;
4151         case MSR_IA32_SYSENTER_ESP:
4152                 msr_info->data = svm->sysenter_esp;
4153                 break;
4154         case MSR_TSC_AUX:
4155                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4156                         return 1;
4157                 msr_info->data = svm->tsc_aux;
4158                 break;
4159         /*
4160          * Nobody will change the following 5 values in the VMCB so we can
4161          * safely return them on rdmsr. They will always be 0 until LBRV is
4162          * implemented.
4163          */
4164         case MSR_IA32_DEBUGCTLMSR:
4165                 msr_info->data = svm->vmcb->save.dbgctl;
4166                 break;
4167         case MSR_IA32_LASTBRANCHFROMIP:
4168                 msr_info->data = svm->vmcb->save.br_from;
4169                 break;
4170         case MSR_IA32_LASTBRANCHTOIP:
4171                 msr_info->data = svm->vmcb->save.br_to;
4172                 break;
4173         case MSR_IA32_LASTINTFROMIP:
4174                 msr_info->data = svm->vmcb->save.last_excp_from;
4175                 break;
4176         case MSR_IA32_LASTINTTOIP:
4177                 msr_info->data = svm->vmcb->save.last_excp_to;
4178                 break;
4179         case MSR_VM_HSAVE_PA:
4180                 msr_info->data = svm->nested.hsave_msr;
4181                 break;
4182         case MSR_VM_CR:
4183                 msr_info->data = svm->nested.vm_cr_msr;
4184                 break;
4185         case MSR_IA32_SPEC_CTRL:
4186                 if (!msr_info->host_initiated &&
4187                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4188                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4189                         return 1;
4190
4191                 msr_info->data = svm->spec_ctrl;
4192                 break;
4193         case MSR_AMD64_VIRT_SPEC_CTRL:
4194                 if (!msr_info->host_initiated &&
4195                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4196                         return 1;
4197
4198                 msr_info->data = svm->virt_spec_ctrl;
4199                 break;
4200         case MSR_F15H_IC_CFG: {
4201
4202                 int family, model;
4203
4204                 family = guest_cpuid_family(vcpu);
4205                 model  = guest_cpuid_model(vcpu);
4206
4207                 if (family < 0 || model < 0)
4208                         return kvm_get_msr_common(vcpu, msr_info);
4209
4210                 msr_info->data = 0;
4211
4212                 if (family == 0x15 &&
4213                     (model >= 0x2 && model < 0x20))
4214                         msr_info->data = 0x1E;
4215                 }
4216                 break;
4217         case MSR_F10H_DECFG:
4218                 msr_info->data = svm->msr_decfg;
4219                 break;
4220         default:
4221                 return kvm_get_msr_common(vcpu, msr_info);
4222         }
4223         return 0;
4224 }
4225
4226 static int rdmsr_interception(struct vcpu_svm *svm)
4227 {
4228         u32 ecx = kvm_rcx_read(&svm->vcpu);
4229         struct msr_data msr_info;
4230
4231         msr_info.index = ecx;
4232         msr_info.host_initiated = false;
4233         if (svm_get_msr(&svm->vcpu, &msr_info)) {
4234                 trace_kvm_msr_read_ex(ecx);
4235                 kvm_inject_gp(&svm->vcpu, 0);
4236                 return 1;
4237         } else {
4238                 trace_kvm_msr_read(ecx, msr_info.data);
4239
4240                 kvm_rax_write(&svm->vcpu, msr_info.data & 0xffffffff);
4241                 kvm_rdx_write(&svm->vcpu, msr_info.data >> 32);
4242                 return kvm_skip_emulated_instruction(&svm->vcpu);
4243         }
4244 }
4245
4246 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
4247 {
4248         struct vcpu_svm *svm = to_svm(vcpu);
4249         int svm_dis, chg_mask;
4250
4251         if (data & ~SVM_VM_CR_VALID_MASK)
4252                 return 1;
4253
4254         chg_mask = SVM_VM_CR_VALID_MASK;
4255
4256         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
4257                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
4258
4259         svm->nested.vm_cr_msr &= ~chg_mask;
4260         svm->nested.vm_cr_msr |= (data & chg_mask);
4261
4262         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
4263
4264         /* check for svm_disable while efer.svme is set */
4265         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
4266                 return 1;
4267
4268         return 0;
4269 }
4270
4271 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
4272 {
4273         struct vcpu_svm *svm = to_svm(vcpu);
4274
4275         u32 ecx = msr->index;
4276         u64 data = msr->data;
4277         switch (ecx) {
4278         case MSR_IA32_CR_PAT:
4279                 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
4280                         return 1;
4281                 vcpu->arch.pat = data;
4282                 svm->vmcb->save.g_pat = data;
4283                 mark_dirty(svm->vmcb, VMCB_NPT);
4284                 break;
4285         case MSR_IA32_SPEC_CTRL:
4286                 if (!msr->host_initiated &&
4287                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4288                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4289                         return 1;
4290
4291                 /* The STIBP bit doesn't fault even if it's not advertised */
4292                 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
4293                         return 1;
4294
4295                 svm->spec_ctrl = data;
4296
4297                 if (!data)
4298                         break;
4299
4300                 /*
4301                  * For non-nested:
4302                  * When it's written (to non-zero) for the first time, pass
4303                  * it through.
4304                  *
4305                  * For nested:
4306                  * The handling of the MSR bitmap for L2 guests is done in
4307                  * nested_svm_vmrun_msrpm.
4308                  * We update the L1 MSR bit as well since it will end up
4309                  * touching the MSR anyway now.
4310                  */
4311                 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
4312                 break;
4313         case MSR_IA32_PRED_CMD:
4314                 if (!msr->host_initiated &&
4315                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
4316                         return 1;
4317
4318                 if (data & ~PRED_CMD_IBPB)
4319                         return 1;
4320
4321                 if (!data)
4322                         break;
4323
4324                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4325                 if (is_guest_mode(vcpu))
4326                         break;
4327                 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
4328                 break;
4329         case MSR_AMD64_VIRT_SPEC_CTRL:
4330                 if (!msr->host_initiated &&
4331                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4332                         return 1;
4333
4334                 if (data & ~SPEC_CTRL_SSBD)
4335                         return 1;
4336
4337                 svm->virt_spec_ctrl = data;
4338                 break;
4339         case MSR_STAR:
4340                 svm->vmcb->save.star = data;
4341                 break;
4342 #ifdef CONFIG_X86_64
4343         case MSR_LSTAR:
4344                 svm->vmcb->save.lstar = data;
4345                 break;
4346         case MSR_CSTAR:
4347                 svm->vmcb->save.cstar = data;
4348                 break;
4349         case MSR_KERNEL_GS_BASE:
4350                 svm->vmcb->save.kernel_gs_base = data;
4351                 break;
4352         case MSR_SYSCALL_MASK:
4353                 svm->vmcb->save.sfmask = data;
4354                 break;
4355 #endif
4356         case MSR_IA32_SYSENTER_CS:
4357                 svm->vmcb->save.sysenter_cs = data;
4358                 break;
4359         case MSR_IA32_SYSENTER_EIP:
4360                 svm->sysenter_eip = data;
4361                 svm->vmcb->save.sysenter_eip = data;
4362                 break;
4363         case MSR_IA32_SYSENTER_ESP:
4364                 svm->sysenter_esp = data;
4365                 svm->vmcb->save.sysenter_esp = data;
4366                 break;
4367         case MSR_TSC_AUX:
4368                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4369                         return 1;
4370
4371                 /*
4372                  * This is rare, so we update the MSR here instead of using
4373                  * direct_access_msrs.  Doing that would require a rdmsr in
4374                  * svm_vcpu_put.
4375                  */
4376                 svm->tsc_aux = data;
4377                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
4378                 break;
4379         case MSR_IA32_DEBUGCTLMSR:
4380                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
4381                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
4382                                     __func__, data);
4383                         break;
4384                 }
4385                 if (data & DEBUGCTL_RESERVED_BITS)
4386                         return 1;
4387
4388                 svm->vmcb->save.dbgctl = data;
4389                 mark_dirty(svm->vmcb, VMCB_LBR);
4390                 if (data & (1ULL<<0))
4391                         svm_enable_lbrv(svm);
4392                 else
4393                         svm_disable_lbrv(svm);
4394                 break;
4395         case MSR_VM_HSAVE_PA:
4396                 svm->nested.hsave_msr = data;
4397                 break;
4398         case MSR_VM_CR:
4399                 return svm_set_vm_cr(vcpu, data);
4400         case MSR_VM_IGNNE:
4401                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
4402                 break;
4403         case MSR_F10H_DECFG: {
4404                 struct kvm_msr_entry msr_entry;
4405
4406                 msr_entry.index = msr->index;
4407                 if (svm_get_msr_feature(&msr_entry))
4408                         return 1;
4409
4410                 /* Check the supported bits */
4411                 if (data & ~msr_entry.data)
4412                         return 1;
4413
4414                 /* Don't allow the guest to change a bit, #GP */
4415                 if (!msr->host_initiated && (data ^ msr_entry.data))
4416                         return 1;
4417
4418                 svm->msr_decfg = data;
4419                 break;
4420         }
4421         case MSR_IA32_APICBASE:
4422                 if (kvm_vcpu_apicv_active(vcpu))
4423                         avic_update_vapic_bar(to_svm(vcpu), data);
4424                 /* Fall through */
4425         default:
4426                 return kvm_set_msr_common(vcpu, msr);
4427         }
4428         return 0;
4429 }
4430
4431 static int wrmsr_interception(struct vcpu_svm *svm)
4432 {
4433         struct msr_data msr;
4434         u32 ecx = kvm_rcx_read(&svm->vcpu);
4435         u64 data = kvm_read_edx_eax(&svm->vcpu);
4436
4437         msr.data = data;
4438         msr.index = ecx;
4439         msr.host_initiated = false;
4440
4441         if (kvm_set_msr(&svm->vcpu, &msr)) {
4442                 trace_kvm_msr_write_ex(ecx, data);
4443                 kvm_inject_gp(&svm->vcpu, 0);
4444                 return 1;
4445         } else {
4446                 trace_kvm_msr_write(ecx, data);
4447                 return kvm_skip_emulated_instruction(&svm->vcpu);
4448         }
4449 }
4450
4451 static int msr_interception(struct vcpu_svm *svm)
4452 {
4453         if (svm->vmcb->control.exit_info_1)
4454                 return wrmsr_interception(svm);
4455         else
4456                 return rdmsr_interception(svm);
4457 }
4458
4459 static int interrupt_window_interception(struct vcpu_svm *svm)
4460 {
4461         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4462         svm_clear_vintr(svm);
4463         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
4464         mark_dirty(svm->vmcb, VMCB_INTR);
4465         ++svm->vcpu.stat.irq_window_exits;
4466         return 1;
4467 }
4468
4469 static int pause_interception(struct vcpu_svm *svm)
4470 {
4471         struct kvm_vcpu *vcpu = &svm->vcpu;
4472         bool in_kernel = (svm_get_cpl(vcpu) == 0);
4473
4474         if (pause_filter_thresh)
4475                 grow_ple_window(vcpu);
4476
4477         kvm_vcpu_on_spin(vcpu, in_kernel);
4478         return 1;
4479 }
4480
4481 static int nop_interception(struct vcpu_svm *svm)
4482 {
4483         return kvm_skip_emulated_instruction(&(svm->vcpu));
4484 }
4485
4486 static int monitor_interception(struct vcpu_svm *svm)
4487 {
4488         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
4489         return nop_interception(svm);
4490 }
4491
4492 static int mwait_interception(struct vcpu_svm *svm)
4493 {
4494         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
4495         return nop_interception(svm);
4496 }
4497
4498 enum avic_ipi_failure_cause {
4499         AVIC_IPI_FAILURE_INVALID_INT_TYPE,
4500         AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
4501         AVIC_IPI_FAILURE_INVALID_TARGET,
4502         AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
4503 };
4504
4505 static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
4506 {
4507         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
4508         u32 icrl = svm->vmcb->control.exit_info_1;
4509         u32 id = svm->vmcb->control.exit_info_2 >> 32;
4510         u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
4511         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4512
4513         trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
4514
4515         switch (id) {
4516         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
4517                 /*
4518                  * AVIC hardware handles the generation of
4519                  * IPIs when the specified Message Type is Fixed
4520                  * (also known as fixed delivery mode) and
4521                  * the Trigger Mode is edge-triggered. The hardware
4522                  * also supports self and broadcast delivery modes
4523                  * specified via the Destination Shorthand(DSH)
4524                  * field of the ICRL. Logical and physical APIC ID
4525                  * formats are supported. All other IPI types cause
4526                  * a #VMEXIT, which needs to emulated.
4527                  */
4528                 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
4529                 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
4530                 break;
4531         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
4532                 int i;
4533                 struct kvm_vcpu *vcpu;
4534                 struct kvm *kvm = svm->vcpu.kvm;
4535                 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4536
4537                 /*
4538                  * At this point, we expect that the AVIC HW has already
4539                  * set the appropriate IRR bits on the valid target
4540                  * vcpus. So, we just need to kick the appropriate vcpu.
4541                  */
4542                 kvm_for_each_vcpu(i, vcpu, kvm) {
4543                         bool m = kvm_apic_match_dest(vcpu, apic,
4544                                                      icrl & KVM_APIC_SHORT_MASK,
4545                                                      GET_APIC_DEST_FIELD(icrh),
4546                                                      icrl & KVM_APIC_DEST_MASK);
4547
4548                         if (m && !avic_vcpu_is_running(vcpu))
4549                                 kvm_vcpu_wake_up(vcpu);
4550                 }
4551                 break;
4552         }
4553         case AVIC_IPI_FAILURE_INVALID_TARGET:
4554                 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
4555                           index, svm->vcpu.vcpu_id, icrh, icrl);
4556                 break;
4557         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
4558                 WARN_ONCE(1, "Invalid backing page\n");
4559                 break;
4560         default:
4561                 pr_err("Unknown IPI interception\n");
4562         }
4563
4564         return 1;
4565 }
4566
4567 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
4568 {
4569         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
4570         int index;
4571         u32 *logical_apic_id_table;
4572         int dlid = GET_APIC_LOGICAL_ID(ldr);
4573
4574         if (!dlid)
4575                 return NULL;
4576
4577         if (flat) { /* flat */
4578                 index = ffs(dlid) - 1;
4579                 if (index > 7)
4580                         return NULL;
4581         } else { /* cluster */
4582                 int cluster = (dlid & 0xf0) >> 4;
4583                 int apic = ffs(dlid & 0x0f) - 1;
4584
4585                 if ((apic < 0) || (apic > 7) ||
4586                     (cluster >= 0xf))
4587                         return NULL;
4588                 index = (cluster << 2) + apic;
4589         }
4590
4591         logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
4592
4593         return &logical_apic_id_table[index];
4594 }
4595
4596 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
4597 {
4598         bool flat;
4599         u32 *entry, new_entry;
4600
4601         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
4602         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
4603         if (!entry)
4604                 return -EINVAL;
4605
4606         new_entry = READ_ONCE(*entry);
4607         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
4608         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
4609         new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4610         WRITE_ONCE(*entry, new_entry);
4611
4612         return 0;
4613 }
4614
4615 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
4616 {
4617         struct vcpu_svm *svm = to_svm(vcpu);
4618         bool flat = svm->dfr_reg == APIC_DFR_FLAT;
4619         u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
4620
4621         if (entry)
4622                 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
4623 }
4624
4625 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
4626 {
4627         int ret = 0;
4628         struct vcpu_svm *svm = to_svm(vcpu);
4629         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
4630
4631         if (ldr == svm->ldr_reg)
4632                 return 0;
4633
4634         avic_invalidate_logical_id_entry(vcpu);
4635
4636         if (ldr)
4637                 ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr);
4638
4639         if (!ret)
4640                 svm->ldr_reg = ldr;
4641
4642         return ret;
4643 }
4644
4645 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
4646 {
4647         u64 *old, *new;
4648         struct vcpu_svm *svm = to_svm(vcpu);
4649         u32 apic_id_reg = kvm_lapic_get_reg(vcpu->arch.apic, APIC_ID);
4650         u32 id = (apic_id_reg >> 24) & 0xff;
4651
4652         if (vcpu->vcpu_id == id)
4653                 return 0;
4654
4655         old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
4656         new = avic_get_physical_id_entry(vcpu, id);
4657         if (!new || !old)
4658                 return 1;
4659
4660         /* We need to move physical_id_entry to new offset */
4661         *new = *old;
4662         *old = 0ULL;
4663         to_svm(vcpu)->avic_physical_id_cache = new;
4664
4665         /*
4666          * Also update the guest physical APIC ID in the logical
4667          * APIC ID table entry if already setup the LDR.
4668          */
4669         if (svm->ldr_reg)
4670                 avic_handle_ldr_update(vcpu);
4671
4672         return 0;
4673 }
4674
4675 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
4676 {
4677         struct vcpu_svm *svm = to_svm(vcpu);
4678         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
4679
4680         if (svm->dfr_reg == dfr)
4681                 return;
4682
4683         avic_invalidate_logical_id_entry(vcpu);
4684         svm->dfr_reg = dfr;
4685 }
4686
4687 static int avic_unaccel_trap_write(struct vcpu_svm *svm)
4688 {
4689         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4690         u32 offset = svm->vmcb->control.exit_info_1 &
4691                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4692
4693         switch (offset) {
4694         case APIC_ID:
4695                 if (avic_handle_apic_id_update(&svm->vcpu))
4696                         return 0;
4697                 break;
4698         case APIC_LDR:
4699                 if (avic_handle_ldr_update(&svm->vcpu))
4700                         return 0;
4701                 break;
4702         case APIC_DFR:
4703                 avic_handle_dfr_update(&svm->vcpu);
4704                 break;
4705         default:
4706                 break;
4707         }
4708
4709         kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4710
4711         return 1;
4712 }
4713
4714 static bool is_avic_unaccelerated_access_trap(u32 offset)
4715 {
4716         bool ret = false;
4717
4718         switch (offset) {
4719         case APIC_ID:
4720         case APIC_EOI:
4721         case APIC_RRR:
4722         case APIC_LDR:
4723         case APIC_DFR:
4724         case APIC_SPIV:
4725         case APIC_ESR:
4726         case APIC_ICR:
4727         case APIC_LVTT:
4728         case APIC_LVTTHMR:
4729         case APIC_LVTPC:
4730         case APIC_LVT0:
4731         case APIC_LVT1:
4732         case APIC_LVTERR:
4733         case APIC_TMICT:
4734         case APIC_TDCR:
4735                 ret = true;
4736                 break;
4737         default:
4738                 break;
4739         }
4740         return ret;
4741 }
4742
4743 static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4744 {
4745         int ret = 0;
4746         u32 offset = svm->vmcb->control.exit_info_1 &
4747                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4748         u32 vector = svm->vmcb->control.exit_info_2 &
4749                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4750         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4751                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
4752         bool trap = is_avic_unaccelerated_access_trap(offset);
4753
4754         trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4755                                             trap, write, vector);
4756         if (trap) {
4757                 /* Handling Trap */
4758                 WARN_ONCE(!write, "svm: Handling trap read.\n");
4759                 ret = avic_unaccel_trap_write(svm);
4760         } else {
4761                 /* Handling Fault */
4762                 ret = (kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
4763         }
4764
4765         return ret;
4766 }
4767
4768 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4769         [SVM_EXIT_READ_CR0]                     = cr_interception,
4770         [SVM_EXIT_READ_CR3]                     = cr_interception,
4771         [SVM_EXIT_READ_CR4]                     = cr_interception,
4772         [SVM_EXIT_READ_CR8]                     = cr_interception,
4773         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
4774         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
4775         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
4776         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
4777         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
4778         [SVM_EXIT_READ_DR0]                     = dr_interception,
4779         [SVM_EXIT_READ_DR1]                     = dr_interception,
4780         [SVM_EXIT_READ_DR2]                     = dr_interception,
4781         [SVM_EXIT_READ_DR3]                     = dr_interception,
4782         [SVM_EXIT_READ_DR4]                     = dr_interception,
4783         [SVM_EXIT_READ_DR5]                     = dr_interception,
4784         [SVM_EXIT_READ_DR6]                     = dr_interception,
4785         [SVM_EXIT_READ_DR7]                     = dr_interception,
4786         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
4787         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
4788         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
4789         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
4790         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
4791         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
4792         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
4793         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
4794         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
4795         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
4796         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
4797         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
4798         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
4799         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
4800         [SVM_EXIT_EXCP_BASE + GP_VECTOR]        = gp_interception,
4801         [SVM_EXIT_INTR]                         = intr_interception,
4802         [SVM_EXIT_NMI]                          = nmi_interception,
4803         [SVM_EXIT_SMI]                          = nop_on_interception,
4804         [SVM_EXIT_INIT]                         = nop_on_interception,
4805         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
4806         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
4807         [SVM_EXIT_CPUID]                        = cpuid_interception,
4808         [SVM_EXIT_IRET]                         = iret_interception,
4809         [SVM_EXIT_INVD]                         = emulate_on_interception,
4810         [SVM_EXIT_PAUSE]                        = pause_interception,
4811         [SVM_EXIT_HLT]                          = halt_interception,
4812         [SVM_EXIT_INVLPG]                       = invlpg_interception,
4813         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
4814         [SVM_EXIT_IOIO]                         = io_interception,
4815         [SVM_EXIT_MSR]                          = msr_interception,
4816         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
4817         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
4818         [SVM_EXIT_VMRUN]                        = vmrun_interception,
4819         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
4820         [SVM_EXIT_VMLOAD]                       = vmload_interception,
4821         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
4822         [SVM_EXIT_STGI]                         = stgi_interception,
4823         [SVM_EXIT_CLGI]                         = clgi_interception,
4824         [SVM_EXIT_SKINIT]                       = skinit_interception,
4825         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
4826         [SVM_EXIT_MONITOR]                      = monitor_interception,
4827         [SVM_EXIT_MWAIT]                        = mwait_interception,
4828         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
4829         [SVM_EXIT_NPF]                          = npf_interception,
4830         [SVM_EXIT_RSM]                          = rsm_interception,
4831         [SVM_EXIT_AVIC_INCOMPLETE_IPI]          = avic_incomplete_ipi_interception,
4832         [SVM_EXIT_AVIC_UNACCELERATED_ACCESS]    = avic_unaccelerated_access_interception,
4833 };
4834
4835 static void dump_vmcb(struct kvm_vcpu *vcpu)
4836 {
4837         struct vcpu_svm *svm = to_svm(vcpu);
4838         struct vmcb_control_area *control = &svm->vmcb->control;
4839         struct vmcb_save_area *save = &svm->vmcb->save;
4840
4841         if (!dump_invalid_vmcb) {
4842                 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
4843                 return;
4844         }
4845
4846         pr_err("VMCB Control Area:\n");
4847         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4848         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4849         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4850         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4851         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4852         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4853         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
4854         pr_err("%-20s%d\n", "pause filter threshold:",
4855                control->pause_filter_thresh);
4856         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4857         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4858         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4859         pr_err("%-20s%d\n", "asid:", control->asid);
4860         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4861         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4862         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4863         pr_err("%-20s%08x\n", "int_state:", control->int_state);
4864         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4865         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4866         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4867         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4868         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4869         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4870         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4871         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4872         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4873         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4874         pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4875         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4876         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4877         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4878         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4879         pr_err("VMCB State Save Area:\n");
4880         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4881                "es:",
4882                save->es.selector, save->es.attrib,
4883                save->es.limit, save->es.base);
4884         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4885                "cs:",
4886                save->cs.selector, save->cs.attrib,
4887                save->cs.limit, save->cs.base);
4888         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4889                "ss:",
4890                save->ss.selector, save->ss.attrib,
4891                save->ss.limit, save->ss.base);
4892         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4893                "ds:",
4894                save->ds.selector, save->ds.attrib,
4895                save->ds.limit, save->ds.base);
4896         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4897                "fs:",
4898                save->fs.selector, save->fs.attrib,
4899                save->fs.limit, save->fs.base);
4900         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4901                "gs:",
4902                save->gs.selector, save->gs.attrib,
4903                save->gs.limit, save->gs.base);
4904         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4905                "gdtr:",
4906                save->gdtr.selector, save->gdtr.attrib,
4907                save->gdtr.limit, save->gdtr.base);
4908         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4909                "ldtr:",
4910                save->ldtr.selector, save->ldtr.attrib,
4911                save->ldtr.limit, save->ldtr.base);
4912         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4913                "idtr:",
4914                save->idtr.selector, save->idtr.attrib,
4915                save->idtr.limit, save->idtr.base);
4916         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4917                "tr:",
4918                save->tr.selector, save->tr.attrib,
4919                save->tr.limit, save->tr.base);
4920         pr_err("cpl:            %d                efer:         %016llx\n",
4921                 save->cpl, save->efer);
4922         pr_err("%-15s %016llx %-13s %016llx\n",
4923                "cr0:", save->cr0, "cr2:", save->cr2);
4924         pr_err("%-15s %016llx %-13s %016llx\n",
4925                "cr3:", save->cr3, "cr4:", save->cr4);
4926         pr_err("%-15s %016llx %-13s %016llx\n",
4927                "dr6:", save->dr6, "dr7:", save->dr7);
4928         pr_err("%-15s %016llx %-13s %016llx\n",
4929                "rip:", save->rip, "rflags:", save->rflags);
4930         pr_err("%-15s %016llx %-13s %016llx\n",
4931                "rsp:", save->rsp, "rax:", save->rax);
4932         pr_err("%-15s %016llx %-13s %016llx\n",
4933                "star:", save->star, "lstar:", save->lstar);
4934         pr_err("%-15s %016llx %-13s %016llx\n",
4935                "cstar:", save->cstar, "sfmask:", save->sfmask);
4936         pr_err("%-15s %016llx %-13s %016llx\n",
4937                "kernel_gs_base:", save->kernel_gs_base,
4938                "sysenter_cs:", save->sysenter_cs);
4939         pr_err("%-15s %016llx %-13s %016llx\n",
4940                "sysenter_esp:", save->sysenter_esp,
4941                "sysenter_eip:", save->sysenter_eip);
4942         pr_err("%-15s %016llx %-13s %016llx\n",
4943                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4944         pr_err("%-15s %016llx %-13s %016llx\n",
4945                "br_from:", save->br_from, "br_to:", save->br_to);
4946         pr_err("%-15s %016llx %-13s %016llx\n",
4947                "excp_from:", save->last_excp_from,
4948                "excp_to:", save->last_excp_to);
4949 }
4950
4951 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4952 {
4953         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4954
4955         *info1 = control->exit_info_1;
4956         *info2 = control->exit_info_2;
4957 }
4958
4959 static int handle_exit(struct kvm_vcpu *vcpu)
4960 {
4961         struct vcpu_svm *svm = to_svm(vcpu);
4962         struct kvm_run *kvm_run = vcpu->run;
4963         u32 exit_code = svm->vmcb->control.exit_code;
4964
4965         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4966
4967         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4968                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4969         if (npt_enabled)
4970                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
4971
4972         if (unlikely(svm->nested.exit_required)) {
4973                 nested_svm_vmexit(svm);
4974                 svm->nested.exit_required = false;
4975
4976                 return 1;
4977         }
4978
4979         if (is_guest_mode(vcpu)) {
4980                 int vmexit;
4981
4982                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
4983                                         svm->vmcb->control.exit_info_1,
4984                                         svm->vmcb->control.exit_info_2,
4985                                         svm->vmcb->control.exit_int_info,
4986                                         svm->vmcb->control.exit_int_info_err,
4987                                         KVM_ISA_SVM);
4988
4989                 vmexit = nested_svm_exit_special(svm);
4990
4991                 if (vmexit == NESTED_EXIT_CONTINUE)
4992                         vmexit = nested_svm_exit_handled(svm);
4993
4994                 if (vmexit == NESTED_EXIT_DONE)
4995                         return 1;
4996         }
4997
4998         svm_complete_interrupts(svm);
4999
5000         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
5001                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5002                 kvm_run->fail_entry.hardware_entry_failure_reason
5003                         = svm->vmcb->control.exit_code;
5004                 dump_vmcb(vcpu);
5005                 return 0;
5006         }
5007
5008         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
5009             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
5010             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
5011             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
5012                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
5013                        "exit_code 0x%x\n",
5014                        __func__, svm->vmcb->control.exit_int_info,
5015                        exit_code);
5016
5017         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
5018             || !svm_exit_handlers[exit_code]) {
5019                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
5020                 kvm_queue_exception(vcpu, UD_VECTOR);
5021                 return 1;
5022         }
5023
5024         return svm_exit_handlers[exit_code](svm);
5025 }
5026
5027 static void reload_tss(struct kvm_vcpu *vcpu)
5028 {
5029         int cpu = raw_smp_processor_id();
5030
5031         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5032         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
5033         load_TR_desc();
5034 }
5035
5036 static void pre_sev_run(struct vcpu_svm *svm, int cpu)
5037 {
5038         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5039         int asid = sev_get_asid(svm->vcpu.kvm);
5040
5041         /* Assign the asid allocated with this SEV guest */
5042         svm->vmcb->control.asid = asid;
5043
5044         /*
5045          * Flush guest TLB:
5046          *
5047          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
5048          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
5049          */
5050         if (sd->sev_vmcbs[asid] == svm->vmcb &&
5051             svm->last_cpu == cpu)
5052                 return;
5053
5054         svm->last_cpu = cpu;
5055         sd->sev_vmcbs[asid] = svm->vmcb;
5056         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5057         mark_dirty(svm->vmcb, VMCB_ASID);
5058 }
5059
5060 static void pre_svm_run(struct vcpu_svm *svm)
5061 {
5062         int cpu = raw_smp_processor_id();
5063
5064         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5065
5066         if (sev_guest(svm->vcpu.kvm))
5067                 return pre_sev_run(svm, cpu);
5068
5069         /* FIXME: handle wraparound of asid_generation */
5070         if (svm->asid_generation != sd->asid_generation)
5071                 new_asid(svm, sd);
5072 }
5073
5074 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
5075 {
5076         struct vcpu_svm *svm = to_svm(vcpu);
5077
5078         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
5079         vcpu->arch.hflags |= HF_NMI_MASK;
5080         set_intercept(svm, INTERCEPT_IRET);
5081         ++vcpu->stat.nmi_injections;
5082 }
5083
5084 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
5085 {
5086         struct vmcb_control_area *control;
5087
5088         /* The following fields are ignored when AVIC is enabled */
5089         control = &svm->vmcb->control;
5090         control->int_vector = irq;
5091         control->int_ctl &= ~V_INTR_PRIO_MASK;
5092         control->int_ctl |= V_IRQ_MASK |
5093                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
5094         mark_dirty(svm->vmcb, VMCB_INTR);
5095 }
5096
5097 static void svm_set_irq(struct kvm_vcpu *vcpu)
5098 {
5099         struct vcpu_svm *svm = to_svm(vcpu);
5100
5101         BUG_ON(!(gif_set(svm)));
5102
5103         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
5104         ++vcpu->stat.irq_injections;
5105
5106         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
5107                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
5108 }
5109
5110 static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
5111 {
5112         return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
5113 }
5114
5115 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
5116 {
5117         struct vcpu_svm *svm = to_svm(vcpu);
5118
5119         if (svm_nested_virtualize_tpr(vcpu) ||
5120             kvm_vcpu_apicv_active(vcpu))
5121                 return;
5122
5123         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5124
5125         if (irr == -1)
5126                 return;
5127
5128         if (tpr >= irr)
5129                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5130 }
5131
5132 static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
5133 {
5134         return;
5135 }
5136
5137 static bool svm_get_enable_apicv(struct kvm_vcpu *vcpu)
5138 {
5139         return avic && irqchip_split(vcpu->kvm);
5140 }
5141
5142 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
5143 {
5144 }
5145
5146 static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
5147 {
5148 }
5149
5150 /* Note: Currently only used by Hyper-V. */
5151 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5152 {
5153         struct vcpu_svm *svm = to_svm(vcpu);
5154         struct vmcb *vmcb = svm->vmcb;
5155
5156         if (kvm_vcpu_apicv_active(vcpu))
5157                 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
5158         else
5159                 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
5160         mark_dirty(vmcb, VMCB_AVIC);
5161 }
5162
5163 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
5164 {
5165         return;
5166 }
5167
5168 static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
5169 {
5170         kvm_lapic_set_irr(vec, vcpu->arch.apic);
5171         smp_mb__after_atomic();
5172
5173         if (avic_vcpu_is_running(vcpu)) {
5174                 int cpuid = vcpu->cpu;
5175
5176                 if (cpuid != get_cpu())
5177                         wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid));
5178                 put_cpu();
5179         } else
5180                 kvm_vcpu_wake_up(vcpu);
5181 }
5182
5183 static bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu)
5184 {
5185         return false;
5186 }
5187
5188 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5189 {
5190         unsigned long flags;
5191         struct amd_svm_iommu_ir *cur;
5192
5193         spin_lock_irqsave(&svm->ir_list_lock, flags);
5194         list_for_each_entry(cur, &svm->ir_list, node) {
5195                 if (cur->data != pi->ir_data)
5196                         continue;
5197                 list_del(&cur->node);
5198                 kfree(cur);
5199                 break;
5200         }
5201         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5202 }
5203
5204 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5205 {
5206         int ret = 0;
5207         unsigned long flags;
5208         struct amd_svm_iommu_ir *ir;
5209
5210         /**
5211          * In some cases, the existing irte is updaed and re-set,
5212          * so we need to check here if it's already been * added
5213          * to the ir_list.
5214          */
5215         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
5216                 struct kvm *kvm = svm->vcpu.kvm;
5217                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
5218                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
5219                 struct vcpu_svm *prev_svm;
5220
5221                 if (!prev_vcpu) {
5222                         ret = -EINVAL;
5223                         goto out;
5224                 }
5225
5226                 prev_svm = to_svm(prev_vcpu);
5227                 svm_ir_list_del(prev_svm, pi);
5228         }
5229
5230         /**
5231          * Allocating new amd_iommu_pi_data, which will get
5232          * add to the per-vcpu ir_list.
5233          */
5234         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
5235         if (!ir) {
5236                 ret = -ENOMEM;
5237                 goto out;
5238         }
5239         ir->data = pi->ir_data;
5240
5241         spin_lock_irqsave(&svm->ir_list_lock, flags);
5242         list_add(&ir->node, &svm->ir_list);
5243         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5244 out:
5245         return ret;
5246 }
5247
5248 /**
5249  * Note:
5250  * The HW cannot support posting multicast/broadcast
5251  * interrupts to a vCPU. So, we still use legacy interrupt
5252  * remapping for these kind of interrupts.
5253  *
5254  * For lowest-priority interrupts, we only support
5255  * those with single CPU as the destination, e.g. user
5256  * configures the interrupts via /proc/irq or uses
5257  * irqbalance to make the interrupts single-CPU.
5258  */
5259 static int
5260 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
5261                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
5262 {
5263         struct kvm_lapic_irq irq;
5264         struct kvm_vcpu *vcpu = NULL;
5265
5266         kvm_set_msi_irq(kvm, e, &irq);
5267
5268         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
5269                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5270                          __func__, irq.vector);
5271                 return -1;
5272         }
5273
5274         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
5275                  irq.vector);
5276         *svm = to_svm(vcpu);
5277         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
5278         vcpu_info->vector = irq.vector;
5279
5280         return 0;
5281 }
5282
5283 /*
5284  * svm_update_pi_irte - set IRTE for Posted-Interrupts
5285  *
5286  * @kvm: kvm
5287  * @host_irq: host irq of the interrupt
5288  * @guest_irq: gsi of the interrupt
5289  * @set: set or unset PI
5290  * returns 0 on success, < 0 on failure
5291  */
5292 static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
5293                               uint32_t guest_irq, bool set)
5294 {
5295         struct kvm_kernel_irq_routing_entry *e;
5296         struct kvm_irq_routing_table *irq_rt;
5297         int idx, ret = -EINVAL;
5298
5299         if (!kvm_arch_has_assigned_device(kvm) ||
5300             !irq_remapping_cap(IRQ_POSTING_CAP))
5301                 return 0;
5302
5303         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
5304                  __func__, host_irq, guest_irq, set);
5305
5306         idx = srcu_read_lock(&kvm->irq_srcu);
5307         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
5308         WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
5309
5310         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
5311                 struct vcpu_data vcpu_info;
5312                 struct vcpu_svm *svm = NULL;
5313
5314                 if (e->type != KVM_IRQ_ROUTING_MSI)
5315                         continue;
5316
5317                 /**
5318                  * Here, we setup with legacy mode in the following cases:
5319                  * 1. When cannot target interrupt to a specific vcpu.
5320                  * 2. Unsetting posted interrupt.
5321                  * 3. APIC virtialization is disabled for the vcpu.
5322                  */
5323                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5324                     kvm_vcpu_apicv_active(&svm->vcpu)) {
5325                         struct amd_iommu_pi_data pi;
5326
5327                         /* Try to enable guest_mode in IRTE */
5328                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
5329                                             AVIC_HPA_MASK);
5330                         pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
5331                                                      svm->vcpu.vcpu_id);
5332                         pi.is_guest_mode = true;
5333                         pi.vcpu_data = &vcpu_info;
5334                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5335
5336                         /**
5337                          * Here, we successfully setting up vcpu affinity in
5338                          * IOMMU guest mode. Now, we need to store the posted
5339                          * interrupt information in a per-vcpu ir_list so that
5340                          * we can reference to them directly when we update vcpu
5341                          * scheduling information in IOMMU irte.
5342                          */
5343                         if (!ret && pi.is_guest_mode)
5344                                 svm_ir_list_add(svm, &pi);
5345                 } else {
5346                         /* Use legacy mode in IRTE */
5347                         struct amd_iommu_pi_data pi;
5348
5349                         /**
5350                          * Here, pi is used to:
5351                          * - Tell IOMMU to use legacy mode for this interrupt.
5352                          * - Retrieve ga_tag of prior interrupt remapping data.
5353                          */
5354                         pi.is_guest_mode = false;
5355                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5356
5357                         /**
5358                          * Check if the posted interrupt was previously
5359                          * setup with the guest_mode by checking if the ga_tag
5360                          * was cached. If so, we need to clean up the per-vcpu
5361                          * ir_list.
5362                          */
5363                         if (!ret && pi.prev_ga_tag) {
5364                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
5365                                 struct kvm_vcpu *vcpu;
5366
5367                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
5368                                 if (vcpu)
5369                                         svm_ir_list_del(to_svm(vcpu), &pi);
5370                         }
5371                 }
5372
5373                 if (!ret && svm) {
5374                         trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
5375                                                  e->gsi, vcpu_info.vector,
5376                                                  vcpu_info.pi_desc_addr, set);
5377                 }
5378
5379                 if (ret < 0) {
5380                         pr_err("%s: failed to update PI IRTE\n", __func__);
5381                         goto out;
5382                 }
5383         }
5384
5385         ret = 0;
5386 out:
5387         srcu_read_unlock(&kvm->irq_srcu, idx);
5388         return ret;
5389 }
5390
5391 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
5392 {
5393         struct vcpu_svm *svm = to_svm(vcpu);
5394         struct vmcb *vmcb = svm->vmcb;
5395         int ret;
5396         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
5397               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
5398         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
5399
5400         return ret;
5401 }
5402
5403 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
5404 {
5405         struct vcpu_svm *svm = to_svm(vcpu);
5406
5407         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
5408 }
5409
5410 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5411 {
5412         struct vcpu_svm *svm = to_svm(vcpu);
5413
5414         if (masked) {
5415                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
5416                 set_intercept(svm, INTERCEPT_IRET);
5417         } else {
5418                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
5419                 clr_intercept(svm, INTERCEPT_IRET);
5420         }
5421 }
5422
5423 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
5424 {
5425         struct vcpu_svm *svm = to_svm(vcpu);
5426         struct vmcb *vmcb = svm->vmcb;
5427         int ret;
5428
5429         if (!gif_set(svm) ||
5430              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
5431                 return 0;
5432
5433         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
5434
5435         if (is_guest_mode(vcpu))
5436                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
5437
5438         return ret;
5439 }
5440
5441 static void enable_irq_window(struct kvm_vcpu *vcpu)
5442 {
5443         struct vcpu_svm *svm = to_svm(vcpu);
5444
5445         if (kvm_vcpu_apicv_active(vcpu))
5446                 return;
5447
5448         /*
5449          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
5450          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
5451          * get that intercept, this function will be called again though and
5452          * we'll get the vintr intercept. However, if the vGIF feature is
5453          * enabled, the STGI interception will not occur. Enable the irq
5454          * window under the assumption that the hardware will set the GIF.
5455          */
5456         if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
5457                 svm_set_vintr(svm);
5458                 svm_inject_irq(svm, 0x0);
5459         }
5460 }
5461
5462 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5463 {
5464         struct vcpu_svm *svm = to_svm(vcpu);
5465
5466         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
5467             == HF_NMI_MASK)
5468                 return; /* IRET will cause a vm exit */
5469
5470         if (!gif_set(svm)) {
5471                 if (vgif_enabled(svm))
5472                         set_intercept(svm, INTERCEPT_STGI);
5473                 return; /* STGI will cause a vm exit */
5474         }
5475
5476         if (svm->nested.exit_required)
5477                 return; /* we're not going to run the guest yet */
5478
5479         /*
5480          * Something prevents NMI from been injected. Single step over possible
5481          * problem (IRET or exception injection or interrupt shadow)
5482          */
5483         svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
5484         svm->nmi_singlestep = true;
5485         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
5486 }
5487
5488 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
5489 {
5490         return 0;
5491 }
5492
5493 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
5494 {
5495         return 0;
5496 }
5497
5498 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5499 {
5500         struct vcpu_svm *svm = to_svm(vcpu);
5501
5502         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
5503                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5504         else
5505                 svm->asid_generation--;
5506 }
5507
5508 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
5509 {
5510         struct vcpu_svm *svm = to_svm(vcpu);
5511
5512         invlpga(gva, svm->vmcb->control.asid);
5513 }
5514
5515 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
5516 {
5517 }
5518
5519 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
5520 {
5521         struct vcpu_svm *svm = to_svm(vcpu);
5522
5523         if (svm_nested_virtualize_tpr(vcpu))
5524                 return;
5525
5526         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
5527                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
5528                 kvm_set_cr8(vcpu, cr8);
5529         }
5530 }
5531
5532 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
5533 {
5534         struct vcpu_svm *svm = to_svm(vcpu);
5535         u64 cr8;
5536
5537         if (svm_nested_virtualize_tpr(vcpu) ||
5538             kvm_vcpu_apicv_active(vcpu))
5539                 return;
5540
5541         cr8 = kvm_get_cr8(vcpu);
5542         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
5543         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
5544 }
5545
5546 static void svm_complete_interrupts(struct vcpu_svm *svm)
5547 {
5548         u8 vector;
5549         int type;
5550         u32 exitintinfo = svm->vmcb->control.exit_int_info;
5551         unsigned int3_injected = svm->int3_injected;
5552
5553         svm->int3_injected = 0;
5554
5555         /*
5556          * If we've made progress since setting HF_IRET_MASK, we've
5557          * executed an IRET and can allow NMI injection.
5558          */
5559         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
5560             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
5561                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
5562                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5563         }
5564
5565         svm->vcpu.arch.nmi_injected = false;
5566         kvm_clear_exception_queue(&svm->vcpu);
5567         kvm_clear_interrupt_queue(&svm->vcpu);
5568
5569         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
5570                 return;
5571
5572         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5573
5574         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
5575         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
5576
5577         switch (type) {
5578         case SVM_EXITINTINFO_TYPE_NMI:
5579                 svm->vcpu.arch.nmi_injected = true;
5580                 break;
5581         case SVM_EXITINTINFO_TYPE_EXEPT:
5582                 /*
5583                  * In case of software exceptions, do not reinject the vector,
5584                  * but re-execute the instruction instead. Rewind RIP first
5585                  * if we emulated INT3 before.
5586                  */
5587                 if (kvm_exception_is_soft(vector)) {
5588                         if (vector == BP_VECTOR && int3_injected &&
5589                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
5590                                 kvm_rip_write(&svm->vcpu,
5591                                               kvm_rip_read(&svm->vcpu) -
5592                                               int3_injected);
5593                         break;
5594                 }
5595                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
5596                         u32 err = svm->vmcb->control.exit_int_info_err;
5597                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
5598
5599                 } else
5600                         kvm_requeue_exception(&svm->vcpu, vector);
5601                 break;
5602         case SVM_EXITINTINFO_TYPE_INTR:
5603                 kvm_queue_interrupt(&svm->vcpu, vector, false);
5604                 break;
5605         default:
5606                 break;
5607         }
5608 }
5609
5610 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
5611 {
5612         struct vcpu_svm *svm = to_svm(vcpu);
5613         struct vmcb_control_area *control = &svm->vmcb->control;
5614
5615         control->exit_int_info = control->event_inj;
5616         control->exit_int_info_err = control->event_inj_err;
5617         control->event_inj = 0;
5618         svm_complete_interrupts(svm);
5619 }
5620
5621 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
5622 {
5623         struct vcpu_svm *svm = to_svm(vcpu);
5624
5625         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5626         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5627         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5628
5629         /*
5630          * A vmexit emulation is required before the vcpu can be executed
5631          * again.
5632          */
5633         if (unlikely(svm->nested.exit_required))
5634                 return;
5635
5636         /*
5637          * Disable singlestep if we're injecting an interrupt/exception.
5638          * We don't want our modified rflags to be pushed on the stack where
5639          * we might not be able to easily reset them if we disabled NMI
5640          * singlestep later.
5641          */
5642         if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
5643                 /*
5644                  * Event injection happens before external interrupts cause a
5645                  * vmexit and interrupts are disabled here, so smp_send_reschedule
5646                  * is enough to force an immediate vmexit.
5647                  */
5648                 disable_nmi_singlestep(svm);
5649                 smp_send_reschedule(vcpu->cpu);
5650         }
5651
5652         pre_svm_run(svm);
5653
5654         sync_lapic_to_cr8(vcpu);
5655
5656         svm->vmcb->save.cr2 = vcpu->arch.cr2;
5657
5658         clgi();
5659         kvm_load_guest_xcr0(vcpu);
5660
5661         if (lapic_in_kernel(vcpu) &&
5662                 vcpu->arch.apic->lapic_timer.timer_advance_ns)
5663                 kvm_wait_lapic_expire(vcpu);
5664
5665         /*
5666          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
5667          * it's non-zero. Since vmentry is serialising on affected CPUs, there
5668          * is no need to worry about the conditional branch over the wrmsr
5669          * being speculatively taken.
5670          */
5671         x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
5672
5673         local_irq_enable();
5674
5675         asm volatile (
5676                 "push %%" _ASM_BP "; \n\t"
5677                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
5678                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
5679                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
5680                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
5681                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
5682                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
5683 #ifdef CONFIG_X86_64
5684                 "mov %c[r8](%[svm]),  %%r8  \n\t"
5685                 "mov %c[r9](%[svm]),  %%r9  \n\t"
5686                 "mov %c[r10](%[svm]), %%r10 \n\t"
5687                 "mov %c[r11](%[svm]), %%r11 \n\t"
5688                 "mov %c[r12](%[svm]), %%r12 \n\t"
5689                 "mov %c[r13](%[svm]), %%r13 \n\t"
5690                 "mov %c[r14](%[svm]), %%r14 \n\t"
5691                 "mov %c[r15](%[svm]), %%r15 \n\t"
5692 #endif
5693
5694                 /* Enter guest mode */
5695                 "push %%" _ASM_AX " \n\t"
5696                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
5697                 __ex("vmload %%" _ASM_AX) "\n\t"
5698                 __ex("vmrun %%" _ASM_AX) "\n\t"
5699                 __ex("vmsave %%" _ASM_AX) "\n\t"
5700                 "pop %%" _ASM_AX " \n\t"
5701
5702                 /* Save guest registers, load host registers */
5703                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
5704                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
5705                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
5706                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
5707                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
5708                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
5709 #ifdef CONFIG_X86_64
5710                 "mov %%r8,  %c[r8](%[svm]) \n\t"
5711                 "mov %%r9,  %c[r9](%[svm]) \n\t"
5712                 "mov %%r10, %c[r10](%[svm]) \n\t"
5713                 "mov %%r11, %c[r11](%[svm]) \n\t"
5714                 "mov %%r12, %c[r12](%[svm]) \n\t"
5715                 "mov %%r13, %c[r13](%[svm]) \n\t"
5716                 "mov %%r14, %c[r14](%[svm]) \n\t"
5717                 "mov %%r15, %c[r15](%[svm]) \n\t"
5718                 /*
5719                 * Clear host registers marked as clobbered to prevent
5720                 * speculative use.
5721                 */
5722                 "xor %%r8d, %%r8d \n\t"
5723                 "xor %%r9d, %%r9d \n\t"
5724                 "xor %%r10d, %%r10d \n\t"
5725                 "xor %%r11d, %%r11d \n\t"
5726                 "xor %%r12d, %%r12d \n\t"
5727                 "xor %%r13d, %%r13d \n\t"
5728                 "xor %%r14d, %%r14d \n\t"
5729                 "xor %%r15d, %%r15d \n\t"
5730 #endif
5731                 "xor %%ebx, %%ebx \n\t"
5732                 "xor %%ecx, %%ecx \n\t"
5733                 "xor %%edx, %%edx \n\t"
5734                 "xor %%esi, %%esi \n\t"
5735                 "xor %%edi, %%edi \n\t"
5736                 "pop %%" _ASM_BP
5737                 :
5738                 : [svm]"a"(svm),
5739                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
5740                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
5741                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
5742                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
5743                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
5744                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
5745                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
5746 #ifdef CONFIG_X86_64
5747                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
5748                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
5749                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
5750                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
5751                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
5752                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
5753                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
5754                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
5755 #endif
5756                 : "cc", "memory"
5757 #ifdef CONFIG_X86_64
5758                 , "rbx", "rcx", "rdx", "rsi", "rdi"
5759                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
5760 #else
5761                 , "ebx", "ecx", "edx", "esi", "edi"
5762 #endif
5763                 );
5764
5765         /* Eliminate branch target predictions from guest mode */
5766         vmexit_fill_RSB();
5767
5768 #ifdef CONFIG_X86_64
5769         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
5770 #else
5771         loadsegment(fs, svm->host.fs);
5772 #ifndef CONFIG_X86_32_LAZY_GS
5773         loadsegment(gs, svm->host.gs);
5774 #endif
5775 #endif
5776
5777         /*
5778          * We do not use IBRS in the kernel. If this vCPU has used the
5779          * SPEC_CTRL MSR it may have left it on; save the value and
5780          * turn it off. This is much more efficient than blindly adding
5781          * it to the atomic save/restore list. Especially as the former
5782          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
5783          *
5784          * For non-nested case:
5785          * If the L01 MSR bitmap does not intercept the MSR, then we need to
5786          * save it.
5787          *
5788          * For nested case:
5789          * If the L02 MSR bitmap does not intercept the MSR, then we need to
5790          * save it.
5791          */
5792         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
5793                 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
5794
5795         reload_tss(vcpu);
5796
5797         local_irq_disable();
5798
5799         x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
5800
5801         vcpu->arch.cr2 = svm->vmcb->save.cr2;
5802         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5803         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5804         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5805
5806         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5807                 kvm_before_interrupt(&svm->vcpu);
5808
5809         kvm_put_guest_xcr0(vcpu);
5810         stgi();
5811
5812         /* Any pending NMI will happen here */
5813
5814         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5815                 kvm_after_interrupt(&svm->vcpu);
5816
5817         sync_cr8_to_lapic(vcpu);
5818
5819         svm->next_rip = 0;
5820
5821         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5822
5823         /* if exit due to PF check for async PF */
5824         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5825                 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
5826
5827         if (npt_enabled) {
5828                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5829                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5830         }
5831
5832         /*
5833          * We need to handle MC intercepts here before the vcpu has a chance to
5834          * change the physical cpu
5835          */
5836         if (unlikely(svm->vmcb->control.exit_code ==
5837                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
5838                 svm_handle_mce(svm);
5839
5840         mark_all_clean(svm->vmcb);
5841 }
5842 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
5843
5844 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5845 {
5846         struct vcpu_svm *svm = to_svm(vcpu);
5847
5848         svm->vmcb->save.cr3 = __sme_set(root);
5849         mark_dirty(svm->vmcb, VMCB_CR);
5850 }
5851
5852 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5853 {
5854         struct vcpu_svm *svm = to_svm(vcpu);
5855
5856         svm->vmcb->control.nested_cr3 = __sme_set(root);
5857         mark_dirty(svm->vmcb, VMCB_NPT);
5858
5859         /* Also sync guest cr3 here in case we live migrate */
5860         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5861         mark_dirty(svm->vmcb, VMCB_CR);
5862 }
5863
5864 static int is_disabled(void)
5865 {
5866         u64 vm_cr;
5867
5868         rdmsrl(MSR_VM_CR, vm_cr);
5869         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5870                 return 1;
5871
5872         return 0;
5873 }
5874
5875 static void
5876 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5877 {
5878         /*
5879          * Patch in the VMMCALL instruction:
5880          */
5881         hypercall[0] = 0x0f;
5882         hypercall[1] = 0x01;
5883         hypercall[2] = 0xd9;
5884 }
5885
5886 static int __init svm_check_processor_compat(void)
5887 {
5888         return 0;
5889 }
5890
5891 static bool svm_cpu_has_accelerated_tpr(void)
5892 {
5893         return false;
5894 }
5895
5896 static bool svm_has_emulated_msr(int index)
5897 {
5898         switch (index) {
5899         case MSR_IA32_MCG_EXT_CTL:
5900         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
5901                 return false;
5902         default:
5903                 break;
5904         }
5905
5906         return true;
5907 }
5908
5909 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
5910 {
5911         return 0;
5912 }
5913
5914 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
5915 {
5916         struct vcpu_svm *svm = to_svm(vcpu);
5917
5918         /* Update nrips enabled cache */
5919         svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
5920
5921         if (!kvm_vcpu_apicv_active(vcpu))
5922                 return;
5923
5924         guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5925 }
5926
5927 #define F(x) bit(X86_FEATURE_##x)
5928
5929 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5930 {
5931         switch (func) {
5932         case 0x1:
5933                 if (avic)
5934                         entry->ecx &= ~bit(X86_FEATURE_X2APIC);
5935                 break;
5936         case 0x80000001:
5937                 if (nested)
5938                         entry->ecx |= (1 << 2); /* Set SVM bit */
5939                 break;
5940         case 0x80000008:
5941                 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5942                      boot_cpu_has(X86_FEATURE_AMD_SSBD))
5943                         entry->ebx |= F(VIRT_SSBD);
5944                 break;
5945         case 0x8000000A:
5946                 entry->eax = 1; /* SVM revision 1 */
5947                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
5948                                    ASID emulation to nested SVM */
5949                 entry->ecx = 0; /* Reserved */
5950                 entry->edx = 0; /* Per default do not support any
5951                                    additional features */
5952
5953                 /* Support next_rip if host supports it */
5954                 if (boot_cpu_has(X86_FEATURE_NRIPS))
5955                         entry->edx |= F(NRIPS);
5956
5957                 /* Support NPT for the guest if enabled */
5958                 if (npt_enabled)
5959                         entry->edx |= F(NPT);
5960
5961                 break;
5962         case 0x8000001F:
5963                 /* Support memory encryption cpuid if host supports it */
5964                 if (boot_cpu_has(X86_FEATURE_SEV))
5965                         cpuid(0x8000001f, &entry->eax, &entry->ebx,
5966                                 &entry->ecx, &entry->edx);
5967
5968         }
5969 }
5970
5971 static int svm_get_lpage_level(void)
5972 {
5973         return PT_PDPE_LEVEL;
5974 }
5975
5976 static bool svm_rdtscp_supported(void)
5977 {
5978         return boot_cpu_has(X86_FEATURE_RDTSCP);
5979 }
5980
5981 static bool svm_invpcid_supported(void)
5982 {
5983         return false;
5984 }
5985
5986 static bool svm_mpx_supported(void)
5987 {
5988         return false;
5989 }
5990
5991 static bool svm_xsaves_supported(void)
5992 {
5993         return false;
5994 }
5995
5996 static bool svm_umip_emulated(void)
5997 {
5998         return false;
5999 }
6000
6001 static bool svm_pt_supported(void)
6002 {
6003         return false;
6004 }
6005
6006 static bool svm_has_wbinvd_exit(void)
6007 {
6008         return true;
6009 }
6010
6011 #define PRE_EX(exit)  { .exit_code = (exit), \
6012                         .stage = X86_ICPT_PRE_EXCEPT, }
6013 #define POST_EX(exit) { .exit_code = (exit), \
6014                         .stage = X86_ICPT_POST_EXCEPT, }
6015 #define POST_MEM(exit) { .exit_code = (exit), \
6016                         .stage = X86_ICPT_POST_MEMACCESS, }
6017
6018 static const struct __x86_intercept {
6019         u32 exit_code;
6020         enum x86_intercept_stage stage;
6021 } x86_intercept_map[] = {
6022         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
6023         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
6024         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
6025         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
6026         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
6027         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
6028         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
6029         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
6030         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
6031         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
6032         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
6033         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
6034         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
6035         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
6036         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
6037         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
6038         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
6039         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
6040         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
6041         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
6042         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
6043         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
6044         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
6045         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
6046         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
6047         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
6048         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
6049         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
6050         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
6051         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
6052         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
6053         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
6054         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
6055         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
6056         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
6057         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
6058         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
6059         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
6060         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
6061         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
6062         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
6063         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
6064         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
6065         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
6066         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
6067         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
6068         [x86_intercept_xsetbv]          = PRE_EX(SVM_EXIT_XSETBV),
6069 };
6070
6071 #undef PRE_EX
6072 #undef POST_EX
6073 #undef POST_MEM
6074
6075 static int svm_check_intercept(struct kvm_vcpu *vcpu,
6076                                struct x86_instruction_info *info,
6077                                enum x86_intercept_stage stage)
6078 {
6079         struct vcpu_svm *svm = to_svm(vcpu);
6080         int vmexit, ret = X86EMUL_CONTINUE;
6081         struct __x86_intercept icpt_info;
6082         struct vmcb *vmcb = svm->vmcb;
6083
6084         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
6085                 goto out;
6086
6087         icpt_info = x86_intercept_map[info->intercept];
6088
6089         if (stage != icpt_info.stage)
6090                 goto out;
6091
6092         switch (icpt_info.exit_code) {
6093         case SVM_EXIT_READ_CR0:
6094                 if (info->intercept == x86_intercept_cr_read)
6095                         icpt_info.exit_code += info->modrm_reg;
6096                 break;
6097         case SVM_EXIT_WRITE_CR0: {
6098                 unsigned long cr0, val;
6099                 u64 intercept;
6100
6101                 if (info->intercept == x86_intercept_cr_write)
6102                         icpt_info.exit_code += info->modrm_reg;
6103
6104                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
6105                     info->intercept == x86_intercept_clts)
6106                         break;
6107
6108                 intercept = svm->nested.intercept;
6109
6110                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
6111                         break;
6112
6113                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
6114                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
6115
6116                 if (info->intercept == x86_intercept_lmsw) {
6117                         cr0 &= 0xfUL;
6118                         val &= 0xfUL;
6119                         /* lmsw can't clear PE - catch this here */
6120                         if (cr0 & X86_CR0_PE)
6121                                 val |= X86_CR0_PE;
6122                 }
6123
6124                 if (cr0 ^ val)
6125                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
6126
6127                 break;
6128         }
6129         case SVM_EXIT_READ_DR0:
6130         case SVM_EXIT_WRITE_DR0:
6131                 icpt_info.exit_code += info->modrm_reg;
6132                 break;
6133         case SVM_EXIT_MSR:
6134                 if (info->intercept == x86_intercept_wrmsr)
6135                         vmcb->control.exit_info_1 = 1;
6136                 else
6137                         vmcb->control.exit_info_1 = 0;
6138                 break;
6139         case SVM_EXIT_PAUSE:
6140                 /*
6141                  * We get this for NOP only, but pause
6142                  * is rep not, check this here
6143                  */
6144                 if (info->rep_prefix != REPE_PREFIX)
6145                         goto out;
6146                 break;
6147         case SVM_EXIT_IOIO: {
6148                 u64 exit_info;
6149                 u32 bytes;
6150
6151                 if (info->intercept == x86_intercept_in ||
6152                     info->intercept == x86_intercept_ins) {
6153                         exit_info = ((info->src_val & 0xffff) << 16) |
6154                                 SVM_IOIO_TYPE_MASK;
6155                         bytes = info->dst_bytes;
6156                 } else {
6157                         exit_info = (info->dst_val & 0xffff) << 16;
6158                         bytes = info->src_bytes;
6159                 }
6160
6161                 if (info->intercept == x86_intercept_outs ||
6162                     info->intercept == x86_intercept_ins)
6163                         exit_info |= SVM_IOIO_STR_MASK;
6164
6165                 if (info->rep_prefix)
6166                         exit_info |= SVM_IOIO_REP_MASK;
6167
6168                 bytes = min(bytes, 4u);
6169
6170                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
6171
6172                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
6173
6174                 vmcb->control.exit_info_1 = exit_info;
6175                 vmcb->control.exit_info_2 = info->next_rip;
6176
6177                 break;
6178         }
6179         default:
6180                 break;
6181         }
6182
6183         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
6184         if (static_cpu_has(X86_FEATURE_NRIPS))
6185                 vmcb->control.next_rip  = info->next_rip;
6186         vmcb->control.exit_code = icpt_info.exit_code;
6187         vmexit = nested_svm_exit_handled(svm);
6188
6189         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
6190                                            : X86EMUL_CONTINUE;
6191
6192 out:
6193         return ret;
6194 }
6195
6196 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
6197 {
6198
6199 }
6200
6201 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
6202 {
6203         if (pause_filter_thresh)
6204                 shrink_ple_window(vcpu);
6205 }
6206
6207 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
6208 {
6209         if (avic_handle_apic_id_update(vcpu) != 0)
6210                 return;
6211         avic_handle_dfr_update(vcpu);
6212         avic_handle_ldr_update(vcpu);
6213 }
6214
6215 static void svm_setup_mce(struct kvm_vcpu *vcpu)
6216 {
6217         /* [63:9] are reserved. */
6218         vcpu->arch.mcg_cap &= 0x1ff;
6219 }
6220
6221 static int svm_smi_allowed(struct kvm_vcpu *vcpu)
6222 {
6223         struct vcpu_svm *svm = to_svm(vcpu);
6224
6225         /* Per APM Vol.2 15.22.2 "Response to SMI" */
6226         if (!gif_set(svm))
6227                 return 0;
6228
6229         if (is_guest_mode(&svm->vcpu) &&
6230             svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
6231                 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
6232                 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
6233                 svm->nested.exit_required = true;
6234                 return 0;
6235         }
6236
6237         return 1;
6238 }
6239
6240 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
6241 {
6242         struct vcpu_svm *svm = to_svm(vcpu);
6243         int ret;
6244
6245         if (is_guest_mode(vcpu)) {
6246                 /* FED8h - SVM Guest */
6247                 put_smstate(u64, smstate, 0x7ed8, 1);
6248                 /* FEE0h - SVM Guest VMCB Physical Address */
6249                 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
6250
6251                 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
6252                 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
6253                 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
6254
6255                 ret = nested_svm_vmexit(svm);
6256                 if (ret)
6257                         return ret;
6258         }
6259         return 0;
6260 }
6261
6262 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
6263 {
6264         struct vcpu_svm *svm = to_svm(vcpu);
6265         struct vmcb *nested_vmcb;
6266         struct kvm_host_map map;
6267         u64 guest;
6268         u64 vmcb;
6269
6270         guest = GET_SMSTATE(u64, smstate, 0x7ed8);
6271         vmcb = GET_SMSTATE(u64, smstate, 0x7ee0);
6272
6273         if (guest) {
6274                 if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL)
6275                         return 1;
6276                 nested_vmcb = map.hva;
6277                 enter_svm_guest_mode(svm, vmcb, nested_vmcb, &map);
6278         }
6279         return 0;
6280 }
6281
6282 static int enable_smi_window(struct kvm_vcpu *vcpu)
6283 {
6284         struct vcpu_svm *svm = to_svm(vcpu);
6285
6286         if (!gif_set(svm)) {
6287                 if (vgif_enabled(svm))
6288                         set_intercept(svm, INTERCEPT_STGI);
6289                 /* STGI will cause a vm exit */
6290                 return 1;
6291         }
6292         return 0;
6293 }
6294
6295 static int sev_asid_new(void)
6296 {
6297         int pos;
6298
6299         /*
6300          * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
6301          */
6302         pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
6303         if (pos >= max_sev_asid)
6304                 return -EBUSY;
6305
6306         set_bit(pos, sev_asid_bitmap);
6307         return pos + 1;
6308 }
6309
6310 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
6311 {
6312         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6313         int asid, ret;
6314
6315         ret = -EBUSY;
6316         if (unlikely(sev->active))
6317                 return ret;
6318
6319         asid = sev_asid_new();
6320         if (asid < 0)
6321                 return ret;
6322
6323         ret = sev_platform_init(&argp->error);
6324         if (ret)
6325                 goto e_free;
6326
6327         sev->active = true;
6328         sev->asid = asid;
6329         INIT_LIST_HEAD(&sev->regions_list);
6330
6331         return 0;
6332
6333 e_free:
6334         __sev_asid_free(asid);
6335         return ret;
6336 }
6337
6338 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
6339 {
6340         struct sev_data_activate *data;
6341         int asid = sev_get_asid(kvm);
6342         int ret;
6343
6344         wbinvd_on_all_cpus();
6345
6346         ret = sev_guest_df_flush(error);
6347         if (ret)
6348                 return ret;
6349
6350         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6351         if (!data)
6352                 return -ENOMEM;
6353
6354         /* activate ASID on the given handle */
6355         data->handle = handle;
6356         data->asid   = asid;
6357         ret = sev_guest_activate(data, error);
6358         kfree(data);
6359
6360         return ret;
6361 }
6362
6363 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
6364 {
6365         struct fd f;
6366         int ret;
6367
6368         f = fdget(fd);
6369         if (!f.file)
6370                 return -EBADF;
6371
6372         ret = sev_issue_cmd_external_user(f.file, id, data, error);
6373
6374         fdput(f);
6375         return ret;
6376 }
6377
6378 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
6379 {
6380         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6381
6382         return __sev_issue_cmd(sev->fd, id, data, error);
6383 }
6384
6385 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
6386 {
6387         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6388         struct sev_data_launch_start *start;
6389         struct kvm_sev_launch_start params;
6390         void *dh_blob, *session_blob;
6391         int *error = &argp->error;
6392         int ret;
6393
6394         if (!sev_guest(kvm))
6395                 return -ENOTTY;
6396
6397         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6398                 return -EFAULT;
6399
6400         start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT);
6401         if (!start)
6402                 return -ENOMEM;
6403
6404         dh_blob = NULL;
6405         if (params.dh_uaddr) {
6406                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
6407                 if (IS_ERR(dh_blob)) {
6408                         ret = PTR_ERR(dh_blob);
6409                         goto e_free;
6410                 }
6411
6412                 start->dh_cert_address = __sme_set(__pa(dh_blob));
6413                 start->dh_cert_len = params.dh_len;
6414         }
6415
6416         session_blob = NULL;
6417         if (params.session_uaddr) {
6418                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
6419                 if (IS_ERR(session_blob)) {
6420                         ret = PTR_ERR(session_blob);
6421                         goto e_free_dh;
6422                 }
6423
6424                 start->session_address = __sme_set(__pa(session_blob));
6425                 start->session_len = params.session_len;
6426         }
6427
6428         start->handle = params.handle;
6429         start->policy = params.policy;
6430
6431         /* create memory encryption context */
6432         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
6433         if (ret)
6434                 goto e_free_session;
6435
6436         /* Bind ASID to this guest */
6437         ret = sev_bind_asid(kvm, start->handle, error);
6438         if (ret)
6439                 goto e_free_session;
6440
6441         /* return handle to userspace */
6442         params.handle = start->handle;
6443         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
6444                 sev_unbind_asid(kvm, start->handle);
6445                 ret = -EFAULT;
6446                 goto e_free_session;
6447         }
6448
6449         sev->handle = start->handle;
6450         sev->fd = argp->sev_fd;
6451
6452 e_free_session:
6453         kfree(session_blob);
6454 e_free_dh:
6455         kfree(dh_blob);
6456 e_free:
6457         kfree(start);
6458         return ret;
6459 }
6460
6461 static unsigned long get_num_contig_pages(unsigned long idx,
6462                                 struct page **inpages, unsigned long npages)
6463 {
6464         unsigned long paddr, next_paddr;
6465         unsigned long i = idx + 1, pages = 1;
6466
6467         /* find the number of contiguous pages starting from idx */
6468         paddr = __sme_page_pa(inpages[idx]);
6469         while (i < npages) {
6470                 next_paddr = __sme_page_pa(inpages[i++]);
6471                 if ((paddr + PAGE_SIZE) == next_paddr) {
6472                         pages++;
6473                         paddr = next_paddr;
6474                         continue;
6475                 }
6476                 break;
6477         }
6478
6479         return pages;
6480 }
6481
6482 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
6483 {
6484         unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
6485         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6486         struct kvm_sev_launch_update_data params;
6487         struct sev_data_launch_update_data *data;
6488         struct page **inpages;
6489         int ret;
6490
6491         if (!sev_guest(kvm))
6492                 return -ENOTTY;
6493
6494         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6495                 return -EFAULT;
6496
6497         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6498         if (!data)
6499                 return -ENOMEM;
6500
6501         vaddr = params.uaddr;
6502         size = params.len;
6503         vaddr_end = vaddr + size;
6504
6505         /* Lock the user memory. */
6506         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
6507         if (!inpages) {
6508                 ret = -ENOMEM;
6509                 goto e_free;
6510         }
6511
6512         /*
6513          * The LAUNCH_UPDATE command will perform in-place encryption of the
6514          * memory content (i.e it will write the same memory region with C=1).
6515          * It's possible that the cache may contain the data with C=0, i.e.,
6516          * unencrypted so invalidate it first.
6517          */
6518         sev_clflush_pages(inpages, npages);
6519
6520         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
6521                 int offset, len;
6522
6523                 /*
6524                  * If the user buffer is not page-aligned, calculate the offset
6525                  * within the page.
6526                  */
6527                 offset = vaddr & (PAGE_SIZE - 1);
6528
6529                 /* Calculate the number of pages that can be encrypted in one go. */
6530                 pages = get_num_contig_pages(i, inpages, npages);
6531
6532                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
6533
6534                 data->handle = sev->handle;
6535                 data->len = len;
6536                 data->address = __sme_page_pa(inpages[i]) + offset;
6537                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
6538                 if (ret)
6539                         goto e_unpin;
6540
6541                 size -= len;
6542                 next_vaddr = vaddr + len;
6543         }
6544
6545 e_unpin:
6546         /* content of memory is updated, mark pages dirty */
6547         for (i = 0; i < npages; i++) {
6548                 set_page_dirty_lock(inpages[i]);
6549                 mark_page_accessed(inpages[i]);
6550         }
6551         /* unlock the user pages */
6552         sev_unpin_memory(kvm, inpages, npages);
6553 e_free:
6554         kfree(data);
6555         return ret;
6556 }
6557
6558 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
6559 {
6560         void __user *measure = (void __user *)(uintptr_t)argp->data;
6561         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6562         struct sev_data_launch_measure *data;
6563         struct kvm_sev_launch_measure params;
6564         void __user *p = NULL;
6565         void *blob = NULL;
6566         int ret;
6567
6568         if (!sev_guest(kvm))
6569                 return -ENOTTY;
6570
6571         if (copy_from_user(&params, measure, sizeof(params)))
6572                 return -EFAULT;
6573
6574         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6575         if (!data)
6576                 return -ENOMEM;
6577
6578         /* User wants to query the blob length */
6579         if (!params.len)
6580                 goto cmd;
6581
6582         p = (void __user *)(uintptr_t)params.uaddr;
6583         if (p) {
6584                 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
6585                         ret = -EINVAL;
6586                         goto e_free;
6587                 }
6588
6589                 ret = -ENOMEM;
6590                 blob = kmalloc(params.len, GFP_KERNEL);
6591                 if (!blob)
6592                         goto e_free;
6593
6594                 data->address = __psp_pa(blob);
6595                 data->len = params.len;
6596         }
6597
6598 cmd:
6599         data->handle = sev->handle;
6600         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
6601
6602         /*
6603          * If we query the session length, FW responded with expected data.
6604          */
6605         if (!params.len)
6606                 goto done;
6607
6608         if (ret)
6609                 goto e_free_blob;
6610
6611         if (blob) {
6612                 if (copy_to_user(p, blob, params.len))
6613                         ret = -EFAULT;
6614         }
6615
6616 done:
6617         params.len = data->len;
6618         if (copy_to_user(measure, &params, sizeof(params)))
6619                 ret = -EFAULT;
6620 e_free_blob:
6621         kfree(blob);
6622 e_free:
6623         kfree(data);
6624         return ret;
6625 }
6626
6627 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
6628 {
6629         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6630         struct sev_data_launch_finish *data;
6631         int ret;
6632
6633         if (!sev_guest(kvm))
6634                 return -ENOTTY;
6635
6636         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6637         if (!data)
6638                 return -ENOMEM;
6639
6640         data->handle = sev->handle;
6641         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
6642
6643         kfree(data);
6644         return ret;
6645 }
6646
6647 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
6648 {
6649         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6650         struct kvm_sev_guest_status params;
6651         struct sev_data_guest_status *data;
6652         int ret;
6653
6654         if (!sev_guest(kvm))
6655                 return -ENOTTY;
6656
6657         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6658         if (!data)
6659                 return -ENOMEM;
6660
6661         data->handle = sev->handle;
6662         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
6663         if (ret)
6664                 goto e_free;
6665
6666         params.policy = data->policy;
6667         params.state = data->state;
6668         params.handle = data->handle;
6669
6670         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
6671                 ret = -EFAULT;
6672 e_free:
6673         kfree(data);
6674         return ret;
6675 }
6676
6677 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
6678                                unsigned long dst, int size,
6679                                int *error, bool enc)
6680 {
6681         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6682         struct sev_data_dbg *data;
6683         int ret;
6684
6685         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6686         if (!data)
6687                 return -ENOMEM;
6688
6689         data->handle = sev->handle;
6690         data->dst_addr = dst;
6691         data->src_addr = src;
6692         data->len = size;
6693
6694         ret = sev_issue_cmd(kvm,
6695                             enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
6696                             data, error);
6697         kfree(data);
6698         return ret;
6699 }
6700
6701 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
6702                              unsigned long dst_paddr, int sz, int *err)
6703 {
6704         int offset;
6705
6706         /*
6707          * Its safe to read more than we are asked, caller should ensure that
6708          * destination has enough space.
6709          */
6710         src_paddr = round_down(src_paddr, 16);
6711         offset = src_paddr & 15;
6712         sz = round_up(sz + offset, 16);
6713
6714         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
6715 }
6716
6717 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
6718                                   unsigned long __user dst_uaddr,
6719                                   unsigned long dst_paddr,
6720                                   int size, int *err)
6721 {
6722         struct page *tpage = NULL;
6723         int ret, offset;
6724
6725         /* if inputs are not 16-byte then use intermediate buffer */
6726         if (!IS_ALIGNED(dst_paddr, 16) ||
6727             !IS_ALIGNED(paddr,     16) ||
6728             !IS_ALIGNED(size,      16)) {
6729                 tpage = (void *)alloc_page(GFP_KERNEL);
6730                 if (!tpage)
6731                         return -ENOMEM;
6732
6733                 dst_paddr = __sme_page_pa(tpage);
6734         }
6735
6736         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
6737         if (ret)
6738                 goto e_free;
6739
6740         if (tpage) {
6741                 offset = paddr & 15;
6742                 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
6743                                  page_address(tpage) + offset, size))
6744                         ret = -EFAULT;
6745         }
6746
6747 e_free:
6748         if (tpage)
6749                 __free_page(tpage);
6750
6751         return ret;
6752 }
6753
6754 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
6755                                   unsigned long __user vaddr,
6756                                   unsigned long dst_paddr,
6757                                   unsigned long __user dst_vaddr,
6758                                   int size, int *error)
6759 {
6760         struct page *src_tpage = NULL;
6761         struct page *dst_tpage = NULL;
6762         int ret, len = size;
6763
6764         /* If source buffer is not aligned then use an intermediate buffer */
6765         if (!IS_ALIGNED(vaddr, 16)) {
6766                 src_tpage = alloc_page(GFP_KERNEL);
6767                 if (!src_tpage)
6768                         return -ENOMEM;
6769
6770                 if (copy_from_user(page_address(src_tpage),
6771                                 (void __user *)(uintptr_t)vaddr, size)) {
6772                         __free_page(src_tpage);
6773                         return -EFAULT;
6774                 }
6775
6776                 paddr = __sme_page_pa(src_tpage);
6777         }
6778
6779         /*
6780          *  If destination buffer or length is not aligned then do read-modify-write:
6781          *   - decrypt destination in an intermediate buffer
6782          *   - copy the source buffer in an intermediate buffer
6783          *   - use the intermediate buffer as source buffer
6784          */
6785         if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
6786                 int dst_offset;
6787
6788                 dst_tpage = alloc_page(GFP_KERNEL);
6789                 if (!dst_tpage) {
6790                         ret = -ENOMEM;
6791                         goto e_free;
6792                 }
6793
6794                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
6795                                         __sme_page_pa(dst_tpage), size, error);
6796                 if (ret)
6797                         goto e_free;
6798
6799                 /*
6800                  *  If source is kernel buffer then use memcpy() otherwise
6801                  *  copy_from_user().
6802                  */
6803                 dst_offset = dst_paddr & 15;
6804
6805                 if (src_tpage)
6806                         memcpy(page_address(dst_tpage) + dst_offset,
6807                                page_address(src_tpage), size);
6808                 else {
6809                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
6810                                            (void __user *)(uintptr_t)vaddr, size)) {
6811                                 ret = -EFAULT;
6812                                 goto e_free;
6813                         }
6814                 }
6815
6816                 paddr = __sme_page_pa(dst_tpage);
6817                 dst_paddr = round_down(dst_paddr, 16);
6818                 len = round_up(size, 16);
6819         }
6820
6821         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
6822
6823 e_free:
6824         if (src_tpage)
6825                 __free_page(src_tpage);
6826         if (dst_tpage)
6827                 __free_page(dst_tpage);
6828         return ret;
6829 }
6830
6831 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6832 {
6833         unsigned long vaddr, vaddr_end, next_vaddr;
6834         unsigned long dst_vaddr;
6835         struct page **src_p, **dst_p;
6836         struct kvm_sev_dbg debug;
6837         unsigned long n;
6838         unsigned int size;
6839         int ret;
6840
6841         if (!sev_guest(kvm))
6842                 return -ENOTTY;
6843
6844         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
6845                 return -EFAULT;
6846
6847         if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
6848                 return -EINVAL;
6849         if (!debug.dst_uaddr)
6850                 return -EINVAL;
6851
6852         vaddr = debug.src_uaddr;
6853         size = debug.len;
6854         vaddr_end = vaddr + size;
6855         dst_vaddr = debug.dst_uaddr;
6856
6857         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
6858                 int len, s_off, d_off;
6859
6860                 /* lock userspace source and destination page */
6861                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
6862                 if (!src_p)
6863                         return -EFAULT;
6864
6865                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
6866                 if (!dst_p) {
6867                         sev_unpin_memory(kvm, src_p, n);
6868                         return -EFAULT;
6869                 }
6870
6871                 /*
6872                  * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
6873                  * memory content (i.e it will write the same memory region with C=1).
6874                  * It's possible that the cache may contain the data with C=0, i.e.,
6875                  * unencrypted so invalidate it first.
6876                  */
6877                 sev_clflush_pages(src_p, 1);
6878                 sev_clflush_pages(dst_p, 1);
6879
6880                 /*
6881                  * Since user buffer may not be page aligned, calculate the
6882                  * offset within the page.
6883                  */
6884                 s_off = vaddr & ~PAGE_MASK;
6885                 d_off = dst_vaddr & ~PAGE_MASK;
6886                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
6887
6888                 if (dec)
6889                         ret = __sev_dbg_decrypt_user(kvm,
6890                                                      __sme_page_pa(src_p[0]) + s_off,
6891                                                      dst_vaddr,
6892                                                      __sme_page_pa(dst_p[0]) + d_off,
6893                                                      len, &argp->error);
6894                 else
6895                         ret = __sev_dbg_encrypt_user(kvm,
6896                                                      __sme_page_pa(src_p[0]) + s_off,
6897                                                      vaddr,
6898                                                      __sme_page_pa(dst_p[0]) + d_off,
6899                                                      dst_vaddr,
6900                                                      len, &argp->error);
6901
6902                 sev_unpin_memory(kvm, src_p, n);
6903                 sev_unpin_memory(kvm, dst_p, n);
6904
6905                 if (ret)
6906                         goto err;
6907
6908                 next_vaddr = vaddr + len;
6909                 dst_vaddr = dst_vaddr + len;
6910                 size -= len;
6911         }
6912 err:
6913         return ret;
6914 }
6915
6916 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
6917 {
6918         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6919         struct sev_data_launch_secret *data;
6920         struct kvm_sev_launch_secret params;
6921         struct page **pages;
6922         void *blob, *hdr;
6923         unsigned long n;
6924         int ret, offset;
6925
6926         if (!sev_guest(kvm))
6927                 return -ENOTTY;
6928
6929         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6930                 return -EFAULT;
6931
6932         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
6933         if (!pages)
6934                 return -ENOMEM;
6935
6936         /*
6937          * The secret must be copied into contiguous memory region, lets verify
6938          * that userspace memory pages are contiguous before we issue command.
6939          */
6940         if (get_num_contig_pages(0, pages, n) != n) {
6941                 ret = -EINVAL;
6942                 goto e_unpin_memory;
6943         }
6944
6945         ret = -ENOMEM;
6946         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6947         if (!data)
6948                 goto e_unpin_memory;
6949
6950         offset = params.guest_uaddr & (PAGE_SIZE - 1);
6951         data->guest_address = __sme_page_pa(pages[0]) + offset;
6952         data->guest_len = params.guest_len;
6953
6954         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
6955         if (IS_ERR(blob)) {
6956                 ret = PTR_ERR(blob);
6957                 goto e_free;
6958         }
6959
6960         data->trans_address = __psp_pa(blob);
6961         data->trans_len = params.trans_len;
6962
6963         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
6964         if (IS_ERR(hdr)) {
6965                 ret = PTR_ERR(hdr);
6966                 goto e_free_blob;
6967         }
6968         data->hdr_address = __psp_pa(hdr);
6969         data->hdr_len = params.hdr_len;
6970
6971         data->handle = sev->handle;
6972         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
6973
6974         kfree(hdr);
6975
6976 e_free_blob:
6977         kfree(blob);
6978 e_free:
6979         kfree(data);
6980 e_unpin_memory:
6981         sev_unpin_memory(kvm, pages, n);
6982         return ret;
6983 }
6984
6985 static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
6986 {
6987         struct kvm_sev_cmd sev_cmd;
6988         int r;
6989
6990         if (!svm_sev_enabled())
6991                 return -ENOTTY;
6992
6993         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
6994                 return -EFAULT;
6995
6996         mutex_lock(&kvm->lock);
6997
6998         switch (sev_cmd.id) {
6999         case KVM_SEV_INIT:
7000                 r = sev_guest_init(kvm, &sev_cmd);
7001                 break;
7002         case KVM_SEV_LAUNCH_START:
7003                 r = sev_launch_start(kvm, &sev_cmd);
7004                 break;
7005         case KVM_SEV_LAUNCH_UPDATE_DATA:
7006                 r = sev_launch_update_data(kvm, &sev_cmd);
7007                 break;
7008         case KVM_SEV_LAUNCH_MEASURE:
7009                 r = sev_launch_measure(kvm, &sev_cmd);
7010                 break;
7011         case KVM_SEV_LAUNCH_FINISH:
7012                 r = sev_launch_finish(kvm, &sev_cmd);
7013                 break;
7014         case KVM_SEV_GUEST_STATUS:
7015                 r = sev_guest_status(kvm, &sev_cmd);
7016                 break;
7017         case KVM_SEV_DBG_DECRYPT:
7018                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
7019                 break;
7020         case KVM_SEV_DBG_ENCRYPT:
7021                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
7022                 break;
7023         case KVM_SEV_LAUNCH_SECRET:
7024                 r = sev_launch_secret(kvm, &sev_cmd);
7025                 break;
7026         default:
7027                 r = -EINVAL;
7028                 goto out;
7029         }
7030
7031         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
7032                 r = -EFAULT;
7033
7034 out:
7035         mutex_unlock(&kvm->lock);
7036         return r;
7037 }
7038
7039 static int svm_register_enc_region(struct kvm *kvm,
7040                                    struct kvm_enc_region *range)
7041 {
7042         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7043         struct enc_region *region;
7044         int ret = 0;
7045
7046         if (!sev_guest(kvm))
7047                 return -ENOTTY;
7048
7049         if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
7050                 return -EINVAL;
7051
7052         region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
7053         if (!region)
7054                 return -ENOMEM;
7055
7056         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
7057         if (!region->pages) {
7058                 ret = -ENOMEM;
7059                 goto e_free;
7060         }
7061
7062         /*
7063          * The guest may change the memory encryption attribute from C=0 -> C=1
7064          * or vice versa for this memory range. Lets make sure caches are
7065          * flushed to ensure that guest data gets written into memory with
7066          * correct C-bit.
7067          */
7068         sev_clflush_pages(region->pages, region->npages);
7069
7070         region->uaddr = range->addr;
7071         region->size = range->size;
7072
7073         mutex_lock(&kvm->lock);
7074         list_add_tail(&region->list, &sev->regions_list);
7075         mutex_unlock(&kvm->lock);
7076
7077         return ret;
7078
7079 e_free:
7080         kfree(region);
7081         return ret;
7082 }
7083
7084 static struct enc_region *
7085 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
7086 {
7087         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7088         struct list_head *head = &sev->regions_list;
7089         struct enc_region *i;
7090
7091         list_for_each_entry(i, head, list) {
7092                 if (i->uaddr == range->addr &&
7093                     i->size == range->size)
7094                         return i;
7095         }
7096
7097         return NULL;
7098 }
7099
7100
7101 static int svm_unregister_enc_region(struct kvm *kvm,
7102                                      struct kvm_enc_region *range)
7103 {
7104         struct enc_region *region;
7105         int ret;
7106
7107         mutex_lock(&kvm->lock);
7108
7109         if (!sev_guest(kvm)) {
7110                 ret = -ENOTTY;
7111                 goto failed;
7112         }
7113
7114         region = find_enc_region(kvm, range);
7115         if (!region) {
7116                 ret = -EINVAL;
7117                 goto failed;
7118         }
7119
7120         __unregister_enc_region_locked(kvm, region);
7121
7122         mutex_unlock(&kvm->lock);
7123         return 0;
7124
7125 failed:
7126         mutex_unlock(&kvm->lock);
7127         return ret;
7128 }
7129
7130 static uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu)
7131 {
7132         /* Not supported */
7133         return 0;
7134 }
7135
7136 static int nested_enable_evmcs(struct kvm_vcpu *vcpu,
7137                                    uint16_t *vmcs_version)
7138 {
7139         /* Intel-only feature */
7140         return -ENODEV;
7141 }
7142
7143 static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu)
7144 {
7145         unsigned long cr4 = kvm_read_cr4(vcpu);
7146         bool smep = cr4 & X86_CR4_SMEP;
7147         bool smap = cr4 & X86_CR4_SMAP;
7148         bool is_user = svm_get_cpl(vcpu) == 3;
7149
7150         /*
7151          * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
7152          *
7153          * Errata:
7154          * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
7155          * possible that CPU microcode implementing DecodeAssist will fail
7156          * to read bytes of instruction which caused #NPF. In this case,
7157          * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
7158          * return 0 instead of the correct guest instruction bytes.
7159          *
7160          * This happens because CPU microcode reading instruction bytes
7161          * uses a special opcode which attempts to read data using CPL=0
7162          * priviledges. The microcode reads CS:RIP and if it hits a SMAP
7163          * fault, it gives up and returns no instruction bytes.
7164          *
7165          * Detection:
7166          * We reach here in case CPU supports DecodeAssist, raised #NPF and
7167          * returned 0 in GuestIntrBytes field of the VMCB.
7168          * First, errata can only be triggered in case vCPU CR4.SMAP=1.
7169          * Second, if vCPU CR4.SMEP=1, errata could only be triggered
7170          * in case vCPU CPL==3 (Because otherwise guest would have triggered
7171          * a SMEP fault instead of #NPF).
7172          * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
7173          * As most guests enable SMAP if they have also enabled SMEP, use above
7174          * logic in order to attempt minimize false-positive of detecting errata
7175          * while still preserving all cases semantic correctness.
7176          *
7177          * Workaround:
7178          * To determine what instruction the guest was executing, the hypervisor
7179          * will have to decode the instruction at the instruction pointer.
7180          *
7181          * In non SEV guest, hypervisor will be able to read the guest
7182          * memory to decode the instruction pointer when insn_len is zero
7183          * so we return true to indicate that decoding is possible.
7184          *
7185          * But in the SEV guest, the guest memory is encrypted with the
7186          * guest specific key and hypervisor will not be able to decode the
7187          * instruction pointer so we will not able to workaround it. Lets
7188          * print the error and request to kill the guest.
7189          */
7190         if (smap && (!smep || is_user)) {
7191                 if (!sev_guest(vcpu->kvm))
7192                         return true;
7193
7194                 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
7195                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7196         }
7197
7198         return false;
7199 }
7200
7201 static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7202         .cpu_has_kvm_support = has_svm,
7203         .disabled_by_bios = is_disabled,
7204         .hardware_setup = svm_hardware_setup,
7205         .hardware_unsetup = svm_hardware_unsetup,
7206         .check_processor_compatibility = svm_check_processor_compat,
7207         .hardware_enable = svm_hardware_enable,
7208         .hardware_disable = svm_hardware_disable,
7209         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
7210         .has_emulated_msr = svm_has_emulated_msr,
7211
7212         .vcpu_create = svm_create_vcpu,
7213         .vcpu_free = svm_free_vcpu,
7214         .vcpu_reset = svm_vcpu_reset,
7215
7216         .vm_alloc = svm_vm_alloc,
7217         .vm_free = svm_vm_free,
7218         .vm_init = avic_vm_init,
7219         .vm_destroy = svm_vm_destroy,
7220
7221         .prepare_guest_switch = svm_prepare_guest_switch,
7222         .vcpu_load = svm_vcpu_load,
7223         .vcpu_put = svm_vcpu_put,
7224         .vcpu_blocking = svm_vcpu_blocking,
7225         .vcpu_unblocking = svm_vcpu_unblocking,
7226
7227         .update_bp_intercept = update_bp_intercept,
7228         .get_msr_feature = svm_get_msr_feature,
7229         .get_msr = svm_get_msr,
7230         .set_msr = svm_set_msr,
7231         .get_segment_base = svm_get_segment_base,
7232         .get_segment = svm_get_segment,
7233         .set_segment = svm_set_segment,
7234         .get_cpl = svm_get_cpl,
7235         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
7236         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
7237         .decache_cr3 = svm_decache_cr3,
7238         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
7239         .set_cr0 = svm_set_cr0,
7240         .set_cr3 = svm_set_cr3,
7241         .set_cr4 = svm_set_cr4,
7242         .set_efer = svm_set_efer,
7243         .get_idt = svm_get_idt,
7244         .set_idt = svm_set_idt,
7245         .get_gdt = svm_get_gdt,
7246         .set_gdt = svm_set_gdt,
7247         .get_dr6 = svm_get_dr6,
7248         .set_dr6 = svm_set_dr6,
7249         .set_dr7 = svm_set_dr7,
7250         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
7251         .cache_reg = svm_cache_reg,
7252         .get_rflags = svm_get_rflags,
7253         .set_rflags = svm_set_rflags,
7254
7255         .tlb_flush = svm_flush_tlb,
7256         .tlb_flush_gva = svm_flush_tlb_gva,
7257
7258         .run = svm_vcpu_run,
7259         .handle_exit = handle_exit,
7260         .skip_emulated_instruction = skip_emulated_instruction,
7261         .set_interrupt_shadow = svm_set_interrupt_shadow,
7262         .get_interrupt_shadow = svm_get_interrupt_shadow,
7263         .patch_hypercall = svm_patch_hypercall,
7264         .set_irq = svm_set_irq,
7265         .set_nmi = svm_inject_nmi,
7266         .queue_exception = svm_queue_exception,
7267         .cancel_injection = svm_cancel_injection,
7268         .interrupt_allowed = svm_interrupt_allowed,
7269         .nmi_allowed = svm_nmi_allowed,
7270         .get_nmi_mask = svm_get_nmi_mask,
7271         .set_nmi_mask = svm_set_nmi_mask,
7272         .enable_nmi_window = enable_nmi_window,
7273         .enable_irq_window = enable_irq_window,
7274         .update_cr8_intercept = update_cr8_intercept,
7275         .set_virtual_apic_mode = svm_set_virtual_apic_mode,
7276         .get_enable_apicv = svm_get_enable_apicv,
7277         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
7278         .load_eoi_exitmap = svm_load_eoi_exitmap,
7279         .hwapic_irr_update = svm_hwapic_irr_update,
7280         .hwapic_isr_update = svm_hwapic_isr_update,
7281         .sync_pir_to_irr = kvm_lapic_find_highest_irr,
7282         .apicv_post_state_restore = avic_post_state_restore,
7283
7284         .set_tss_addr = svm_set_tss_addr,
7285         .set_identity_map_addr = svm_set_identity_map_addr,
7286         .get_tdp_level = get_npt_level,
7287         .get_mt_mask = svm_get_mt_mask,
7288
7289         .get_exit_info = svm_get_exit_info,
7290
7291         .get_lpage_level = svm_get_lpage_level,
7292
7293         .cpuid_update = svm_cpuid_update,
7294
7295         .rdtscp_supported = svm_rdtscp_supported,
7296         .invpcid_supported = svm_invpcid_supported,
7297         .mpx_supported = svm_mpx_supported,
7298         .xsaves_supported = svm_xsaves_supported,
7299         .umip_emulated = svm_umip_emulated,
7300         .pt_supported = svm_pt_supported,
7301
7302         .set_supported_cpuid = svm_set_supported_cpuid,
7303
7304         .has_wbinvd_exit = svm_has_wbinvd_exit,
7305
7306         .read_l1_tsc_offset = svm_read_l1_tsc_offset,
7307         .write_l1_tsc_offset = svm_write_l1_tsc_offset,
7308
7309         .set_tdp_cr3 = set_tdp_cr3,
7310
7311         .check_intercept = svm_check_intercept,
7312         .handle_exit_irqoff = svm_handle_exit_irqoff,
7313
7314         .request_immediate_exit = __kvm_request_immediate_exit,
7315
7316         .sched_in = svm_sched_in,
7317
7318         .pmu_ops = &amd_pmu_ops,
7319         .deliver_posted_interrupt = svm_deliver_avic_intr,
7320         .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
7321         .update_pi_irte = svm_update_pi_irte,
7322         .setup_mce = svm_setup_mce,
7323
7324         .smi_allowed = svm_smi_allowed,
7325         .pre_enter_smm = svm_pre_enter_smm,
7326         .pre_leave_smm = svm_pre_leave_smm,
7327         .enable_smi_window = enable_smi_window,
7328
7329         .mem_enc_op = svm_mem_enc_op,
7330         .mem_enc_reg_region = svm_register_enc_region,
7331         .mem_enc_unreg_region = svm_unregister_enc_region,
7332
7333         .nested_enable_evmcs = nested_enable_evmcs,
7334         .nested_get_evmcs_version = nested_get_evmcs_version,
7335
7336         .need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
7337 };
7338
7339 static int __init svm_init(void)
7340 {
7341         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
7342                         __alignof__(struct vcpu_svm), THIS_MODULE);
7343 }
7344
7345 static void __exit svm_exit(void)
7346 {
7347         kvm_exit();
7348 }
7349
7350 module_init(svm_init)
7351 module_exit(svm_exit)