arch/x86/kvm/svm/sev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM-SEV support
   6  *
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  */
   9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11 #include <linux/kvm_types.h>
  12 #include <linux/kvm_host.h>
  13 #include <linux/kernel.h>
  14 #include <linux/highmem.h>
  15 #include <linux/psp.h>
  16 #include <linux/psp-sev.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/swap.h>
  19 #include <linux/misc_cgroup.h>
  20 #include <linux/processor.h>
  21 #include <linux/trace_events.h>
  22
  23 #include <asm/pkru.h>
  24 #include <asm/trapnr.h>
  25 #include <asm/fpu/xcr.h>
  26
  27 #include "mmu.h"
  28 #include "x86.h"
  29 #include "svm.h"
  30 #include "svm_ops.h"
  31 #include "cpuid.h"
  32 #include "trace.h"
  33
  34 #ifndef CONFIG_KVM_AMD_SEV
  35 /*
  36  * When this config is not defined, SEV feature is not supported and APIs in
  37  * this file are not used but this file still gets compiled into the KVM AMD
  38  * module.
  39  *
  40  * We will not have MISC_CG_RES_SEV and MISC_CG_RES_SEV_ES entries in the enum
  41  * misc_res_type {} defined in linux/misc_cgroup.h.
  42  *
  43  * Below macros allow compilation to succeed.
  44  */
  45 #define MISC_CG_RES_SEV MISC_CG_RES_TYPES
  46 #define MISC_CG_RES_SEV_ES MISC_CG_RES_TYPES
  47 #endif
  48
  49 #ifdef CONFIG_KVM_AMD_SEV
  50 /* enable/disable SEV support */
  51 static bool sev_enabled = true;
  52 module_param_named(sev, sev_enabled, bool, 0444);
  53
  54 /* enable/disable SEV-ES support */
  55 static bool sev_es_enabled = true;
  56 module_param_named(sev_es, sev_es_enabled, bool, 0444);
  57 #else
  58 #define sev_enabled false
  59 #define sev_es_enabled false
  60 #endif /* CONFIG_KVM_AMD_SEV */
  61
  62 static u8 sev_enc_bit;
  63 static DECLARE_RWSEM(sev_deactivate_lock);
  64 static DEFINE_MUTEX(sev_bitmap_lock);
  65 unsigned int max_sev_asid;
  66 static unsigned int min_sev_asid;
  67 static unsigned long sev_me_mask;
  68 static unsigned int nr_asids;
  69 static unsigned long *sev_asid_bitmap;
  70 static unsigned long *sev_reclaim_asid_bitmap;
  71
  72 struct enc_region {
  73         struct list_head list;
  74         unsigned long npages;
  75         struct page **pages;
  76         unsigned long uaddr;
  77         unsigned long size;
  78 };
  79
  80 /* Called with the sev_bitmap_lock held, or on shutdown  */
  81 static int sev_flush_asids(int min_asid, int max_asid)
  82 {
  83         int ret, asid, error = 0;
  84
  85         /* Check if there are any ASIDs to reclaim before performing a flush */
  86         asid = find_next_bit(sev_reclaim_asid_bitmap, nr_asids, min_asid);
  87         if (asid > max_asid)
  88                 return -EBUSY;
  89
  90         /*
  91          * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail,
  92          * so it must be guarded.
  93          */
  94         down_write(&sev_deactivate_lock);
  95
  96         wbinvd_on_all_cpus();
  97         ret = sev_guest_df_flush(&error);
  98
  99         up_write(&sev_deactivate_lock);
 100
 101         if (ret)
 102                 pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error);
 103
 104         return ret;
 105 }
 106
 107 static inline bool is_mirroring_enc_context(struct kvm *kvm)
 108 {
 109         return !!to_kvm_svm(kvm)->sev_info.enc_context_owner;
 110 }
 111
 112 /* Must be called with the sev_bitmap_lock held */
 113 static bool __sev_recycle_asids(int min_asid, int max_asid)
 114 {
 115         if (sev_flush_asids(min_asid, max_asid))
 116                 return false;
 117
 118         /* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */
 119         bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
 120                    nr_asids);
 121         bitmap_zero(sev_reclaim_asid_bitmap, nr_asids);
 122
 123         return true;
 124 }
 125
 126 static int sev_misc_cg_try_charge(struct kvm_sev_info *sev)
 127 {
 128         enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV;
 129         return misc_cg_try_charge(type, sev->misc_cg, 1);
 130 }
 131
 132 static void sev_misc_cg_uncharge(struct kvm_sev_info *sev)
 133 {
 134         enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV;
 135         misc_cg_uncharge(type, sev->misc_cg, 1);
 136 }
 137
 138 static int sev_asid_new(struct kvm_sev_info *sev)
 139 {
 140         int asid, min_asid, max_asid, ret;
 141         bool retry = true;
 142
 143         WARN_ON(sev->misc_cg);
 144         sev->misc_cg = get_current_misc_cg();
 145         ret = sev_misc_cg_try_charge(sev);
 146         if (ret) {
 147                 put_misc_cg(sev->misc_cg);
 148                 sev->misc_cg = NULL;
 149                 return ret;
 150         }
 151
 152         mutex_lock(&sev_bitmap_lock);
 153
 154         /*
 155          * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid.
 156          * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1.
 157          */
 158         min_asid = sev->es_active ? 1 : min_sev_asid;
 159         max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid;
 160 again:
 161         asid = find_next_zero_bit(sev_asid_bitmap, max_asid + 1, min_asid);
 162         if (asid > max_asid) {
 163                 if (retry && __sev_recycle_asids(min_asid, max_asid)) {
 164                         retry = false;
 165                         goto again;
 166                 }
 167                 mutex_unlock(&sev_bitmap_lock);
 168                 ret = -EBUSY;
 169                 goto e_uncharge;
 170         }
 171
 172         __set_bit(asid, sev_asid_bitmap);
 173
 174         mutex_unlock(&sev_bitmap_lock);
 175
 176         return asid;
 177 e_uncharge:
 178         sev_misc_cg_uncharge(sev);
 179         put_misc_cg(sev->misc_cg);
 180         sev->misc_cg = NULL;
 181         return ret;
 182 }
 183
 184 static int sev_get_asid(struct kvm *kvm)
 185 {
 186         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 187
 188         return sev->asid;
 189 }
 190
 191 static void sev_asid_free(struct kvm_sev_info *sev)
 192 {
 193         struct svm_cpu_data *sd;
 194         int cpu;
 195
 196         mutex_lock(&sev_bitmap_lock);
 197
 198         __set_bit(sev->asid, sev_reclaim_asid_bitmap);
 199
 200         for_each_possible_cpu(cpu) {
 201                 sd = per_cpu_ptr(&svm_data, cpu);
 202                 sd->sev_vmcbs[sev->asid] = NULL;
 203         }
 204
 205         mutex_unlock(&sev_bitmap_lock);
 206
 207         sev_misc_cg_uncharge(sev);
 208         put_misc_cg(sev->misc_cg);
 209         sev->misc_cg = NULL;
 210 }
 211
 212 static void sev_decommission(unsigned int handle)
 213 {
 214         struct sev_data_decommission decommission;
 215
 216         if (!handle)
 217                 return;
 218
 219         decommission.handle = handle;
 220         sev_guest_decommission(&decommission, NULL);
 221 }
 222
 223 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
 224 {
 225         struct sev_data_deactivate deactivate;
 226
 227         if (!handle)
 228                 return;
 229
 230         deactivate.handle = handle;
 231
 232         /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */
 233         down_read(&sev_deactivate_lock);
 234         sev_guest_deactivate(&deactivate, NULL);
 235         up_read(&sev_deactivate_lock);
 236
 237         sev_decommission(handle);
 238 }
 239
 240 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
 241 {
 242         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 243         int asid, ret;
 244
 245         if (kvm->created_vcpus)
 246                 return -EINVAL;
 247
 248         ret = -EBUSY;
 249         if (unlikely(sev->active))
 250                 return ret;
 251
 252         sev->active = true;
 253         sev->es_active = argp->id == KVM_SEV_ES_INIT;
 254         asid = sev_asid_new(sev);
 255         if (asid < 0)
 256                 goto e_no_asid;
 257         sev->asid = asid;
 258
 259         ret = sev_platform_init(&argp->error);
 260         if (ret)
 261                 goto e_free;
 262
 263         INIT_LIST_HEAD(&sev->regions_list);
 264         INIT_LIST_HEAD(&sev->mirror_vms);
 265
 266         kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV);
 267
 268         return 0;
 269
 270 e_free:
 271         sev_asid_free(sev);
 272         sev->asid = 0;
 273 e_no_asid:
 274         sev->es_active = false;
 275         sev->active = false;
 276         return ret;
 277 }
 278
 279 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
 280 {
 281         struct sev_data_activate activate;
 282         int asid = sev_get_asid(kvm);
 283         int ret;
 284
 285         /* activate ASID on the given handle */
 286         activate.handle = handle;
 287         activate.asid   = asid;
 288         ret = sev_guest_activate(&activate, error);
 289
 290         return ret;
 291 }
 292
 293 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
 294 {
 295         struct fd f;
 296         int ret;
 297
 298         f = fdget(fd);
 299         if (!f.file)
 300                 return -EBADF;
 301
 302         ret = sev_issue_cmd_external_user(f.file, id, data, error);
 303
 304         fdput(f);
 305         return ret;
 306 }
 307
 308 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
 309 {
 310         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 311
 312         return __sev_issue_cmd(sev->fd, id, data, error);
 313 }
 314
 315 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
 316 {
 317         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 318         struct sev_data_launch_start start;
 319         struct kvm_sev_launch_start params;
 320         void *dh_blob, *session_blob;
 321         int *error = &argp->error;
 322         int ret;
 323
 324         if (!sev_guest(kvm))
 325                 return -ENOTTY;
 326
 327         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
 328                 return -EFAULT;
 329
 330         memset(&start, 0, sizeof(start));
 331
 332         dh_blob = NULL;
 333         if (params.dh_uaddr) {
 334                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
 335                 if (IS_ERR(dh_blob))
 336                         return PTR_ERR(dh_blob);
 337
 338                 start.dh_cert_address = __sme_set(__pa(dh_blob));
 339                 start.dh_cert_len = params.dh_len;
 340         }
 341
 342         session_blob = NULL;
 343         if (params.session_uaddr) {
 344                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
 345                 if (IS_ERR(session_blob)) {
 346                         ret = PTR_ERR(session_blob);
 347                         goto e_free_dh;
 348                 }
 349
 350                 start.session_address = __sme_set(__pa(session_blob));
 351                 start.session_len = params.session_len;
 352         }
 353
 354         start.handle = params.handle;
 355         start.policy = params.policy;
 356
 357         /* create memory encryption context */
 358         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, &start, error);
 359         if (ret)
 360                 goto e_free_session;
 361
 362         /* Bind ASID to this guest */
 363         ret = sev_bind_asid(kvm, start.handle, error);
 364         if (ret) {
 365                 sev_decommission(start.handle);
 366                 goto e_free_session;
 367         }
 368
 369         /* return handle to userspace */
 370         params.handle = start.handle;
 371         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
 372                 sev_unbind_asid(kvm, start.handle);
 373                 ret = -EFAULT;
 374                 goto e_free_session;
 375         }
 376
 377         sev->handle = start.handle;
 378         sev->fd = argp->sev_fd;
 379
 380 e_free_session:
 381         kfree(session_blob);
 382 e_free_dh:
 383         kfree(dh_blob);
 384         return ret;
 385 }
 386
 387 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
 388                                     unsigned long ulen, unsigned long *n,
 389                                     int write)
 390 {
 391         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 392         unsigned long npages, size;
 393         int npinned;
 394         unsigned long locked, lock_limit;
 395         struct page **pages;
 396         unsigned long first, last;
 397         int ret;
 398
 399         lockdep_assert_held(&kvm->lock);
 400
 401         if (ulen == 0 || uaddr + ulen < uaddr)
 402                 return ERR_PTR(-EINVAL);
 403
 404         /* Calculate number of pages. */
 405         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
 406         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
 407         npages = (last - first + 1);
 408
 409         locked = sev->pages_locked + npages;
 410         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 411         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
 412                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
 413                 return ERR_PTR(-ENOMEM);
 414         }
 415
 416         if (WARN_ON_ONCE(npages > INT_MAX))
 417                 return ERR_PTR(-EINVAL);
 418
 419         /* Avoid using vmalloc for smaller buffers. */
 420         size = npages * sizeof(struct page *);
 421         if (size > PAGE_SIZE)
 422                 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 423         else
 424                 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
 425
 426         if (!pages)
 427                 return ERR_PTR(-ENOMEM);
 428
 429         /* Pin the user virtual address. */
 430         npinned = pin_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
 431         if (npinned != npages) {
 432                 pr_err("SEV: Failure locking %lu pages.\n", npages);
 433                 ret = -ENOMEM;
 434                 goto err;
 435         }
 436
 437         *n = npages;
 438         sev->pages_locked = locked;
 439
 440         return pages;
 441
 442 err:
 443         if (npinned > 0)
 444                 unpin_user_pages(pages, npinned);
 445
 446         kvfree(pages);
 447         return ERR_PTR(ret);
 448 }
 449
 450 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
 451                              unsigned long npages)
 452 {
 453         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 454
 455         unpin_user_pages(pages, npages);
 456         kvfree(pages);
 457         sev->pages_locked -= npages;
 458 }
 459
 460 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
 461 {
 462         uint8_t *page_virtual;
 463         unsigned long i;
 464
 465         if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 ||
 466             pages == NULL)
 467                 return;
 468
 469         for (i = 0; i < npages; i++) {
 470                 page_virtual = kmap_local_page(pages[i]);
 471                 clflush_cache_range(page_virtual, PAGE_SIZE);
 472                 kunmap_local(page_virtual);
 473                 cond_resched();
 474         }
 475 }
 476
 477 static unsigned long get_num_contig_pages(unsigned long idx,
 478                                 struct page **inpages, unsigned long npages)
 479 {
 480         unsigned long paddr, next_paddr;
 481         unsigned long i = idx + 1, pages = 1;
 482
 483         /* find the number of contiguous pages starting from idx */
 484         paddr = __sme_page_pa(inpages[idx]);
 485         while (i < npages) {
 486                 next_paddr = __sme_page_pa(inpages[i++]);
 487                 if ((paddr + PAGE_SIZE) == next_paddr) {
 488                         pages++;
 489                         paddr = next_paddr;
 490                         continue;
 491                 }
 492                 break;
 493         }
 494
 495         return pages;
 496 }
 497
 498 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 499 {
 500         unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
 501         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 502         struct kvm_sev_launch_update_data params;
 503         struct sev_data_launch_update_data data;
 504         struct page **inpages;
 505         int ret;
 506
 507         if (!sev_guest(kvm))
 508                 return -ENOTTY;
 509
 510         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
 511                 return -EFAULT;
 512
 513         vaddr = params.uaddr;
 514         size = params.len;
 515         vaddr_end = vaddr + size;
 516
 517         /* Lock the user memory. */
 518         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
 519         if (IS_ERR(inpages))
 520                 return PTR_ERR(inpages);
 521
 522         /*
 523          * Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in
 524          * place; the cache may contain the data that was written unencrypted.
 525          */
 526         sev_clflush_pages(inpages, npages);
 527
 528         data.reserved = 0;
 529         data.handle = sev->handle;
 530
 531         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
 532                 int offset, len;
 533
 534                 /*
 535                  * If the user buffer is not page-aligned, calculate the offset
 536                  * within the page.
 537                  */
 538                 offset = vaddr & (PAGE_SIZE - 1);
 539
 540                 /* Calculate the number of pages that can be encrypted in one go. */
 541                 pages = get_num_contig_pages(i, inpages, npages);
 542
 543                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
 544
 545                 data.len = len;
 546                 data.address = __sme_page_pa(inpages[i]) + offset;
 547                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, &data, &argp->error);
 548                 if (ret)
 549                         goto e_unpin;
 550
 551                 size -= len;
 552                 next_vaddr = vaddr + len;
 553         }
 554
 555 e_unpin:
 556         /* content of memory is updated, mark pages dirty */
 557         for (i = 0; i < npages; i++) {
 558                 set_page_dirty_lock(inpages[i]);
 559                 mark_page_accessed(inpages[i]);
 560         }
 561         /* unlock the user pages */
 562         sev_unpin_memory(kvm, inpages, npages);
 563         return ret;
 564 }
 565
 566 static int sev_es_sync_vmsa(struct vcpu_svm *svm)
 567 {
 568         struct sev_es_save_area *save = svm->sev_es.vmsa;
 569
 570         /* Check some debug related fields before encrypting the VMSA */
 571         if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1))
 572                 return -EINVAL;
 573
 574         /*
 575          * SEV-ES will use a VMSA that is pointed to by the VMCB, not
 576          * the traditional VMSA that is part of the VMCB. Copy the
 577          * traditional VMSA as it has been built so far (in prep
 578          * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state.
 579          */
 580         memcpy(save, &svm->vmcb->save, sizeof(svm->vmcb->save));
 581
 582         /* Sync registgers */
 583         save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX];
 584         save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX];
 585         save->rcx = svm->vcpu.arch.regs[VCPU_REGS_RCX];
 586         save->rdx = svm->vcpu.arch.regs[VCPU_REGS_RDX];
 587         save->rsp = svm->vcpu.arch.regs[VCPU_REGS_RSP];
 588         save->rbp = svm->vcpu.arch.regs[VCPU_REGS_RBP];
 589         save->rsi = svm->vcpu.arch.regs[VCPU_REGS_RSI];
 590         save->rdi = svm->vcpu.arch.regs[VCPU_REGS_RDI];
 591 #ifdef CONFIG_X86_64
 592         save->r8  = svm->vcpu.arch.regs[VCPU_REGS_R8];
 593         save->r9  = svm->vcpu.arch.regs[VCPU_REGS_R9];
 594         save->r10 = svm->vcpu.arch.regs[VCPU_REGS_R10];
 595         save->r11 = svm->vcpu.arch.regs[VCPU_REGS_R11];
 596         save->r12 = svm->vcpu.arch.regs[VCPU_REGS_R12];
 597         save->r13 = svm->vcpu.arch.regs[VCPU_REGS_R13];
 598         save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14];
 599         save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15];
 600 #endif
 601         save->rip = svm->vcpu.arch.regs[VCPU_REGS_RIP];
 602
 603         /* Sync some non-GPR registers before encrypting */
 604         save->xcr0 = svm->vcpu.arch.xcr0;
 605         save->pkru = svm->vcpu.arch.pkru;
 606         save->xss  = svm->vcpu.arch.ia32_xss;
 607         save->dr6  = svm->vcpu.arch.dr6;
 608
 609         pr_debug("Virtual Machine Save Area (VMSA):\n");
 610         print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false);
 611
 612         return 0;
 613 }
 614
 615 static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu,
 616                                     int *error)
 617 {
 618         struct sev_data_launch_update_vmsa vmsa;
 619         struct vcpu_svm *svm = to_svm(vcpu);
 620         int ret;
 621
 622         /* Perform some pre-encryption checks against the VMSA */
 623         ret = sev_es_sync_vmsa(svm);
 624         if (ret)
 625                 return ret;
 626
 627         /*
 628          * The LAUNCH_UPDATE_VMSA command will perform in-place encryption of
 629          * the VMSA memory content (i.e it will write the same memory region
 630          * with the guest's key), so invalidate it first.
 631          */
 632         clflush_cache_range(svm->sev_es.vmsa, PAGE_SIZE);
 633
 634         vmsa.reserved = 0;
 635         vmsa.handle = to_kvm_svm(kvm)->sev_info.handle;
 636         vmsa.address = __sme_pa(svm->sev_es.vmsa);
 637         vmsa.len = PAGE_SIZE;
 638         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_VMSA, &vmsa, error);
 639         if (ret)
 640           return ret;
 641
 642         vcpu->arch.guest_state_protected = true;
 643         return 0;
 644 }
 645
 646 static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
 647 {
 648         struct kvm_vcpu *vcpu;
 649         unsigned long i;
 650         int ret;
 651
 652         if (!sev_es_guest(kvm))
 653                 return -ENOTTY;
 654
 655         kvm_for_each_vcpu(i, vcpu, kvm) {
 656                 ret = mutex_lock_killable(&vcpu->mutex);
 657                 if (ret)
 658                         return ret;
 659
 660                 ret = __sev_launch_update_vmsa(kvm, vcpu, &argp->error);
 661
 662                 mutex_unlock(&vcpu->mutex);
 663                 if (ret)
 664                         return ret;
 665         }
 666
 667         return 0;
 668 }
 669
 670 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
 671 {
 672         void __user *measure = (void __user *)(uintptr_t)argp->data;
 673         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 674         struct sev_data_launch_measure data;
 675         struct kvm_sev_launch_measure params;
 676         void __user *p = NULL;
 677         void *blob = NULL;
 678         int ret;
 679
 680         if (!sev_guest(kvm))
 681                 return -ENOTTY;
 682
 683         if (copy_from_user(&params, measure, sizeof(params)))
 684                 return -EFAULT;
 685
 686         memset(&data, 0, sizeof(data));
 687
 688         /* User wants to query the blob length */
 689         if (!params.len)
 690                 goto cmd;
 691
 692         p = (void __user *)(uintptr_t)params.uaddr;
 693         if (p) {
 694                 if (params.len > SEV_FW_BLOB_MAX_SIZE)
 695                         return -EINVAL;
 696
 697                 blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT);
 698                 if (!blob)
 699                         return -ENOMEM;
 700
 701                 data.address = __psp_pa(blob);
 702                 data.len = params.len;
 703         }
 704
 705 cmd:
 706         data.handle = sev->handle;
 707         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, &data, &argp->error);
 708
 709         /*
 710          * If we query the session length, FW responded with expected data.
 711          */
 712         if (!params.len)
 713                 goto done;
 714
 715         if (ret)
 716                 goto e_free_blob;
 717
 718         if (blob) {
 719                 if (copy_to_user(p, blob, params.len))
 720                         ret = -EFAULT;
 721         }
 722
 723 done:
 724         params.len = data.len;
 725         if (copy_to_user(measure, &params, sizeof(params)))
 726                 ret = -EFAULT;
 727 e_free_blob:
 728         kfree(blob);
 729         return ret;
 730 }
 731
 732 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
 733 {
 734         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 735         struct sev_data_launch_finish data;
 736
 737         if (!sev_guest(kvm))
 738                 return -ENOTTY;
 739
 740         data.handle = sev->handle;
 741         return sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, &data, &argp->error);
 742 }
 743
 744 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
 745 {
 746         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 747         struct kvm_sev_guest_status params;
 748         struct sev_data_guest_status data;
 749         int ret;
 750
 751         if (!sev_guest(kvm))
 752                 return -ENOTTY;
 753
 754         memset(&data, 0, sizeof(data));
 755
 756         data.handle = sev->handle;
 757         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, &data, &argp->error);
 758         if (ret)
 759                 return ret;
 760
 761         params.policy = data.policy;
 762         params.state = data.state;
 763         params.handle = data.handle;
 764
 765         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
 766                 ret = -EFAULT;
 767
 768         return ret;
 769 }
 770
 771 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
 772                                unsigned long dst, int size,
 773                                int *error, bool enc)
 774 {
 775         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 776         struct sev_data_dbg data;
 777
 778         data.reserved = 0;
 779         data.handle = sev->handle;
 780         data.dst_addr = dst;
 781         data.src_addr = src;
 782         data.len = size;
 783
 784         return sev_issue_cmd(kvm,
 785                              enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
 786                              &data, error);
 787 }
 788
 789 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
 790                              unsigned long dst_paddr, int sz, int *err)
 791 {
 792         int offset;
 793
 794         /*
 795          * Its safe to read more than we are asked, caller should ensure that
 796          * destination has enough space.
 797          */
 798         offset = src_paddr & 15;
 799         src_paddr = round_down(src_paddr, 16);
 800         sz = round_up(sz + offset, 16);
 801
 802         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
 803 }
 804
 805 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
 806                                   void __user *dst_uaddr,
 807                                   unsigned long dst_paddr,
 808                                   int size, int *err)
 809 {
 810         struct page *tpage = NULL;
 811         int ret, offset;
 812
 813         /* if inputs are not 16-byte then use intermediate buffer */
 814         if (!IS_ALIGNED(dst_paddr, 16) ||
 815             !IS_ALIGNED(paddr,     16) ||
 816             !IS_ALIGNED(size,      16)) {
 817                 tpage = (void *)alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 818                 if (!tpage)
 819                         return -ENOMEM;
 820
 821                 dst_paddr = __sme_page_pa(tpage);
 822         }
 823
 824         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
 825         if (ret)
 826                 goto e_free;
 827
 828         if (tpage) {
 829                 offset = paddr & 15;
 830                 if (copy_to_user(dst_uaddr, page_address(tpage) + offset, size))
 831                         ret = -EFAULT;
 832         }
 833
 834 e_free:
 835         if (tpage)
 836                 __free_page(tpage);
 837
 838         return ret;
 839 }
 840
 841 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
 842                                   void __user *vaddr,
 843                                   unsigned long dst_paddr,
 844                                   void __user *dst_vaddr,
 845                                   int size, int *error)
 846 {
 847         struct page *src_tpage = NULL;
 848         struct page *dst_tpage = NULL;
 849         int ret, len = size;
 850
 851         /* If source buffer is not aligned then use an intermediate buffer */
 852         if (!IS_ALIGNED((unsigned long)vaddr, 16)) {
 853                 src_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
 854                 if (!src_tpage)
 855                         return -ENOMEM;
 856
 857                 if (copy_from_user(page_address(src_tpage), vaddr, size)) {
 858                         __free_page(src_tpage);
 859                         return -EFAULT;
 860                 }
 861
 862                 paddr = __sme_page_pa(src_tpage);
 863         }
 864
 865         /*
 866          *  If destination buffer or length is not aligned then do read-modify-write:
 867          *   - decrypt destination in an intermediate buffer
 868          *   - copy the source buffer in an intermediate buffer
 869          *   - use the intermediate buffer as source buffer
 870          */
 871         if (!IS_ALIGNED((unsigned long)dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
 872                 int dst_offset;
 873
 874                 dst_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
 875                 if (!dst_tpage) {
 876                         ret = -ENOMEM;
 877                         goto e_free;
 878                 }
 879
 880                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
 881                                         __sme_page_pa(dst_tpage), size, error);
 882                 if (ret)
 883                         goto e_free;
 884
 885                 /*
 886                  *  If source is kernel buffer then use memcpy() otherwise
 887                  *  copy_from_user().
 888                  */
 889                 dst_offset = dst_paddr & 15;
 890
 891                 if (src_tpage)
 892                         memcpy(page_address(dst_tpage) + dst_offset,
 893                                page_address(src_tpage), size);
 894                 else {
 895                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
 896                                            vaddr, size)) {
 897                                 ret = -EFAULT;
 898                                 goto e_free;
 899                         }
 900                 }
 901
 902                 paddr = __sme_page_pa(dst_tpage);
 903                 dst_paddr = round_down(dst_paddr, 16);
 904                 len = round_up(size, 16);
 905         }
 906
 907         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
 908
 909 e_free:
 910         if (src_tpage)
 911                 __free_page(src_tpage);
 912         if (dst_tpage)
 913                 __free_page(dst_tpage);
 914         return ret;
 915 }
 916
 917 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
 918 {
 919         unsigned long vaddr, vaddr_end, next_vaddr;
 920         unsigned long dst_vaddr;
 921         struct page **src_p, **dst_p;
 922         struct kvm_sev_dbg debug;
 923         unsigned long n;
 924         unsigned int size;
 925         int ret;
 926
 927         if (!sev_guest(kvm))
 928                 return -ENOTTY;
 929
 930         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
 931                 return -EFAULT;
 932
 933         if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
 934                 return -EINVAL;
 935         if (!debug.dst_uaddr)
 936                 return -EINVAL;
 937
 938         vaddr = debug.src_uaddr;
 939         size = debug.len;
 940         vaddr_end = vaddr + size;
 941         dst_vaddr = debug.dst_uaddr;
 942
 943         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
 944                 int len, s_off, d_off;
 945
 946                 /* lock userspace source and destination page */
 947                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
 948                 if (IS_ERR(src_p))
 949                         return PTR_ERR(src_p);
 950
 951                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
 952                 if (IS_ERR(dst_p)) {
 953                         sev_unpin_memory(kvm, src_p, n);
 954                         return PTR_ERR(dst_p);
 955                 }
 956
 957                 /*
 958                  * Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify
 959                  * the pages; flush the destination too so that future accesses do not
 960                  * see stale data.
 961                  */
 962                 sev_clflush_pages(src_p, 1);
 963                 sev_clflush_pages(dst_p, 1);
 964
 965                 /*
 966                  * Since user buffer may not be page aligned, calculate the
 967                  * offset within the page.
 968                  */
 969                 s_off = vaddr & ~PAGE_MASK;
 970                 d_off = dst_vaddr & ~PAGE_MASK;
 971                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
 972
 973                 if (dec)
 974                         ret = __sev_dbg_decrypt_user(kvm,
 975                                                      __sme_page_pa(src_p[0]) + s_off,
 976                                                      (void __user *)dst_vaddr,
 977                                                      __sme_page_pa(dst_p[0]) + d_off,
 978                                                      len, &argp->error);
 979                 else
 980                         ret = __sev_dbg_encrypt_user(kvm,
 981                                                      __sme_page_pa(src_p[0]) + s_off,
 982                                                      (void __user *)vaddr,
 983                                                      __sme_page_pa(dst_p[0]) + d_off,
 984                                                      (void __user *)dst_vaddr,
 985                                                      len, &argp->error);
 986
 987                 sev_unpin_memory(kvm, src_p, n);
 988                 sev_unpin_memory(kvm, dst_p, n);
 989
 990                 if (ret)
 991                         goto err;
 992
 993                 next_vaddr = vaddr + len;
 994                 dst_vaddr = dst_vaddr + len;
 995                 size -= len;
 996         }
 997 err:
 998         return ret;
 999 }
1000
1001 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
1002 {
1003         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1004         struct sev_data_launch_secret data;
1005         struct kvm_sev_launch_secret params;
1006         struct page **pages;
1007         void *blob, *hdr;
1008         unsigned long n, i;
1009         int ret, offset;
1010
1011         if (!sev_guest(kvm))
1012                 return -ENOTTY;
1013
1014         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
1015                 return -EFAULT;
1016
1017         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
1018         if (IS_ERR(pages))
1019                 return PTR_ERR(pages);
1020
1021         /*
1022          * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in
1023          * place; the cache may contain the data that was written unencrypted.
1024          */
1025         sev_clflush_pages(pages, n);
1026
1027         /*
1028          * The secret must be copied into contiguous memory region, lets verify
1029          * that userspace memory pages are contiguous before we issue command.
1030          */
1031         if (get_num_contig_pages(0, pages, n) != n) {
1032                 ret = -EINVAL;
1033                 goto e_unpin_memory;
1034         }
1035
1036         memset(&data, 0, sizeof(data));
1037
1038         offset = params.guest_uaddr & (PAGE_SIZE - 1);
1039         data.guest_address = __sme_page_pa(pages[0]) + offset;
1040         data.guest_len = params.guest_len;
1041
1042         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
1043         if (IS_ERR(blob)) {
1044                 ret = PTR_ERR(blob);
1045                 goto e_unpin_memory;
1046         }
1047
1048         data.trans_address = __psp_pa(blob);
1049         data.trans_len = params.trans_len;
1050
1051         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
1052         if (IS_ERR(hdr)) {
1053                 ret = PTR_ERR(hdr);
1054                 goto e_free_blob;
1055         }
1056         data.hdr_address = __psp_pa(hdr);
1057         data.hdr_len = params.hdr_len;
1058
1059         data.handle = sev->handle;
1060         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, &data, &argp->error);
1061
1062         kfree(hdr);
1063
1064 e_free_blob:
1065         kfree(blob);
1066 e_unpin_memory:
1067         /* content of memory is updated, mark pages dirty */
1068         for (i = 0; i < n; i++) {
1069                 set_page_dirty_lock(pages[i]);
1070                 mark_page_accessed(pages[i]);
1071         }
1072         sev_unpin_memory(kvm, pages, n);
1073         return ret;
1074 }
1075
1076 static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp)
1077 {
1078         void __user *report = (void __user *)(uintptr_t)argp->data;
1079         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1080         struct sev_data_attestation_report data;
1081         struct kvm_sev_attestation_report params;
1082         void __user *p;
1083         void *blob = NULL;
1084         int ret;
1085
1086         if (!sev_guest(kvm))
1087                 return -ENOTTY;
1088
1089         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
1090                 return -EFAULT;
1091
1092         memset(&data, 0, sizeof(data));
1093
1094         /* User wants to query the blob length */
1095         if (!params.len)
1096                 goto cmd;
1097
1098         p = (void __user *)(uintptr_t)params.uaddr;
1099         if (p) {
1100                 if (params.len > SEV_FW_BLOB_MAX_SIZE)
1101                         return -EINVAL;
1102
1103                 blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT);
1104                 if (!blob)
1105                         return -ENOMEM;
1106
1107                 data.address = __psp_pa(blob);
1108                 data.len = params.len;
1109                 memcpy(data.mnonce, params.mnonce, sizeof(params.mnonce));
1110         }
1111 cmd:
1112         data.handle = sev->handle;
1113         ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, &data, &argp->error);
1114         /*
1115          * If we query the session length, FW responded with expected data.
1116          */
1117         if (!params.len)
1118                 goto done;
1119
1120         if (ret)
1121                 goto e_free_blob;
1122
1123         if (blob) {
1124                 if (copy_to_user(p, blob, params.len))
1125                         ret = -EFAULT;
1126         }
1127
1128 done:
1129         params.len = data.len;
1130         if (copy_to_user(report, &params, sizeof(params)))
1131                 ret = -EFAULT;
1132 e_free_blob:
1133         kfree(blob);
1134         return ret;
1135 }
1136
1137 /* Userspace wants to query session length. */
1138 static int
1139 __sev_send_start_query_session_length(struct kvm *kvm, struct kvm_sev_cmd *argp,
1140                                       struct kvm_sev_send_start *params)
1141 {
1142         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1143         struct sev_data_send_start data;
1144         int ret;
1145
1146         memset(&data, 0, sizeof(data));
1147         data.handle = sev->handle;
1148         ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error);
1149
1150         params->session_len = data.session_len;
1151         if (copy_to_user((void __user *)(uintptr_t)argp->data, params,
1152                                 sizeof(struct kvm_sev_send_start)))
1153                 ret = -EFAULT;
1154
1155         return ret;
1156 }
1157
1158 static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
1159 {
1160         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1161         struct sev_data_send_start data;
1162         struct kvm_sev_send_start params;
1163         void *amd_certs, *session_data;
1164         void *pdh_cert, *plat_certs;
1165         int ret;
1166
1167         if (!sev_guest(kvm))
1168                 return -ENOTTY;
1169
1170         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data,
1171                                 sizeof(struct kvm_sev_send_start)))
1172                 return -EFAULT;
1173
1174         /* if session_len is zero, userspace wants to query the session length */
1175         if (!params.session_len)
1176                 return __sev_send_start_query_session_length(kvm, argp,
1177                                 &params);
1178
1179         /* some sanity checks */
1180         if (!params.pdh_cert_uaddr || !params.pdh_cert_len ||
1181             !params.session_uaddr || params.session_len > SEV_FW_BLOB_MAX_SIZE)
1182                 return -EINVAL;
1183
1184         /* allocate the memory to hold the session data blob */
1185         session_data = kzalloc(params.session_len, GFP_KERNEL_ACCOUNT);
1186         if (!session_data)
1187                 return -ENOMEM;
1188
1189         /* copy the certificate blobs from userspace */
1190         pdh_cert = psp_copy_user_blob(params.pdh_cert_uaddr,
1191                                 params.pdh_cert_len);
1192         if (IS_ERR(pdh_cert)) {
1193                 ret = PTR_ERR(pdh_cert);
1194                 goto e_free_session;
1195         }
1196
1197         plat_certs = psp_copy_user_blob(params.plat_certs_uaddr,
1198                                 params.plat_certs_len);
1199         if (IS_ERR(plat_certs)) {
1200                 ret = PTR_ERR(plat_certs);
1201                 goto e_free_pdh;
1202         }
1203
1204         amd_certs = psp_copy_user_blob(params.amd_certs_uaddr,
1205                                 params.amd_certs_len);
1206         if (IS_ERR(amd_certs)) {
1207                 ret = PTR_ERR(amd_certs);
1208                 goto e_free_plat_cert;
1209         }
1210
1211         /* populate the FW SEND_START field with system physical address */
1212         memset(&data, 0, sizeof(data));
1213         data.pdh_cert_address = __psp_pa(pdh_cert);
1214         data.pdh_cert_len = params.pdh_cert_len;
1215         data.plat_certs_address = __psp_pa(plat_certs);
1216         data.plat_certs_len = params.plat_certs_len;
1217         data.amd_certs_address = __psp_pa(amd_certs);
1218         data.amd_certs_len = params.amd_certs_len;
1219         data.session_address = __psp_pa(session_data);
1220         data.session_len = params.session_len;
1221         data.handle = sev->handle;
1222
1223         ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error);
1224
1225         if (!ret && copy_to_user((void __user *)(uintptr_t)params.session_uaddr,
1226                         session_data, params.session_len)) {
1227                 ret = -EFAULT;
1228                 goto e_free_amd_cert;
1229         }
1230
1231         params.policy = data.policy;
1232         params.session_len = data.session_len;
1233         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params,
1234                                 sizeof(struct kvm_sev_send_start)))
1235                 ret = -EFAULT;
1236
1237 e_free_amd_cert:
1238         kfree(amd_certs);
1239 e_free_plat_cert:
1240         kfree(plat_certs);
1241 e_free_pdh:
1242         kfree(pdh_cert);
1243 e_free_session:
1244         kfree(session_data);
1245         return ret;
1246 }
1247
1248 /* Userspace wants to query either header or trans length. */
1249 static int
1250 __sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp,
1251                                      struct kvm_sev_send_update_data *params)
1252 {
1253         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1254         struct sev_data_send_update_data data;
1255         int ret;
1256
1257         memset(&data, 0, sizeof(data));
1258         data.handle = sev->handle;
1259         ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error);
1260
1261         params->hdr_len = data.hdr_len;
1262         params->trans_len = data.trans_len;
1263
1264         if (copy_to_user((void __user *)(uintptr_t)argp->data, params,
1265                          sizeof(struct kvm_sev_send_update_data)))
1266                 ret = -EFAULT;
1267
1268         return ret;
1269 }
1270
1271 static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
1272 {
1273         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1274         struct sev_data_send_update_data data;
1275         struct kvm_sev_send_update_data params;
1276         void *hdr, *trans_data;
1277         struct page **guest_page;
1278         unsigned long n;
1279         int ret, offset;
1280
1281         if (!sev_guest(kvm))
1282                 return -ENOTTY;
1283
1284         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data,
1285                         sizeof(struct kvm_sev_send_update_data)))
1286                 return -EFAULT;
1287
1288         /* userspace wants to query either header or trans length */
1289         if (!params.trans_len || !params.hdr_len)
1290                 return __sev_send_update_data_query_lengths(kvm, argp, &params);
1291
1292         if (!params.trans_uaddr || !params.guest_uaddr ||
1293             !params.guest_len || !params.hdr_uaddr)
1294                 return -EINVAL;
1295
1296         /* Check if we are crossing the page boundary */
1297         offset = params.guest_uaddr & (PAGE_SIZE - 1);
1298         if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE)
1299                 return -EINVAL;
1300
1301         /* Pin guest memory */
1302         guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
1303                                     PAGE_SIZE, &n, 0);
1304         if (IS_ERR(guest_page))
1305                 return PTR_ERR(guest_page);
1306
1307         /* allocate memory for header and transport buffer */
1308         ret = -ENOMEM;
1309         hdr = kzalloc(params.hdr_len, GFP_KERNEL_ACCOUNT);
1310         if (!hdr)
1311                 goto e_unpin;
1312
1313         trans_data = kzalloc(params.trans_len, GFP_KERNEL_ACCOUNT);
1314         if (!trans_data)
1315                 goto e_free_hdr;
1316
1317         memset(&data, 0, sizeof(data));
1318         data.hdr_address = __psp_pa(hdr);
1319         data.hdr_len = params.hdr_len;
1320         data.trans_address = __psp_pa(trans_data);
1321         data.trans_len = params.trans_len;
1322
1323         /* The SEND_UPDATE_DATA command requires C-bit to be always set. */
1324         data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
1325         data.guest_address |= sev_me_mask;
1326         data.guest_len = params.guest_len;
1327         data.handle = sev->handle;
1328
1329         ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error);
1330
1331         if (ret)
1332                 goto e_free_trans_data;
1333
1334         /* copy transport buffer to user space */
1335         if (copy_to_user((void __user *)(uintptr_t)params.trans_uaddr,
1336                          trans_data, params.trans_len)) {
1337                 ret = -EFAULT;
1338                 goto e_free_trans_data;
1339         }
1340
1341         /* Copy packet header to userspace. */
1342         if (copy_to_user((void __user *)(uintptr_t)params.hdr_uaddr, hdr,
1343                          params.hdr_len))
1344                 ret = -EFAULT;
1345
1346 e_free_trans_data:
1347         kfree(trans_data);
1348 e_free_hdr:
1349         kfree(hdr);
1350 e_unpin:
1351         sev_unpin_memory(kvm, guest_page, n);
1352
1353         return ret;
1354 }
1355
1356 static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
1357 {
1358         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1359         struct sev_data_send_finish data;
1360
1361         if (!sev_guest(kvm))
1362                 return -ENOTTY;
1363
1364         data.handle = sev->handle;
1365         return sev_issue_cmd(kvm, SEV_CMD_SEND_FINISH, &data, &argp->error);
1366 }
1367
1368 static int sev_send_cancel(struct kvm *kvm, struct kvm_sev_cmd *argp)
1369 {
1370         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1371         struct sev_data_send_cancel data;
1372
1373         if (!sev_guest(kvm))
1374                 return -ENOTTY;
1375
1376         data.handle = sev->handle;
1377         return sev_issue_cmd(kvm, SEV_CMD_SEND_CANCEL, &data, &argp->error);
1378 }
1379
1380 static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
1381 {
1382         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1383         struct sev_data_receive_start start;
1384         struct kvm_sev_receive_start params;
1385         int *error = &argp->error;
1386         void *session_data;
1387         void *pdh_data;
1388         int ret;
1389
1390         if (!sev_guest(kvm))
1391                 return -ENOTTY;
1392
1393         /* Get parameter from the userspace */
1394         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data,
1395                         sizeof(struct kvm_sev_receive_start)))
1396                 return -EFAULT;
1397
1398         /* some sanity checks */
1399         if (!params.pdh_uaddr || !params.pdh_len ||
1400             !params.session_uaddr || !params.session_len)
1401                 return -EINVAL;
1402
1403         pdh_data = psp_copy_user_blob(params.pdh_uaddr, params.pdh_len);
1404         if (IS_ERR(pdh_data))
1405                 return PTR_ERR(pdh_data);
1406
1407         session_data = psp_copy_user_blob(params.session_uaddr,
1408                         params.session_len);
1409         if (IS_ERR(session_data)) {
1410                 ret = PTR_ERR(session_data);
1411                 goto e_free_pdh;
1412         }
1413
1414         memset(&start, 0, sizeof(start));
1415         start.handle = params.handle;
1416         start.policy = params.policy;
1417         start.pdh_cert_address = __psp_pa(pdh_data);
1418         start.pdh_cert_len = params.pdh_len;
1419         start.session_address = __psp_pa(session_data);
1420         start.session_len = params.session_len;
1421
1422         /* create memory encryption context */
1423         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_RECEIVE_START, &start,
1424                                 error);
1425         if (ret)
1426                 goto e_free_session;
1427
1428         /* Bind ASID to this guest */
1429         ret = sev_bind_asid(kvm, start.handle, error);
1430         if (ret) {
1431                 sev_decommission(start.handle);
1432                 goto e_free_session;
1433         }
1434
1435         params.handle = start.handle;
1436         if (copy_to_user((void __user *)(uintptr_t)argp->data,
1437                          &params, sizeof(struct kvm_sev_receive_start))) {
1438                 ret = -EFAULT;
1439                 sev_unbind_asid(kvm, start.handle);
1440                 goto e_free_session;
1441         }
1442
1443         sev->handle = start.handle;
1444         sev->fd = argp->sev_fd;
1445
1446 e_free_session:
1447         kfree(session_data);
1448 e_free_pdh:
1449         kfree(pdh_data);
1450
1451         return ret;
1452 }
1453
1454 static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
1455 {
1456         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1457         struct kvm_sev_receive_update_data params;
1458         struct sev_data_receive_update_data data;
1459         void *hdr = NULL, *trans = NULL;
1460         struct page **guest_page;
1461         unsigned long n;
1462         int ret, offset;
1463
1464         if (!sev_guest(kvm))
1465                 return -EINVAL;
1466
1467         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data,
1468                         sizeof(struct kvm_sev_receive_update_data)))
1469                 return -EFAULT;
1470
1471         if (!params.hdr_uaddr || !params.hdr_len ||
1472             !params.guest_uaddr || !params.guest_len ||
1473             !params.trans_uaddr || !params.trans_len)
1474                 return -EINVAL;
1475
1476         /* Check if we are crossing the page boundary */
1477         offset = params.guest_uaddr & (PAGE_SIZE - 1);
1478         if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE)
1479                 return -EINVAL;
1480
1481         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
1482         if (IS_ERR(hdr))
1483                 return PTR_ERR(hdr);
1484
1485         trans = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
1486         if (IS_ERR(trans)) {
1487                 ret = PTR_ERR(trans);
1488                 goto e_free_hdr;
1489         }
1490
1491         memset(&data, 0, sizeof(data));
1492         data.hdr_address = __psp_pa(hdr);
1493         data.hdr_len = params.hdr_len;
1494         data.trans_address = __psp_pa(trans);
1495         data.trans_len = params.trans_len;
1496
1497         /* Pin guest memory */
1498         guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
1499                                     PAGE_SIZE, &n, 1);
1500         if (IS_ERR(guest_page)) {
1501                 ret = PTR_ERR(guest_page);
1502                 goto e_free_trans;
1503         }
1504
1505         /*
1506          * Flush (on non-coherent CPUs) before RECEIVE_UPDATE_DATA, the PSP
1507          * encrypts the written data with the guest's key, and the cache may
1508          * contain dirty, unencrypted data.
1509          */
1510         sev_clflush_pages(guest_page, n);
1511
1512         /* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */
1513         data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
1514         data.guest_address |= sev_me_mask;
1515         data.guest_len = params.guest_len;
1516         data.handle = sev->handle;
1517
1518         ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data,
1519                                 &argp->error);
1520
1521         sev_unpin_memory(kvm, guest_page, n);
1522
1523 e_free_trans:
1524         kfree(trans);
1525 e_free_hdr:
1526         kfree(hdr);
1527
1528         return ret;
1529 }
1530
1531 static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
1532 {
1533         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1534         struct sev_data_receive_finish data;
1535
1536         if (!sev_guest(kvm))
1537                 return -ENOTTY;
1538
1539         data.handle = sev->handle;
1540         return sev_issue_cmd(kvm, SEV_CMD_RECEIVE_FINISH, &data, &argp->error);
1541 }
1542
1543 static bool is_cmd_allowed_from_mirror(u32 cmd_id)
1544 {
1545         /*
1546          * Allow mirrors VM to call KVM_SEV_LAUNCH_UPDATE_VMSA to enable SEV-ES
1547          * active mirror VMs. Also allow the debugging and status commands.
1548          */
1549         if (cmd_id == KVM_SEV_LAUNCH_UPDATE_VMSA ||
1550             cmd_id == KVM_SEV_GUEST_STATUS || cmd_id == KVM_SEV_DBG_DECRYPT ||
1551             cmd_id == KVM_SEV_DBG_ENCRYPT)
1552                 return true;
1553
1554         return false;
1555 }
1556
1557 static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm)
1558 {
1559         struct kvm_sev_info *dst_sev = &to_kvm_svm(dst_kvm)->sev_info;
1560         struct kvm_sev_info *src_sev = &to_kvm_svm(src_kvm)->sev_info;
1561         int r = -EBUSY;
1562
1563         if (dst_kvm == src_kvm)
1564                 return -EINVAL;
1565
1566         /*
1567          * Bail if these VMs are already involved in a migration to avoid
1568          * deadlock between two VMs trying to migrate to/from each other.
1569          */
1570         if (atomic_cmpxchg_acquire(&dst_sev->migration_in_progress, 0, 1))
1571                 return -EBUSY;
1572
1573         if (atomic_cmpxchg_acquire(&src_sev->migration_in_progress, 0, 1))
1574                 goto release_dst;
1575
1576         r = -EINTR;
1577         if (mutex_lock_killable(&dst_kvm->lock))
1578                 goto release_src;
1579         if (mutex_lock_killable_nested(&src_kvm->lock, SINGLE_DEPTH_NESTING))
1580                 goto unlock_dst;
1581         return 0;
1582
1583 unlock_dst:
1584         mutex_unlock(&dst_kvm->lock);
1585 release_src:
1586         atomic_set_release(&src_sev->migration_in_progress, 0);
1587 release_dst:
1588         atomic_set_release(&dst_sev->migration_in_progress, 0);
1589         return r;
1590 }
1591
1592 static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm)
1593 {
1594         struct kvm_sev_info *dst_sev = &to_kvm_svm(dst_kvm)->sev_info;
1595         struct kvm_sev_info *src_sev = &to_kvm_svm(src_kvm)->sev_info;
1596
1597         mutex_unlock(&dst_kvm->lock);
1598         mutex_unlock(&src_kvm->lock);
1599         atomic_set_release(&dst_sev->migration_in_progress, 0);
1600         atomic_set_release(&src_sev->migration_in_progress, 0);
1601 }
1602
1603 /* vCPU mutex subclasses.  */
1604 enum sev_migration_role {
1605         SEV_MIGRATION_SOURCE = 0,
1606         SEV_MIGRATION_TARGET,
1607         SEV_NR_MIGRATION_ROLES,
1608 };
1609
1610 static int sev_lock_vcpus_for_migration(struct kvm *kvm,
1611                                         enum sev_migration_role role)
1612 {
1613         struct kvm_vcpu *vcpu;
1614         unsigned long i, j;
1615
1616         kvm_for_each_vcpu(i, vcpu, kvm) {
1617                 if (mutex_lock_killable_nested(&vcpu->mutex, role))
1618                         goto out_unlock;
1619
1620 #ifdef CONFIG_PROVE_LOCKING
1621                 if (!i)
1622                         /*
1623                          * Reset the role to one that avoids colliding with
1624                          * the role used for the first vcpu mutex.
1625                          */
1626                         role = SEV_NR_MIGRATION_ROLES;
1627                 else
1628                         mutex_release(&vcpu->mutex.dep_map, _THIS_IP_);
1629 #endif
1630         }
1631
1632         return 0;
1633
1634 out_unlock:
1635
1636         kvm_for_each_vcpu(j, vcpu, kvm) {
1637                 if (i == j)
1638                         break;
1639
1640 #ifdef CONFIG_PROVE_LOCKING
1641                 if (j)
1642                         mutex_acquire(&vcpu->mutex.dep_map, role, 0, _THIS_IP_);
1643 #endif
1644
1645                 mutex_unlock(&vcpu->mutex);
1646         }
1647         return -EINTR;
1648 }
1649
1650 static void sev_unlock_vcpus_for_migration(struct kvm *kvm)
1651 {
1652         struct kvm_vcpu *vcpu;
1653         unsigned long i;
1654         bool first = true;
1655
1656         kvm_for_each_vcpu(i, vcpu, kvm) {
1657                 if (first)
1658                         first = false;
1659                 else
1660                         mutex_acquire(&vcpu->mutex.dep_map,
1661                                       SEV_NR_MIGRATION_ROLES, 0, _THIS_IP_);
1662
1663                 mutex_unlock(&vcpu->mutex);
1664         }
1665 }
1666
1667 static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
1668 {
1669         struct kvm_sev_info *dst = &to_kvm_svm(dst_kvm)->sev_info;
1670         struct kvm_sev_info *src = &to_kvm_svm(src_kvm)->sev_info;
1671         struct kvm_vcpu *dst_vcpu, *src_vcpu;
1672         struct vcpu_svm *dst_svm, *src_svm;
1673         struct kvm_sev_info *mirror;
1674         unsigned long i;
1675
1676         dst->active = true;
1677         dst->asid = src->asid;
1678         dst->handle = src->handle;
1679         dst->pages_locked = src->pages_locked;
1680         dst->enc_context_owner = src->enc_context_owner;
1681         dst->es_active = src->es_active;
1682
1683         src->asid = 0;
1684         src->active = false;
1685         src->handle = 0;
1686         src->pages_locked = 0;
1687         src->enc_context_owner = NULL;
1688         src->es_active = false;
1689
1690         list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list);
1691
1692         /*
1693          * If this VM has mirrors, "transfer" each mirror's refcount of the
1694          * source to the destination (this KVM).  The caller holds a reference
1695          * to the source, so there's no danger of use-after-free.
1696          */
1697         list_cut_before(&dst->mirror_vms, &src->mirror_vms, &src->mirror_vms);
1698         list_for_each_entry(mirror, &dst->mirror_vms, mirror_entry) {
1699                 kvm_get_kvm(dst_kvm);
1700                 kvm_put_kvm(src_kvm);
1701                 mirror->enc_context_owner = dst_kvm;
1702         }
1703
1704         /*
1705          * If this VM is a mirror, remove the old mirror from the owners list
1706          * and add the new mirror to the list.
1707          */
1708         if (is_mirroring_enc_context(dst_kvm)) {
1709                 struct kvm_sev_info *owner_sev_info =
1710                         &to_kvm_svm(dst->enc_context_owner)->sev_info;
1711
1712                 list_del(&src->mirror_entry);
1713                 list_add_tail(&dst->mirror_entry, &owner_sev_info->mirror_vms);
1714         }
1715
1716         kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) {
1717                 dst_svm = to_svm(dst_vcpu);
1718
1719                 sev_init_vmcb(dst_svm);
1720
1721                 if (!dst->es_active)
1722                         continue;
1723
1724                 /*
1725                  * Note, the source is not required to have the same number of
1726                  * vCPUs as the destination when migrating a vanilla SEV VM.
1727                  */
1728                 src_vcpu = kvm_get_vcpu(dst_kvm, i);
1729                 src_svm = to_svm(src_vcpu);
1730
1731                 /*
1732                  * Transfer VMSA and GHCB state to the destination.  Nullify and
1733                  * clear source fields as appropriate, the state now belongs to
1734                  * the destination.
1735                  */
1736                 memcpy(&dst_svm->sev_es, &src_svm->sev_es, sizeof(src_svm->sev_es));
1737                 dst_svm->vmcb->control.ghcb_gpa = src_svm->vmcb->control.ghcb_gpa;
1738                 dst_svm->vmcb->control.vmsa_pa = src_svm->vmcb->control.vmsa_pa;
1739                 dst_vcpu->arch.guest_state_protected = true;
1740
1741                 memset(&src_svm->sev_es, 0, sizeof(src_svm->sev_es));
1742                 src_svm->vmcb->control.ghcb_gpa = INVALID_PAGE;
1743                 src_svm->vmcb->control.vmsa_pa = INVALID_PAGE;
1744                 src_vcpu->arch.guest_state_protected = false;
1745         }
1746 }
1747
1748 static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src)
1749 {
1750         struct kvm_vcpu *src_vcpu;
1751         unsigned long i;
1752
1753         if (!sev_es_guest(src))
1754                 return 0;
1755
1756         if (atomic_read(&src->online_vcpus) != atomic_read(&dst->online_vcpus))
1757                 return -EINVAL;
1758
1759         kvm_for_each_vcpu(i, src_vcpu, src) {
1760                 if (!src_vcpu->arch.guest_state_protected)
1761                         return -EINVAL;
1762         }
1763
1764         return 0;
1765 }
1766
1767 int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd)
1768 {
1769         struct kvm_sev_info *dst_sev = &to_kvm_svm(kvm)->sev_info;
1770         struct kvm_sev_info *src_sev, *cg_cleanup_sev;
1771         struct fd f = fdget(source_fd);
1772         struct kvm *source_kvm;
1773         bool charged = false;
1774         int ret;
1775
1776         if (!f.file)
1777                 return -EBADF;
1778
1779         if (!file_is_kvm(f.file)) {
1780                 ret = -EBADF;
1781                 goto out_fput;
1782         }
1783
1784         source_kvm = f.file->private_data;
1785         ret = sev_lock_two_vms(kvm, source_kvm);
1786         if (ret)
1787                 goto out_fput;
1788
1789         if (sev_guest(kvm) || !sev_guest(source_kvm)) {
1790                 ret = -EINVAL;
1791                 goto out_unlock;
1792         }
1793
1794         src_sev = &to_kvm_svm(source_kvm)->sev_info;
1795
1796         dst_sev->misc_cg = get_current_misc_cg();
1797         cg_cleanup_sev = dst_sev;
1798         if (dst_sev->misc_cg != src_sev->misc_cg) {
1799                 ret = sev_misc_cg_try_charge(dst_sev);
1800                 if (ret)
1801                         goto out_dst_cgroup;
1802                 charged = true;
1803         }
1804
1805         ret = sev_lock_vcpus_for_migration(kvm, SEV_MIGRATION_SOURCE);
1806         if (ret)
1807                 goto out_dst_cgroup;
1808         ret = sev_lock_vcpus_for_migration(source_kvm, SEV_MIGRATION_TARGET);
1809         if (ret)
1810                 goto out_dst_vcpu;
1811
1812         ret = sev_check_source_vcpus(kvm, source_kvm);
1813         if (ret)
1814                 goto out_source_vcpu;
1815
1816         sev_migrate_from(kvm, source_kvm);
1817         kvm_vm_dead(source_kvm);
1818         cg_cleanup_sev = src_sev;
1819         ret = 0;
1820
1821 out_source_vcpu:
1822         sev_unlock_vcpus_for_migration(source_kvm);
1823 out_dst_vcpu:
1824         sev_unlock_vcpus_for_migration(kvm);
1825 out_dst_cgroup:
1826         /* Operates on the source on success, on the destination on failure.  */
1827         if (charged)
1828                 sev_misc_cg_uncharge(cg_cleanup_sev);
1829         put_misc_cg(cg_cleanup_sev->misc_cg);
1830         cg_cleanup_sev->misc_cg = NULL;
1831 out_unlock:
1832         sev_unlock_two_vms(kvm, source_kvm);
1833 out_fput:
1834         fdput(f);
1835         return ret;
1836 }
1837
1838 int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
1839 {
1840         struct kvm_sev_cmd sev_cmd;
1841         int r;
1842
1843         if (!sev_enabled)
1844                 return -ENOTTY;
1845
1846         if (!argp)
1847                 return 0;
1848
1849         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
1850                 return -EFAULT;
1851
1852         mutex_lock(&kvm->lock);
1853
1854         /* Only the enc_context_owner handles some memory enc operations. */
1855         if (is_mirroring_enc_context(kvm) &&
1856             !is_cmd_allowed_from_mirror(sev_cmd.id)) {
1857                 r = -EINVAL;
1858                 goto out;
1859         }
1860
1861         switch (sev_cmd.id) {
1862         case KVM_SEV_ES_INIT:
1863                 if (!sev_es_enabled) {
1864                         r = -ENOTTY;
1865                         goto out;
1866                 }
1867                 fallthrough;
1868         case KVM_SEV_INIT:
1869                 r = sev_guest_init(kvm, &sev_cmd);
1870                 break;
1871         case KVM_SEV_LAUNCH_START:
1872                 r = sev_launch_start(kvm, &sev_cmd);
1873                 break;
1874         case KVM_SEV_LAUNCH_UPDATE_DATA:
1875                 r = sev_launch_update_data(kvm, &sev_cmd);
1876                 break;
1877         case KVM_SEV_LAUNCH_UPDATE_VMSA:
1878                 r = sev_launch_update_vmsa(kvm, &sev_cmd);
1879                 break;
1880         case KVM_SEV_LAUNCH_MEASURE:
1881                 r = sev_launch_measure(kvm, &sev_cmd);
1882                 break;
1883         case KVM_SEV_LAUNCH_FINISH:
1884                 r = sev_launch_finish(kvm, &sev_cmd);
1885                 break;
1886         case KVM_SEV_GUEST_STATUS:
1887                 r = sev_guest_status(kvm, &sev_cmd);
1888                 break;
1889         case KVM_SEV_DBG_DECRYPT:
1890                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
1891                 break;
1892         case KVM_SEV_DBG_ENCRYPT:
1893                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
1894                 break;
1895         case KVM_SEV_LAUNCH_SECRET:
1896                 r = sev_launch_secret(kvm, &sev_cmd);
1897                 break;
1898         case KVM_SEV_GET_ATTESTATION_REPORT:
1899                 r = sev_get_attestation_report(kvm, &sev_cmd);
1900                 break;
1901         case KVM_SEV_SEND_START:
1902                 r = sev_send_start(kvm, &sev_cmd);
1903                 break;
1904         case KVM_SEV_SEND_UPDATE_DATA:
1905                 r = sev_send_update_data(kvm, &sev_cmd);
1906                 break;
1907         case KVM_SEV_SEND_FINISH:
1908                 r = sev_send_finish(kvm, &sev_cmd);
1909                 break;
1910         case KVM_SEV_SEND_CANCEL:
1911                 r = sev_send_cancel(kvm, &sev_cmd);
1912                 break;
1913         case KVM_SEV_RECEIVE_START:
1914                 r = sev_receive_start(kvm, &sev_cmd);
1915                 break;
1916         case KVM_SEV_RECEIVE_UPDATE_DATA:
1917                 r = sev_receive_update_data(kvm, &sev_cmd);
1918                 break;
1919         case KVM_SEV_RECEIVE_FINISH:
1920                 r = sev_receive_finish(kvm, &sev_cmd);
1921                 break;
1922         default:
1923                 r = -EINVAL;
1924                 goto out;
1925         }
1926
1927         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
1928                 r = -EFAULT;
1929
1930 out:
1931         mutex_unlock(&kvm->lock);
1932         return r;
1933 }
1934
1935 int sev_mem_enc_register_region(struct kvm *kvm,
1936                                 struct kvm_enc_region *range)
1937 {
1938         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1939         struct enc_region *region;
1940         int ret = 0;
1941
1942         if (!sev_guest(kvm))
1943                 return -ENOTTY;
1944
1945         /* If kvm is mirroring encryption context it isn't responsible for it */
1946         if (is_mirroring_enc_context(kvm))
1947                 return -EINVAL;
1948
1949         if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
1950                 return -EINVAL;
1951
1952         region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
1953         if (!region)
1954                 return -ENOMEM;
1955
1956         mutex_lock(&kvm->lock);
1957         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
1958         if (IS_ERR(region->pages)) {
1959                 ret = PTR_ERR(region->pages);
1960                 mutex_unlock(&kvm->lock);
1961                 goto e_free;
1962         }
1963
1964         region->uaddr = range->addr;
1965         region->size = range->size;
1966
1967         list_add_tail(&region->list, &sev->regions_list);
1968         mutex_unlock(&kvm->lock);
1969
1970         /*
1971          * The guest may change the memory encryption attribute from C=0 -> C=1
1972          * or vice versa for this memory range. Lets make sure caches are
1973          * flushed to ensure that guest data gets written into memory with
1974          * correct C-bit.
1975          */
1976         sev_clflush_pages(region->pages, region->npages);
1977
1978         return ret;
1979
1980 e_free:
1981         kfree(region);
1982         return ret;
1983 }
1984
1985 static struct enc_region *
1986 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
1987 {
1988         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1989         struct list_head *head = &sev->regions_list;
1990         struct enc_region *i;
1991
1992         list_for_each_entry(i, head, list) {
1993                 if (i->uaddr == range->addr &&
1994                     i->size == range->size)
1995                         return i;
1996         }
1997
1998         return NULL;
1999 }
2000
2001 static void __unregister_enc_region_locked(struct kvm *kvm,
2002                                            struct enc_region *region)
2003 {
2004         sev_unpin_memory(kvm, region->pages, region->npages);
2005         list_del(&region->list);
2006         kfree(region);
2007 }
2008
2009 int sev_mem_enc_unregister_region(struct kvm *kvm,
2010                                   struct kvm_enc_region *range)
2011 {
2012         struct enc_region *region;
2013         int ret;
2014
2015         /* If kvm is mirroring encryption context it isn't responsible for it */
2016         if (is_mirroring_enc_context(kvm))
2017                 return -EINVAL;
2018
2019         mutex_lock(&kvm->lock);
2020
2021         if (!sev_guest(kvm)) {
2022                 ret = -ENOTTY;
2023                 goto failed;
2024         }
2025
2026         region = find_enc_region(kvm, range);
2027         if (!region) {
2028                 ret = -EINVAL;
2029                 goto failed;
2030         }
2031
2032         /*
2033          * Ensure that all guest tagged cache entries are flushed before
2034          * releasing the pages back to the system for use. CLFLUSH will
2035          * not do this, so issue a WBINVD.
2036          */
2037         wbinvd_on_all_cpus();
2038
2039         __unregister_enc_region_locked(kvm, region);
2040
2041         mutex_unlock(&kvm->lock);
2042         return 0;
2043
2044 failed:
2045         mutex_unlock(&kvm->lock);
2046         return ret;
2047 }
2048
2049 int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
2050 {
2051         struct fd f = fdget(source_fd);
2052         struct kvm *source_kvm;
2053         struct kvm_sev_info *source_sev, *mirror_sev;
2054         int ret;
2055
2056         if (!f.file)
2057                 return -EBADF;
2058
2059         if (!file_is_kvm(f.file)) {
2060                 ret = -EBADF;
2061                 goto e_source_fput;
2062         }
2063
2064         source_kvm = f.file->private_data;
2065         ret = sev_lock_two_vms(kvm, source_kvm);
2066         if (ret)
2067                 goto e_source_fput;
2068
2069         /*
2070          * Mirrors of mirrors should work, but let's not get silly.  Also
2071          * disallow out-of-band SEV/SEV-ES init if the target is already an
2072          * SEV guest, or if vCPUs have been created.  KVM relies on vCPUs being
2073          * created after SEV/SEV-ES initialization, e.g. to init intercepts.
2074          */
2075         if (sev_guest(kvm) || !sev_guest(source_kvm) ||
2076             is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) {
2077                 ret = -EINVAL;
2078                 goto e_unlock;
2079         }
2080
2081         /*
2082          * The mirror kvm holds an enc_context_owner ref so its asid can't
2083          * disappear until we're done with it
2084          */
2085         source_sev = &to_kvm_svm(source_kvm)->sev_info;
2086         kvm_get_kvm(source_kvm);
2087         mirror_sev = &to_kvm_svm(kvm)->sev_info;
2088         list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms);
2089
2090         /* Set enc_context_owner and copy its encryption context over */
2091         mirror_sev->enc_context_owner = source_kvm;
2092         mirror_sev->active = true;
2093         mirror_sev->asid = source_sev->asid;
2094         mirror_sev->fd = source_sev->fd;
2095         mirror_sev->es_active = source_sev->es_active;
2096         mirror_sev->handle = source_sev->handle;
2097         INIT_LIST_HEAD(&mirror_sev->regions_list);
2098         INIT_LIST_HEAD(&mirror_sev->mirror_vms);
2099         ret = 0;
2100
2101         /*
2102          * Do not copy ap_jump_table. Since the mirror does not share the same
2103          * KVM contexts as the original, and they may have different
2104          * memory-views.
2105          */
2106
2107 e_unlock:
2108         sev_unlock_two_vms(kvm, source_kvm);
2109 e_source_fput:
2110         fdput(f);
2111         return ret;
2112 }
2113
2114 void sev_vm_destroy(struct kvm *kvm)
2115 {
2116         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
2117         struct list_head *head = &sev->regions_list;
2118         struct list_head *pos, *q;
2119
2120         if (!sev_guest(kvm))
2121                 return;
2122
2123         WARN_ON(!list_empty(&sev->mirror_vms));
2124
2125         /* If this is a mirror_kvm release the enc_context_owner and skip sev cleanup */
2126         if (is_mirroring_enc_context(kvm)) {
2127                 struct kvm *owner_kvm = sev->enc_context_owner;
2128
2129                 mutex_lock(&owner_kvm->lock);
2130                 list_del(&sev->mirror_entry);
2131                 mutex_unlock(&owner_kvm->lock);
2132                 kvm_put_kvm(owner_kvm);
2133                 return;
2134         }
2135
2136         /*
2137          * Ensure that all guest tagged cache entries are flushed before
2138          * releasing the pages back to the system for use. CLFLUSH will
2139          * not do this, so issue a WBINVD.
2140          */
2141         wbinvd_on_all_cpus();
2142
2143         /*
2144          * if userspace was terminated before unregistering the memory regions
2145          * then lets unpin all the registered memory.
2146          */
2147         if (!list_empty(head)) {
2148                 list_for_each_safe(pos, q, head) {
2149                         __unregister_enc_region_locked(kvm,
2150                                 list_entry(pos, struct enc_region, list));
2151                         cond_resched();
2152                 }
2153         }
2154
2155         sev_unbind_asid(kvm, sev->handle);
2156         sev_asid_free(sev);
2157 }
2158
2159 void __init sev_set_cpu_caps(void)
2160 {
2161         if (!sev_enabled)
2162                 kvm_cpu_cap_clear(X86_FEATURE_SEV);
2163         if (!sev_es_enabled)
2164                 kvm_cpu_cap_clear(X86_FEATURE_SEV_ES);
2165 }
2166
2167 void __init sev_hardware_setup(void)
2168 {
2169 #ifdef CONFIG_KVM_AMD_SEV
2170         unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count;
2171         bool sev_es_supported = false;
2172         bool sev_supported = false;
2173
2174         if (!sev_enabled || !npt_enabled)
2175                 goto out;
2176
2177         /*
2178          * SEV must obviously be supported in hardware.  Sanity check that the
2179          * CPU supports decode assists, which is mandatory for SEV guests to
2180          * support instruction emulation.
2181          */
2182         if (!boot_cpu_has(X86_FEATURE_SEV) ||
2183             WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS)))
2184                 goto out;
2185
2186         /* Retrieve SEV CPUID information */
2187         cpuid(0x8000001f, &eax, &ebx, &ecx, &edx);
2188
2189         /* Set encryption bit location for SEV-ES guests */
2190         sev_enc_bit = ebx & 0x3f;
2191
2192         /* Maximum number of encrypted guests supported simultaneously */
2193         max_sev_asid = ecx;
2194         if (!max_sev_asid)
2195                 goto out;
2196
2197         /* Minimum ASID value that should be used for SEV guest */
2198         min_sev_asid = edx;
2199         sev_me_mask = 1UL << (ebx & 0x3f);
2200
2201         /*
2202          * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap,
2203          * even though it's never used, so that the bitmap is indexed by the
2204          * actual ASID.
2205          */
2206         nr_asids = max_sev_asid + 1;
2207         sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
2208         if (!sev_asid_bitmap)
2209                 goto out;
2210
2211         sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL);
2212         if (!sev_reclaim_asid_bitmap) {
2213                 bitmap_free(sev_asid_bitmap);
2214                 sev_asid_bitmap = NULL;
2215                 goto out;
2216         }
2217
2218         sev_asid_count = max_sev_asid - min_sev_asid + 1;
2219         WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count));
2220         sev_supported = true;
2221
2222         /* SEV-ES support requested? */
2223         if (!sev_es_enabled)
2224                 goto out;
2225
2226         /*
2227          * SEV-ES requires MMIO caching as KVM doesn't have access to the guest
2228          * instruction stream, i.e. can't emulate in response to a #NPF and
2229          * instead relies on #NPF(RSVD) being reflected into the guest as #VC
2230          * (the guest can then do a #VMGEXIT to request MMIO emulation).
2231          */
2232         if (!enable_mmio_caching)
2233                 goto out;
2234
2235         /* Does the CPU support SEV-ES? */
2236         if (!boot_cpu_has(X86_FEATURE_SEV_ES))
2237                 goto out;
2238
2239         /* Has the system been allocated ASIDs for SEV-ES? */
2240         if (min_sev_asid == 1)
2241                 goto out;
2242
2243         sev_es_asid_count = min_sev_asid - 1;
2244         WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count));
2245         sev_es_supported = true;
2246
2247 out:
2248         if (boot_cpu_has(X86_FEATURE_SEV))
2249                 pr_info("SEV %s (ASIDs %u - %u)\n",
2250                         sev_supported ? "enabled" : "disabled",
2251                         min_sev_asid, max_sev_asid);
2252         if (boot_cpu_has(X86_FEATURE_SEV_ES))
2253                 pr_info("SEV-ES %s (ASIDs %u - %u)\n",
2254                         sev_es_supported ? "enabled" : "disabled",
2255                         min_sev_asid > 1 ? 1 : 0, min_sev_asid - 1);
2256
2257         sev_enabled = sev_supported;
2258         sev_es_enabled = sev_es_supported;
2259 #endif
2260 }
2261
2262 void sev_hardware_unsetup(void)
2263 {
2264         if (!sev_enabled)
2265                 return;
2266
2267         /* No need to take sev_bitmap_lock, all VMs have been destroyed. */
2268         sev_flush_asids(1, max_sev_asid);
2269
2270         bitmap_free(sev_asid_bitmap);
2271         bitmap_free(sev_reclaim_asid_bitmap);
2272
2273         misc_cg_set_capacity(MISC_CG_RES_SEV, 0);
2274         misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0);
2275 }
2276
2277 int sev_cpu_init(struct svm_cpu_data *sd)
2278 {
2279         if (!sev_enabled)
2280                 return 0;
2281
2282         sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL);
2283         if (!sd->sev_vmcbs)
2284                 return -ENOMEM;
2285
2286         return 0;
2287 }
2288
2289 /*
2290  * Pages used by hardware to hold guest encrypted state must be flushed before
2291  * returning them to the system.
2292  */
2293 static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va)
2294 {
2295         int asid = to_kvm_svm(vcpu->kvm)->sev_info.asid;
2296
2297         /*
2298          * Note!  The address must be a kernel address, as regular page walk
2299          * checks are performed by VM_PAGE_FLUSH, i.e. operating on a user
2300          * address is non-deterministic and unsafe.  This function deliberately
2301          * takes a pointer to deter passing in a user address.
2302          */
2303         unsigned long addr = (unsigned long)va;
2304
2305         /*
2306          * If CPU enforced cache coherency for encrypted mappings of the
2307          * same physical page is supported, use CLFLUSHOPT instead. NOTE: cache
2308          * flush is still needed in order to work properly with DMA devices.
2309          */
2310         if (boot_cpu_has(X86_FEATURE_SME_COHERENT)) {
2311                 clflush_cache_range(va, PAGE_SIZE);
2312                 return;
2313         }
2314
2315         /*
2316          * VM Page Flush takes a host virtual address and a guest ASID.  Fall
2317          * back to WBINVD if this faults so as not to make any problems worse
2318          * by leaving stale encrypted data in the cache.
2319          */
2320         if (WARN_ON_ONCE(wrmsrl_safe(MSR_AMD64_VM_PAGE_FLUSH, addr | asid)))
2321                 goto do_wbinvd;
2322
2323         return;
2324
2325 do_wbinvd:
2326         wbinvd_on_all_cpus();
2327 }
2328
2329 void sev_guest_memory_reclaimed(struct kvm *kvm)
2330 {
2331         if (!sev_guest(kvm))
2332                 return;
2333
2334         wbinvd_on_all_cpus();
2335 }
2336
2337 void sev_free_vcpu(struct kvm_vcpu *vcpu)
2338 {
2339         struct vcpu_svm *svm;
2340
2341         if (!sev_es_guest(vcpu->kvm))
2342                 return;
2343
2344         svm = to_svm(vcpu);
2345
2346         if (vcpu->arch.guest_state_protected)
2347                 sev_flush_encrypted_page(vcpu, svm->sev_es.vmsa);
2348
2349         __free_page(virt_to_page(svm->sev_es.vmsa));
2350
2351         if (svm->sev_es.ghcb_sa_free)
2352                 kvfree(svm->sev_es.ghcb_sa);
2353 }
2354
2355 static void dump_ghcb(struct vcpu_svm *svm)
2356 {
2357         struct ghcb *ghcb = svm->sev_es.ghcb;
2358         unsigned int nbits;
2359
2360         /* Re-use the dump_invalid_vmcb module parameter */
2361         if (!dump_invalid_vmcb) {
2362                 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
2363                 return;
2364         }
2365
2366         nbits = sizeof(ghcb->save.valid_bitmap) * 8;
2367
2368         pr_err("GHCB (GPA=%016llx):\n", svm->vmcb->control.ghcb_gpa);
2369         pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code",
2370                ghcb->save.sw_exit_code, ghcb_sw_exit_code_is_valid(ghcb));
2371         pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1",
2372                ghcb->save.sw_exit_info_1, ghcb_sw_exit_info_1_is_valid(ghcb));
2373         pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2",
2374                ghcb->save.sw_exit_info_2, ghcb_sw_exit_info_2_is_valid(ghcb));
2375         pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch",
2376                ghcb->save.sw_scratch, ghcb_sw_scratch_is_valid(ghcb));
2377         pr_err("%-20s%*pb\n", "valid_bitmap", nbits, ghcb->save.valid_bitmap);
2378 }
2379
2380 static void sev_es_sync_to_ghcb(struct vcpu_svm *svm)
2381 {
2382         struct kvm_vcpu *vcpu = &svm->vcpu;
2383         struct ghcb *ghcb = svm->sev_es.ghcb;
2384
2385         /*
2386          * The GHCB protocol so far allows for the following data
2387          * to be returned:
2388          *   GPRs RAX, RBX, RCX, RDX
2389          *
2390          * Copy their values, even if they may not have been written during the
2391          * VM-Exit.  It's the guest's responsibility to not consume random data.
2392          */
2393         ghcb_set_rax(ghcb, vcpu->arch.regs[VCPU_REGS_RAX]);
2394         ghcb_set_rbx(ghcb, vcpu->arch.regs[VCPU_REGS_RBX]);
2395         ghcb_set_rcx(ghcb, vcpu->arch.regs[VCPU_REGS_RCX]);
2396         ghcb_set_rdx(ghcb, vcpu->arch.regs[VCPU_REGS_RDX]);
2397 }
2398
2399 static void sev_es_sync_from_ghcb(struct vcpu_svm *svm)
2400 {
2401         struct vmcb_control_area *control = &svm->vmcb->control;
2402         struct kvm_vcpu *vcpu = &svm->vcpu;
2403         struct ghcb *ghcb = svm->sev_es.ghcb;
2404         u64 exit_code;
2405
2406         /*
2407          * The GHCB protocol so far allows for the following data
2408          * to be supplied:
2409          *   GPRs RAX, RBX, RCX, RDX
2410          *   XCR0
2411          *   CPL
2412          *
2413          * VMMCALL allows the guest to provide extra registers. KVM also
2414          * expects RSI for hypercalls, so include that, too.
2415          *
2416          * Copy their values to the appropriate location if supplied.
2417          */
2418         memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
2419
2420         BUILD_BUG_ON(sizeof(svm->sev_es.valid_bitmap) != sizeof(ghcb->save.valid_bitmap));
2421         memcpy(&svm->sev_es.valid_bitmap, &ghcb->save.valid_bitmap, sizeof(ghcb->save.valid_bitmap));
2422
2423         vcpu->arch.regs[VCPU_REGS_RAX] = kvm_ghcb_get_rax_if_valid(svm, ghcb);
2424         vcpu->arch.regs[VCPU_REGS_RBX] = kvm_ghcb_get_rbx_if_valid(svm, ghcb);
2425         vcpu->arch.regs[VCPU_REGS_RCX] = kvm_ghcb_get_rcx_if_valid(svm, ghcb);
2426         vcpu->arch.regs[VCPU_REGS_RDX] = kvm_ghcb_get_rdx_if_valid(svm, ghcb);
2427         vcpu->arch.regs[VCPU_REGS_RSI] = kvm_ghcb_get_rsi_if_valid(svm, ghcb);
2428
2429         svm->vmcb->save.cpl = kvm_ghcb_get_cpl_if_valid(svm, ghcb);
2430
2431         if (kvm_ghcb_xcr0_is_valid(svm)) {
2432                 vcpu->arch.xcr0 = ghcb_get_xcr0(ghcb);
2433                 kvm_update_cpuid_runtime(vcpu);
2434         }
2435
2436         /* Copy the GHCB exit information into the VMCB fields */
2437         exit_code = ghcb_get_sw_exit_code(ghcb);
2438         control->exit_code = lower_32_bits(exit_code);
2439         control->exit_code_hi = upper_32_bits(exit_code);
2440         control->exit_info_1 = ghcb_get_sw_exit_info_1(ghcb);
2441         control->exit_info_2 = ghcb_get_sw_exit_info_2(ghcb);
2442         svm->sev_es.sw_scratch = kvm_ghcb_get_sw_scratch_if_valid(svm, ghcb);
2443
2444         /* Clear the valid entries fields */
2445         memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
2446 }
2447
2448 static u64 kvm_ghcb_get_sw_exit_code(struct vmcb_control_area *control)
2449 {
2450         return (((u64)control->exit_code_hi) << 32) | control->exit_code;
2451 }
2452
2453 static int sev_es_validate_vmgexit(struct vcpu_svm *svm)
2454 {
2455         struct vmcb_control_area *control = &svm->vmcb->control;
2456         struct kvm_vcpu *vcpu = &svm->vcpu;
2457         u64 exit_code;
2458         u64 reason;
2459
2460         /*
2461          * Retrieve the exit code now even though it may not be marked valid
2462          * as it could help with debugging.
2463          */
2464         exit_code = kvm_ghcb_get_sw_exit_code(control);
2465
2466         /* Only GHCB Usage code 0 is supported */
2467         if (svm->sev_es.ghcb->ghcb_usage) {
2468                 reason = GHCB_ERR_INVALID_USAGE;
2469                 goto vmgexit_err;
2470         }
2471
2472         reason = GHCB_ERR_MISSING_INPUT;
2473
2474         if (!kvm_ghcb_sw_exit_code_is_valid(svm) ||
2475             !kvm_ghcb_sw_exit_info_1_is_valid(svm) ||
2476             !kvm_ghcb_sw_exit_info_2_is_valid(svm))
2477                 goto vmgexit_err;
2478
2479         switch (exit_code) {
2480         case SVM_EXIT_READ_DR7:
2481                 break;
2482         case SVM_EXIT_WRITE_DR7:
2483                 if (!kvm_ghcb_rax_is_valid(svm))
2484                         goto vmgexit_err;
2485                 break;
2486         case SVM_EXIT_RDTSC:
2487                 break;
2488         case SVM_EXIT_RDPMC:
2489                 if (!kvm_ghcb_rcx_is_valid(svm))
2490                         goto vmgexit_err;
2491                 break;
2492         case SVM_EXIT_CPUID:
2493                 if (!kvm_ghcb_rax_is_valid(svm) ||
2494                     !kvm_ghcb_rcx_is_valid(svm))
2495                         goto vmgexit_err;
2496                 if (vcpu->arch.regs[VCPU_REGS_RAX] == 0xd)
2497                         if (!kvm_ghcb_xcr0_is_valid(svm))
2498                                 goto vmgexit_err;
2499                 break;
2500         case SVM_EXIT_INVD:
2501                 break;
2502         case SVM_EXIT_IOIO:
2503                 if (control->exit_info_1 & SVM_IOIO_STR_MASK) {
2504                         if (!kvm_ghcb_sw_scratch_is_valid(svm))
2505                                 goto vmgexit_err;
2506                 } else {
2507                         if (!(control->exit_info_1 & SVM_IOIO_TYPE_MASK))
2508                                 if (!kvm_ghcb_rax_is_valid(svm))
2509                                         goto vmgexit_err;
2510                 }
2511                 break;
2512         case SVM_EXIT_MSR:
2513                 if (!kvm_ghcb_rcx_is_valid(svm))
2514                         goto vmgexit_err;
2515                 if (control->exit_info_1) {
2516                         if (!kvm_ghcb_rax_is_valid(svm) ||
2517                             !kvm_ghcb_rdx_is_valid(svm))
2518                                 goto vmgexit_err;
2519                 }
2520                 break;
2521         case SVM_EXIT_VMMCALL:
2522                 if (!kvm_ghcb_rax_is_valid(svm) ||
2523                     !kvm_ghcb_cpl_is_valid(svm))
2524                         goto vmgexit_err;
2525                 break;
2526         case SVM_EXIT_RDTSCP:
2527                 break;
2528         case SVM_EXIT_WBINVD:
2529                 break;
2530         case SVM_EXIT_MONITOR:
2531                 if (!kvm_ghcb_rax_is_valid(svm) ||
2532                     !kvm_ghcb_rcx_is_valid(svm) ||
2533                     !kvm_ghcb_rdx_is_valid(svm))
2534                         goto vmgexit_err;
2535                 break;
2536         case SVM_EXIT_MWAIT:
2537                 if (!kvm_ghcb_rax_is_valid(svm) ||
2538                     !kvm_ghcb_rcx_is_valid(svm))
2539                         goto vmgexit_err;
2540                 break;
2541         case SVM_VMGEXIT_MMIO_READ:
2542         case SVM_VMGEXIT_MMIO_WRITE:
2543                 if (!kvm_ghcb_sw_scratch_is_valid(svm))
2544                         goto vmgexit_err;
2545                 break;
2546         case SVM_VMGEXIT_NMI_COMPLETE:
2547         case SVM_VMGEXIT_AP_HLT_LOOP:
2548         case SVM_VMGEXIT_AP_JUMP_TABLE:
2549         case SVM_VMGEXIT_UNSUPPORTED_EVENT:
2550                 break;
2551         default:
2552                 reason = GHCB_ERR_INVALID_EVENT;
2553                 goto vmgexit_err;
2554         }
2555
2556         return 0;
2557
2558 vmgexit_err:
2559         if (reason == GHCB_ERR_INVALID_USAGE) {
2560                 vcpu_unimpl(vcpu, "vmgexit: ghcb usage %#x is not valid\n",
2561                             svm->sev_es.ghcb->ghcb_usage);
2562         } else if (reason == GHCB_ERR_INVALID_EVENT) {
2563                 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx is not valid\n",
2564                             exit_code);
2565         } else {
2566                 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx input is not valid\n",
2567                             exit_code);
2568                 dump_ghcb(svm);
2569         }
2570
2571         ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2);
2572         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, reason);
2573
2574         /* Resume the guest to "return" the error code. */
2575         return 1;
2576 }
2577
2578 void sev_es_unmap_ghcb(struct vcpu_svm *svm)
2579 {
2580         if (!svm->sev_es.ghcb)
2581                 return;
2582
2583         if (svm->sev_es.ghcb_sa_free) {
2584                 /*
2585                  * The scratch area lives outside the GHCB, so there is a
2586                  * buffer that, depending on the operation performed, may
2587                  * need to be synced, then freed.
2588                  */
2589                 if (svm->sev_es.ghcb_sa_sync) {
2590                         kvm_write_guest(svm->vcpu.kvm,
2591                                         svm->sev_es.sw_scratch,
2592                                         svm->sev_es.ghcb_sa,
2593                                         svm->sev_es.ghcb_sa_len);
2594                         svm->sev_es.ghcb_sa_sync = false;
2595                 }
2596
2597                 kvfree(svm->sev_es.ghcb_sa);
2598                 svm->sev_es.ghcb_sa = NULL;
2599                 svm->sev_es.ghcb_sa_free = false;
2600         }
2601
2602         trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->sev_es.ghcb);
2603
2604         sev_es_sync_to_ghcb(svm);
2605
2606         kvm_vcpu_unmap(&svm->vcpu, &svm->sev_es.ghcb_map, true);
2607         svm->sev_es.ghcb = NULL;
2608 }
2609
2610 void pre_sev_run(struct vcpu_svm *svm, int cpu)
2611 {
2612         struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
2613         int asid = sev_get_asid(svm->vcpu.kvm);
2614
2615         /* Assign the asid allocated with this SEV guest */
2616         svm->asid = asid;
2617
2618         /*
2619          * Flush guest TLB:
2620          *
2621          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
2622          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
2623          */
2624         if (sd->sev_vmcbs[asid] == svm->vmcb &&
2625             svm->vcpu.arch.last_vmentry_cpu == cpu)
2626                 return;
2627
2628         sd->sev_vmcbs[asid] = svm->vmcb;
2629         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
2630         vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
2631 }
2632
2633 #define GHCB_SCRATCH_AREA_LIMIT         (16ULL * PAGE_SIZE)
2634 static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len)
2635 {
2636         struct vmcb_control_area *control = &svm->vmcb->control;
2637         u64 ghcb_scratch_beg, ghcb_scratch_end;
2638         u64 scratch_gpa_beg, scratch_gpa_end;
2639         void *scratch_va;
2640
2641         scratch_gpa_beg = svm->sev_es.sw_scratch;
2642         if (!scratch_gpa_beg) {
2643                 pr_err("vmgexit: scratch gpa not provided\n");
2644                 goto e_scratch;
2645         }
2646
2647         scratch_gpa_end = scratch_gpa_beg + len;
2648         if (scratch_gpa_end < scratch_gpa_beg) {
2649                 pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n",
2650                        len, scratch_gpa_beg);
2651                 goto e_scratch;
2652         }
2653
2654         if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) {
2655                 /* Scratch area begins within GHCB */
2656                 ghcb_scratch_beg = control->ghcb_gpa +
2657                                    offsetof(struct ghcb, shared_buffer);
2658                 ghcb_scratch_end = control->ghcb_gpa +
2659                                    offsetof(struct ghcb, reserved_0xff0);
2660
2661                 /*
2662                  * If the scratch area begins within the GHCB, it must be
2663                  * completely contained in the GHCB shared buffer area.
2664                  */
2665                 if (scratch_gpa_beg < ghcb_scratch_beg ||
2666                     scratch_gpa_end > ghcb_scratch_end) {
2667                         pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n",
2668                                scratch_gpa_beg, scratch_gpa_end);
2669                         goto e_scratch;
2670                 }
2671
2672                 scratch_va = (void *)svm->sev_es.ghcb;
2673                 scratch_va += (scratch_gpa_beg - control->ghcb_gpa);
2674         } else {
2675                 /*
2676                  * The guest memory must be read into a kernel buffer, so
2677                  * limit the size
2678                  */
2679                 if (len > GHCB_SCRATCH_AREA_LIMIT) {
2680                         pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n",
2681                                len, GHCB_SCRATCH_AREA_LIMIT);
2682                         goto e_scratch;
2683                 }
2684                 scratch_va = kvzalloc(len, GFP_KERNEL_ACCOUNT);
2685                 if (!scratch_va)
2686                         return -ENOMEM;
2687
2688                 if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, len)) {
2689                         /* Unable to copy scratch area from guest */
2690                         pr_err("vmgexit: kvm_read_guest for scratch area failed\n");
2691
2692                         kvfree(scratch_va);
2693                         return -EFAULT;
2694                 }
2695
2696                 /*
2697                  * The scratch area is outside the GHCB. The operation will
2698                  * dictate whether the buffer needs to be synced before running
2699                  * the vCPU next time (i.e. a read was requested so the data
2700                  * must be written back to the guest memory).
2701                  */
2702                 svm->sev_es.ghcb_sa_sync = sync;
2703                 svm->sev_es.ghcb_sa_free = true;
2704         }
2705
2706         svm->sev_es.ghcb_sa = scratch_va;
2707         svm->sev_es.ghcb_sa_len = len;
2708
2709         return 0;
2710
2711 e_scratch:
2712         ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2);
2713         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, GHCB_ERR_INVALID_SCRATCH_AREA);
2714
2715         return 1;
2716 }
2717
2718 static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask,
2719                               unsigned int pos)
2720 {
2721         svm->vmcb->control.ghcb_gpa &= ~(mask << pos);
2722         svm->vmcb->control.ghcb_gpa |= (value & mask) << pos;
2723 }
2724
2725 static u64 get_ghcb_msr_bits(struct vcpu_svm *svm, u64 mask, unsigned int pos)
2726 {
2727         return (svm->vmcb->control.ghcb_gpa >> pos) & mask;
2728 }
2729
2730 static void set_ghcb_msr(struct vcpu_svm *svm, u64 value)
2731 {
2732         svm->vmcb->control.ghcb_gpa = value;
2733 }
2734
2735 static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm)
2736 {
2737         struct vmcb_control_area *control = &svm->vmcb->control;
2738         struct kvm_vcpu *vcpu = &svm->vcpu;
2739         u64 ghcb_info;
2740         int ret = 1;
2741
2742         ghcb_info = control->ghcb_gpa & GHCB_MSR_INFO_MASK;
2743
2744         trace_kvm_vmgexit_msr_protocol_enter(svm->vcpu.vcpu_id,
2745                                              control->ghcb_gpa);
2746
2747         switch (ghcb_info) {
2748         case GHCB_MSR_SEV_INFO_REQ:
2749                 set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX,
2750                                                     GHCB_VERSION_MIN,
2751                                                     sev_enc_bit));
2752                 break;
2753         case GHCB_MSR_CPUID_REQ: {
2754                 u64 cpuid_fn, cpuid_reg, cpuid_value;
2755
2756                 cpuid_fn = get_ghcb_msr_bits(svm,
2757                                              GHCB_MSR_CPUID_FUNC_MASK,
2758                                              GHCB_MSR_CPUID_FUNC_POS);
2759
2760                 /* Initialize the registers needed by the CPUID intercept */
2761                 vcpu->arch.regs[VCPU_REGS_RAX] = cpuid_fn;
2762                 vcpu->arch.regs[VCPU_REGS_RCX] = 0;
2763
2764                 ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_CPUID);
2765                 if (!ret) {
2766                         /* Error, keep GHCB MSR value as-is */
2767                         break;
2768                 }
2769
2770                 cpuid_reg = get_ghcb_msr_bits(svm,
2771                                               GHCB_MSR_CPUID_REG_MASK,
2772                                               GHCB_MSR_CPUID_REG_POS);
2773                 if (cpuid_reg == 0)
2774                         cpuid_value = vcpu->arch.regs[VCPU_REGS_RAX];
2775                 else if (cpuid_reg == 1)
2776                         cpuid_value = vcpu->arch.regs[VCPU_REGS_RBX];
2777                 else if (cpuid_reg == 2)
2778                         cpuid_value = vcpu->arch.regs[VCPU_REGS_RCX];
2779                 else
2780                         cpuid_value = vcpu->arch.regs[VCPU_REGS_RDX];
2781
2782                 set_ghcb_msr_bits(svm, cpuid_value,
2783                                   GHCB_MSR_CPUID_VALUE_MASK,
2784                                   GHCB_MSR_CPUID_VALUE_POS);
2785
2786                 set_ghcb_msr_bits(svm, GHCB_MSR_CPUID_RESP,
2787                                   GHCB_MSR_INFO_MASK,
2788                                   GHCB_MSR_INFO_POS);
2789                 break;
2790         }
2791         case GHCB_MSR_TERM_REQ: {
2792                 u64 reason_set, reason_code;
2793
2794                 reason_set = get_ghcb_msr_bits(svm,
2795                                                GHCB_MSR_TERM_REASON_SET_MASK,
2796                                                GHCB_MSR_TERM_REASON_SET_POS);
2797                 reason_code = get_ghcb_msr_bits(svm,
2798                                                 GHCB_MSR_TERM_REASON_MASK,
2799                                                 GHCB_MSR_TERM_REASON_POS);
2800                 pr_info("SEV-ES guest requested termination: %#llx:%#llx\n",
2801                         reason_set, reason_code);
2802
2803                 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
2804                 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM;
2805                 vcpu->run->system_event.ndata = 1;
2806                 vcpu->run->system_event.data[0] = control->ghcb_gpa;
2807
2808                 return 0;
2809         }
2810         default:
2811                 /* Error, keep GHCB MSR value as-is */
2812                 break;
2813         }
2814
2815         trace_kvm_vmgexit_msr_protocol_exit(svm->vcpu.vcpu_id,
2816                                             control->ghcb_gpa, ret);
2817
2818         return ret;
2819 }
2820
2821 int sev_handle_vmgexit(struct kvm_vcpu *vcpu)
2822 {
2823         struct vcpu_svm *svm = to_svm(vcpu);
2824         struct vmcb_control_area *control = &svm->vmcb->control;
2825         u64 ghcb_gpa, exit_code;
2826         int ret;
2827
2828         /* Validate the GHCB */
2829         ghcb_gpa = control->ghcb_gpa;
2830         if (ghcb_gpa & GHCB_MSR_INFO_MASK)
2831                 return sev_handle_vmgexit_msr_protocol(svm);
2832
2833         if (!ghcb_gpa) {
2834                 vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n");
2835
2836                 /* Without a GHCB, just return right back to the guest */
2837                 return 1;
2838         }
2839
2840         if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->sev_es.ghcb_map)) {
2841                 /* Unable to map GHCB from guest */
2842                 vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n",
2843                             ghcb_gpa);
2844
2845                 /* Without a GHCB, just return right back to the guest */
2846                 return 1;
2847         }
2848
2849         svm->sev_es.ghcb = svm->sev_es.ghcb_map.hva;
2850
2851         trace_kvm_vmgexit_enter(vcpu->vcpu_id, svm->sev_es.ghcb);
2852
2853         sev_es_sync_from_ghcb(svm);
2854         ret = sev_es_validate_vmgexit(svm);
2855         if (ret)
2856                 return ret;
2857
2858         ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 0);
2859         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 0);
2860
2861         exit_code = kvm_ghcb_get_sw_exit_code(control);
2862         switch (exit_code) {
2863         case SVM_VMGEXIT_MMIO_READ:
2864                 ret = setup_vmgexit_scratch(svm, true, control->exit_info_2);
2865                 if (ret)
2866                         break;
2867
2868                 ret = kvm_sev_es_mmio_read(vcpu,
2869                                            control->exit_info_1,
2870                                            control->exit_info_2,
2871                                            svm->sev_es.ghcb_sa);
2872                 break;
2873         case SVM_VMGEXIT_MMIO_WRITE:
2874                 ret = setup_vmgexit_scratch(svm, false, control->exit_info_2);
2875                 if (ret)
2876                         break;
2877
2878                 ret = kvm_sev_es_mmio_write(vcpu,
2879                                             control->exit_info_1,
2880                                             control->exit_info_2,
2881                                             svm->sev_es.ghcb_sa);
2882                 break;
2883         case SVM_VMGEXIT_NMI_COMPLETE:
2884                 ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_IRET);
2885                 break;
2886         case SVM_VMGEXIT_AP_HLT_LOOP:
2887                 ret = kvm_emulate_ap_reset_hold(vcpu);
2888                 break;
2889         case SVM_VMGEXIT_AP_JUMP_TABLE: {
2890                 struct kvm_sev_info *sev = &to_kvm_svm(vcpu->kvm)->sev_info;
2891
2892                 switch (control->exit_info_1) {
2893                 case 0:
2894                         /* Set AP jump table address */
2895                         sev->ap_jump_table = control->exit_info_2;
2896                         break;
2897                 case 1:
2898                         /* Get AP jump table address */
2899                         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, sev->ap_jump_table);
2900                         break;
2901                 default:
2902                         pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n",
2903                                control->exit_info_1);
2904                         ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2);
2905                         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, GHCB_ERR_INVALID_INPUT);
2906                 }
2907
2908                 ret = 1;
2909                 break;
2910         }
2911         case SVM_VMGEXIT_UNSUPPORTED_EVENT:
2912                 vcpu_unimpl(vcpu,
2913                             "vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n",
2914                             control->exit_info_1, control->exit_info_2);
2915                 ret = -EINVAL;
2916                 break;
2917         default:
2918                 ret = svm_invoke_exit_handler(vcpu, exit_code);
2919         }
2920
2921         return ret;
2922 }
2923
2924 int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in)
2925 {
2926         int count;
2927         int bytes;
2928         int r;
2929
2930         if (svm->vmcb->control.exit_info_2 > INT_MAX)
2931                 return -EINVAL;
2932
2933         count = svm->vmcb->control.exit_info_2;
2934         if (unlikely(check_mul_overflow(count, size, &bytes)))
2935                 return -EINVAL;
2936
2937         r = setup_vmgexit_scratch(svm, in, bytes);
2938         if (r)
2939                 return r;
2940
2941         return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->sev_es.ghcb_sa,
2942                                     count, in);
2943 }
2944
2945 static void sev_es_init_vmcb(struct vcpu_svm *svm)
2946 {
2947         struct kvm_vcpu *vcpu = &svm->vcpu;
2948
2949         svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ES_ENABLE;
2950         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
2951
2952         /*
2953          * An SEV-ES guest requires a VMSA area that is a separate from the
2954          * VMCB page. Do not include the encryption mask on the VMSA physical
2955          * address since hardware will access it using the guest key.
2956          */
2957         svm->vmcb->control.vmsa_pa = __pa(svm->sev_es.vmsa);
2958
2959         /* Can't intercept CR register access, HV can't modify CR registers */
2960         svm_clr_intercept(svm, INTERCEPT_CR0_READ);
2961         svm_clr_intercept(svm, INTERCEPT_CR4_READ);
2962         svm_clr_intercept(svm, INTERCEPT_CR8_READ);
2963         svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
2964         svm_clr_intercept(svm, INTERCEPT_CR4_WRITE);
2965         svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
2966
2967         svm_clr_intercept(svm, INTERCEPT_SELECTIVE_CR0);
2968
2969         /* Track EFER/CR register changes */
2970         svm_set_intercept(svm, TRAP_EFER_WRITE);
2971         svm_set_intercept(svm, TRAP_CR0_WRITE);
2972         svm_set_intercept(svm, TRAP_CR4_WRITE);
2973         svm_set_intercept(svm, TRAP_CR8_WRITE);
2974
2975         /* No support for enable_vmware_backdoor */
2976         clr_exception_intercept(svm, GP_VECTOR);
2977
2978         /* Can't intercept XSETBV, HV can't modify XCR0 directly */
2979         svm_clr_intercept(svm, INTERCEPT_XSETBV);
2980
2981         /* Clear intercepts on selected MSRs */
2982         set_msr_interception(vcpu, svm->msrpm, MSR_EFER, 1, 1);
2983         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_CR_PAT, 1, 1);
2984         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
2985         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
2986         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
2987         set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
2988
2989         if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) &&
2990             (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP) ||
2991              guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDPID))) {
2992                 set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, 1, 1);
2993                 if (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP))
2994                         svm_clr_intercept(svm, INTERCEPT_RDTSCP);
2995         }
2996 }
2997
2998 void sev_init_vmcb(struct vcpu_svm *svm)
2999 {
3000         svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
3001         clr_exception_intercept(svm, UD_VECTOR);
3002
3003         if (sev_es_guest(svm->vcpu.kvm))
3004                 sev_es_init_vmcb(svm);
3005 }
3006
3007 void sev_es_vcpu_reset(struct vcpu_svm *svm)
3008 {
3009         /*
3010          * Set the GHCB MSR value as per the GHCB specification when emulating
3011          * vCPU RESET for an SEV-ES guest.
3012          */
3013         set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX,
3014                                             GHCB_VERSION_MIN,
3015                                             sev_enc_bit));
3016 }
3017
3018 void sev_es_prepare_switch_to_guest(struct sev_es_save_area *hostsa)
3019 {
3020         /*
3021          * As an SEV-ES guest, hardware will restore the host state on VMEXIT,
3022          * of which one step is to perform a VMLOAD.  KVM performs the
3023          * corresponding VMSAVE in svm_prepare_guest_switch for both
3024          * traditional and SEV-ES guests.
3025          */
3026
3027         /* XCR0 is restored on VMEXIT, save the current host value */
3028         hostsa->xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
3029
3030         /* PKRU is restored on VMEXIT, save the current host value */
3031         hostsa->pkru = read_pkru();
3032
3033         /* MSR_IA32_XSS is restored on VMEXIT, save the currnet host value */
3034         hostsa->xss = host_xss;
3035 }
3036
3037 void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
3038 {
3039         struct vcpu_svm *svm = to_svm(vcpu);
3040
3041         /* First SIPI: Use the values as initially set by the VMM */
3042         if (!svm->sev_es.received_first_sipi) {
3043                 svm->sev_es.received_first_sipi = true;
3044                 return;
3045         }
3046
3047         /*
3048          * Subsequent SIPI: Return from an AP Reset Hold VMGEXIT, where
3049          * the guest will set the CS and RIP. Set SW_EXIT_INFO_2 to a
3050          * non-zero value.
3051          */
3052         if (!svm->sev_es.ghcb)
3053                 return;
3054
3055         ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 1);
3056 }