Merge branch 'kvm-5.16-fixes' into kvm-master
[platform/kernel/linux-starfive.git] / arch / x86 / kvm / x86.c
index 5c479ae..dd7b8b4 100644 (file)
@@ -68,7 +68,9 @@
 #include <asm/mce.h>
 #include <asm/pkru.h>
 #include <linux/kernel_stat.h>
-#include <asm/fpu/internal.h> /* Ugh! */
+#include <asm/fpu/api.h>
+#include <asm/fpu/xcr.h>
+#include <asm/fpu/xstate.h>
 #include <asm/pvclock.h>
 #include <asm/div64.h>
 #include <asm/irq_remapping.h>
@@ -293,8 +295,6 @@ u64 __read_mostly host_xcr0;
 u64 __read_mostly supported_xcr0;
 EXPORT_SYMBOL_GPL(supported_xcr0);
 
-static struct kmem_cache *x86_fpu_cache;
-
 static struct kmem_cache *x86_emulator_cache;
 
 /*
@@ -4814,144 +4814,27 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
        return 0;
 }
 
-#define XSTATE_COMPACTION_ENABLED (1ULL << 63)
-
-static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
-{
-       struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
-       u64 xstate_bv = xsave->header.xfeatures;
-       u64 valid;
-
-       /*
-        * Copy legacy XSAVE area, to avoid complications with CPUID
-        * leaves 0 and 1 in the loop below.
-        */
-       memcpy(dest, xsave, XSAVE_HDR_OFFSET);
-
-       /* Set XSTATE_BV */
-       xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
-       *(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;
-
-       /*
-        * Copy each region from the possibly compacted offset to the
-        * non-compacted offset.
-        */
-       valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
-       while (valid) {
-               u32 size, offset, ecx, edx;
-               u64 xfeature_mask = valid & -valid;
-               int xfeature_nr = fls64(xfeature_mask) - 1;
-               void *src;
-
-               cpuid_count(XSTATE_CPUID, xfeature_nr,
-                           &size, &offset, &ecx, &edx);
-
-               if (xfeature_nr == XFEATURE_PKRU) {
-                       memcpy(dest + offset, &vcpu->arch.pkru,
-                              sizeof(vcpu->arch.pkru));
-               } else {
-                       src = get_xsave_addr(xsave, xfeature_nr);
-                       if (src)
-                               memcpy(dest + offset, src, size);
-               }
-
-               valid -= xfeature_mask;
-       }
-}
-
-static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
-{
-       struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
-       u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
-       u64 valid;
-
-       /*
-        * Copy legacy XSAVE area, to avoid complications with CPUID
-        * leaves 0 and 1 in the loop below.
-        */
-       memcpy(xsave, src, XSAVE_HDR_OFFSET);
-
-       /* Set XSTATE_BV and possibly XCOMP_BV.  */
-       xsave->header.xfeatures = xstate_bv;
-       if (boot_cpu_has(X86_FEATURE_XSAVES))
-               xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
-
-       /*
-        * Copy each region from the non-compacted offset to the
-        * possibly compacted offset.
-        */
-       valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
-       while (valid) {
-               u32 size, offset, ecx, edx;
-               u64 xfeature_mask = valid & -valid;
-               int xfeature_nr = fls64(xfeature_mask) - 1;
-
-               cpuid_count(XSTATE_CPUID, xfeature_nr,
-                           &size, &offset, &ecx, &edx);
-
-               if (xfeature_nr == XFEATURE_PKRU) {
-                       memcpy(&vcpu->arch.pkru, src + offset,
-                              sizeof(vcpu->arch.pkru));
-               } else {
-                       void *dest = get_xsave_addr(xsave, xfeature_nr);
-
-                       if (dest)
-                               memcpy(dest, src + offset, size);
-               }
-
-               valid -= xfeature_mask;
-       }
-}
-
 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
                                         struct kvm_xsave *guest_xsave)
 {
-       if (!vcpu->arch.guest_fpu)
+       if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
                return;
 
-       if (boot_cpu_has(X86_FEATURE_XSAVE)) {
-               memset(guest_xsave, 0, sizeof(struct kvm_xsave));
-               fill_xsave((u8 *) guest_xsave->region, vcpu);
-       } else {
-               memcpy(guest_xsave->region,
-                       &vcpu->arch.guest_fpu->state.fxsave,
-                       sizeof(struct fxregs_state));
-               *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
-                       XFEATURE_MASK_FPSSE;
-       }
+       fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
+                                      guest_xsave->region,
+                                      sizeof(guest_xsave->region),
+                                      vcpu->arch.pkru);
 }
 
-#define XSAVE_MXCSR_OFFSET 24
-
 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
                                        struct kvm_xsave *guest_xsave)
 {
-       u64 xstate_bv;
-       u32 mxcsr;
-
-       if (!vcpu->arch.guest_fpu)
+       if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
                return 0;
 
-       xstate_bv = *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
-       mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
-
-       if (boot_cpu_has(X86_FEATURE_XSAVE)) {
-               /*
-                * Here we allow setting states that are not present in
-                * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
-                * with old userspace.
-                */
-               if (xstate_bv & ~supported_xcr0 || mxcsr & ~mxcsr_feature_mask)
-                       return -EINVAL;
-               load_xsave(vcpu, (u8 *)guest_xsave->region);
-       } else {
-               if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
-                       mxcsr & ~mxcsr_feature_mask)
-                       return -EINVAL;
-               memcpy(&vcpu->arch.guest_fpu->state.fxsave,
-                       guest_xsave->region, sizeof(struct fxregs_state));
-       }
-       return 0;
+       return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
+                                             guest_xsave->region,
+                                             supported_xcr0, &vcpu->arch.pkru);
 }
 
 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
@@ -5893,6 +5776,12 @@ split_irqchip_unlock:
                if (kvm_x86_ops.vm_copy_enc_context_from)
                        r = kvm_x86_ops.vm_copy_enc_context_from(kvm, cap->args[0]);
                return r;
+       case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
+               r = -EINVAL;
+               if (kvm_x86_ops.vm_move_enc_context_from)
+                       r = kvm_x86_ops.vm_move_enc_context_from(
+                               kvm, cap->args[0]);
+               return r;
        case KVM_CAP_EXIT_HYPERCALL:
                if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {
                        r = -EINVAL;
@@ -7143,7 +7032,7 @@ static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
 }
 
 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
-                              unsigned short port, void *val,
+                              unsigned short port,
                               unsigned int count, bool in)
 {
        vcpu->arch.pio.port = port;
@@ -7151,10 +7040,8 @@ static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
        vcpu->arch.pio.count  = count;
        vcpu->arch.pio.size = size;
 
-       if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
-               vcpu->arch.pio.count = 0;
+       if (!kernel_pio(vcpu, vcpu->arch.pio_data))
                return 1;
-       }
 
        vcpu->run->exit_reason = KVM_EXIT_IO;
        vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
@@ -7166,26 +7053,39 @@ static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
        return 0;
 }
 
-static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
-                          unsigned short port, void *val, unsigned int count)
+static int __emulator_pio_in(struct kvm_vcpu *vcpu, int size,
+                            unsigned short port, unsigned int count)
 {
-       int ret;
+       WARN_ON(vcpu->arch.pio.count);
+       memset(vcpu->arch.pio_data, 0, size * count);
+       return emulator_pio_in_out(vcpu, size, port, count, true);
+}
 
-       if (vcpu->arch.pio.count)
-               goto data_avail;
+static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)
+{
+       int size = vcpu->arch.pio.size;
+       unsigned count = vcpu->arch.pio.count;
+       memcpy(val, vcpu->arch.pio_data, size * count);
+       trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);
+       vcpu->arch.pio.count = 0;
+}
 
-       memset(vcpu->arch.pio_data, 0, size * count);
+static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
+                          unsigned short port, void *val, unsigned int count)
+{
+       if (vcpu->arch.pio.count) {
+               /* Complete previous iteration.  */
+       } else {
+               int r = __emulator_pio_in(vcpu, size, port, count);
+               if (!r)
+                       return r;
 
-       ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
-       if (ret) {
-data_avail:
-               memcpy(val, vcpu->arch.pio_data, size * count);
-               trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
-               vcpu->arch.pio.count = 0;
-               return 1;
+               /* Results already available, fall through.  */
        }
 
-       return 0;
+       WARN_ON(count != vcpu->arch.pio.count);
+       complete_emulator_pio_in(vcpu, val);
+       return 1;
 }
 
 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
@@ -7200,9 +7100,15 @@ static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
                            unsigned short port, const void *val,
                            unsigned int count)
 {
+       int ret;
+
        memcpy(vcpu->arch.pio_data, val, size * count);
        trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
-       return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
+       ret = emulator_pio_in_out(vcpu, size, port, count, false);
+       if (ret)
+                vcpu->arch.pio.count = 0;
+
+        return ret;
 }
 
 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
@@ -8697,18 +8603,11 @@ int kvm_arch_init(void *opaque)
        }
 
        r = -ENOMEM;
-       x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
-                                         __alignof__(struct fpu), SLAB_ACCOUNT,
-                                         NULL);
-       if (!x86_fpu_cache) {
-               printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
-               goto out;
-       }
 
        x86_emulator_cache = kvm_alloc_emulator_cache();
        if (!x86_emulator_cache) {
                pr_err("kvm: failed to allocate cache for x86 emulator\n");
-               goto out_free_x86_fpu_cache;
+               goto out;
        }
 
        user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
@@ -8746,8 +8645,6 @@ out_free_percpu:
        free_percpu(user_return_msrs);
 out_free_x86_emulator_cache:
        kmem_cache_destroy(x86_emulator_cache);
-out_free_x86_fpu_cache:
-       kmem_cache_destroy(x86_fpu_cache);
 out:
        return r;
 }
@@ -8774,7 +8671,6 @@ void kvm_arch_exit(void)
        kvm_mmu_module_exit();
        free_percpu(user_return_msrs);
        kmem_cache_destroy(x86_emulator_cache);
-       kmem_cache_destroy(x86_fpu_cache);
 #ifdef CONFIG_KVM_XEN
        static_key_deferred_flush(&kvm_xen_enabled);
        WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
@@ -9063,9 +8959,17 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
 
        kvm_run->cr8 = kvm_get_cr8(vcpu);
        kvm_run->apic_base = kvm_get_apic_base(vcpu);
+
+       /*
+        * The call to kvm_ready_for_interrupt_injection() may end up in
+        * kvm_xen_has_interrupt() which may require the srcu lock to be
+        * held, to protect against changes in the vcpu_info address.
+        */
+       vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
        kvm_run->ready_for_interrupt_injection =
                pic_in_kernel(vcpu->kvm) ||
                kvm_vcpu_ready_for_interrupt_injection(vcpu);
+       srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 
        if (is_smm(vcpu))
                kvm_run->flags |= KVM_RUN_X86_SMM;
@@ -9704,7 +9608,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
        }
 
        if (kvm_request_pending(vcpu)) {
-               if (kvm_check_request(KVM_REQ_VM_BUGGED, vcpu)) {
+               if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
                        r = -EIO;
                        goto out;
                }
@@ -9938,14 +9842,14 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
                        break;
 
-                if (unlikely(kvm_vcpu_exit_request(vcpu))) {
+               if (vcpu->arch.apicv_active)
+                       static_call(kvm_x86_sync_pir_to_irr)(vcpu);
+
+               if (unlikely(kvm_vcpu_exit_request(vcpu))) {
                        exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
                        break;
                }
-
-               if (vcpu->arch.apicv_active)
-                       static_call(kvm_x86_sync_pir_to_irr)(vcpu);
-        }
+       }
 
        /*
         * Do this here before restoring debug registers on the host.  And
@@ -10208,58 +10112,21 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
        return 0;
 }
 
-static void kvm_save_current_fpu(struct fpu *fpu)
-{
-       /*
-        * If the target FPU state is not resident in the CPU registers, just
-        * memcpy() from current, else save CPU state directly to the target.
-        */
-       if (test_thread_flag(TIF_NEED_FPU_LOAD))
-               memcpy(&fpu->state, &current->thread.fpu.state,
-                      fpu_kernel_xstate_size);
-       else
-               save_fpregs_to_fpstate(fpu);
-}
-
 /* Swap (qemu) user FPU context for the guest FPU context. */
 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 {
-       fpregs_lock();
-
-       kvm_save_current_fpu(vcpu->arch.user_fpu);
-
        /*
-        * Guests with protected state can't have it set by the hypervisor,
-        * so skip trying to set it.
+        * Exclude PKRU from restore as restored separately in
+        * kvm_x86_ops.run().
         */
-       if (vcpu->arch.guest_fpu)
-               /* PKRU is separately restored in kvm_x86_ops.run. */
-               __restore_fpregs_from_fpstate(&vcpu->arch.guest_fpu->state,
-                                       ~XFEATURE_MASK_PKRU);
-
-       fpregs_mark_activate();
-       fpregs_unlock();
-
+       fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
        trace_kvm_fpu(1);
 }
 
 /* When vcpu_run ends, restore user space FPU context. */
 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
 {
-       fpregs_lock();
-
-       /*
-        * Guests with protected state can't have it read by the hypervisor,
-        * so skip trying to save it.
-        */
-       if (vcpu->arch.guest_fpu)
-               kvm_save_current_fpu(vcpu->arch.guest_fpu);
-
-       restore_fpregs_from_fpstate(&vcpu->arch.user_fpu->state);
-
-       fpregs_mark_activate();
-       fpregs_unlock();
-
+       fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
        ++vcpu->stat.fpu_reload;
        trace_kvm_fpu(0);
 }
@@ -10860,12 +10727,12 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
        struct fxregs_state *fxsave;
 
-       if (!vcpu->arch.guest_fpu)
+       if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
                return 0;
 
        vcpu_load(vcpu);
 
-       fxsave = &vcpu->arch.guest_fpu->state.fxsave;
+       fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
        memcpy(fpu->fpr, fxsave->st_space, 128);
        fpu->fcw = fxsave->cwd;
        fpu->fsw = fxsave->swd;
@@ -10883,12 +10750,12 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
        struct fxregs_state *fxsave;
 
-       if (!vcpu->arch.guest_fpu)
+       if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
                return 0;
 
        vcpu_load(vcpu);
 
-       fxsave = &vcpu->arch.guest_fpu->state.fxsave;
+       fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
 
        memcpy(fxsave->st_space, fpu->fpr, 128);
        fxsave->cwd = fpu->fcw;
@@ -10939,15 +10806,6 @@ static int sync_regs(struct kvm_vcpu *vcpu)
        return 0;
 }
 
-void kvm_free_guest_fpu(struct kvm_vcpu *vcpu)
-{
-       if (vcpu->arch.guest_fpu) {
-               kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
-               vcpu->arch.guest_fpu = NULL;
-       }
-}
-EXPORT_SYMBOL_GPL(kvm_free_guest_fpu);
-
 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
 {
        if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
@@ -11004,23 +10862,10 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
        if (!alloc_emulate_ctxt(vcpu))
                goto free_wbinvd_dirty_mask;
 
-       vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
-                                               GFP_KERNEL_ACCOUNT);
-       if (!vcpu->arch.user_fpu) {
-               pr_err("kvm: failed to allocate userspace's fpu\n");
-               goto free_emulate_ctxt;
-       }
-
-       vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
-                                                GFP_KERNEL_ACCOUNT);
-       if (!vcpu->arch.guest_fpu) {
+       if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {
                pr_err("kvm: failed to allocate vcpu's fpu\n");
-               goto free_user_fpu;
+               goto free_emulate_ctxt;
        }
-       fpstate_init(&vcpu->arch.guest_fpu->state);
-       if (boot_cpu_has(X86_FEATURE_XSAVES))
-               vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
-                       host_xcr0 | XSTATE_COMPACTION_ENABLED;
 
        vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
        vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
@@ -11052,9 +10897,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
        return 0;
 
 free_guest_fpu:
-       kvm_free_guest_fpu(vcpu);
-free_user_fpu:
-       kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
+       fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
 free_emulate_ctxt:
        kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
 free_wbinvd_dirty_mask:
@@ -11100,8 +10943,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 
        kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
        free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
-       kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
-       kvm_free_guest_fpu(vcpu);
+       fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
 
        kvm_hv_vcpu_uninit(vcpu);
        kvm_pmu_destroy(vcpu);
@@ -11163,8 +11005,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
        kvm_async_pf_hash_reset(vcpu);
        vcpu->arch.apf.halted = false;
 
-       if (vcpu->arch.guest_fpu && kvm_mpx_supported()) {
-               void *mpx_state_buffer;
+       if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {
+               struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;
 
                /*
                 * To avoid have the INIT path from kvm_apic_has_events() that be
@@ -11172,14 +11014,10 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
                 */
                if (init_event)
                        kvm_put_guest_fpu(vcpu);
-               mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
-                                       XFEATURE_BNDREGS);
-               if (mpx_state_buffer)
-                       memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
-               mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
-                                       XFEATURE_BNDCSR);
-               if (mpx_state_buffer)
-                       memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
+
+               fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS);
+               fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR);
+
                if (init_event)
                        kvm_load_guest_fpu(vcpu);
        }
@@ -12639,44 +12477,81 @@ int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
 }
 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_read);
 
-static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
+static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
+                          unsigned int port);
+
+static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)
 {
-       memcpy(vcpu->arch.guest_ins_data, vcpu->arch.pio_data,
-              vcpu->arch.pio.count * vcpu->arch.pio.size);
-       vcpu->arch.pio.count = 0;
+       int size = vcpu->arch.pio.size;
+       int port = vcpu->arch.pio.port;
 
+       vcpu->arch.pio.count = 0;
+       if (vcpu->arch.sev_pio_count)
+               return kvm_sev_es_outs(vcpu, size, port);
        return 1;
 }
 
 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
-                          unsigned int port, void *data,  unsigned int count)
+                          unsigned int port)
 {
-       int ret;
-
-       ret = emulator_pio_out_emulated(vcpu->arch.emulate_ctxt, size, port,
-                                       data, count);
-       if (ret)
-               return ret;
+       for (;;) {
+               unsigned int count =
+                       min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
+               int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);
+
+               /* memcpy done already by emulator_pio_out.  */
+               vcpu->arch.sev_pio_count -= count;
+               vcpu->arch.sev_pio_data += count * vcpu->arch.pio.size;
+               if (!ret)
+                       break;
 
-       vcpu->arch.pio.count = 0;
+               /* Emulation done by the kernel.  */
+               if (!vcpu->arch.sev_pio_count)
+                       return 1;
+       }
 
+       vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;
        return 0;
 }
 
 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
-                         unsigned int port, void *data, unsigned int count)
+                         unsigned int port);
+
+static void advance_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
 {
-       int ret;
+       unsigned count = vcpu->arch.pio.count;
+       complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);
+       vcpu->arch.sev_pio_count -= count;
+       vcpu->arch.sev_pio_data += count * vcpu->arch.pio.size;
+}
 
-       ret = emulator_pio_in_emulated(vcpu->arch.emulate_ctxt, size, port,
-                                      data, count);
-       if (ret) {
-               vcpu->arch.pio.count = 0;
-       } else {
-               vcpu->arch.guest_ins_data = data;
-               vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
+static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
+{
+       int size = vcpu->arch.pio.size;
+       int port = vcpu->arch.pio.port;
+
+       advance_sev_es_emulated_ins(vcpu);
+       if (vcpu->arch.sev_pio_count)
+               return kvm_sev_es_ins(vcpu, size, port);
+       return 1;
+}
+
+static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
+                         unsigned int port)
+{
+       for (;;) {
+               unsigned int count =
+                       min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
+               if (!__emulator_pio_in(vcpu, size, port, count))
+                       break;
+
+               /* Emulation done by the kernel.  */
+               advance_sev_es_emulated_ins(vcpu);
+               if (!vcpu->arch.sev_pio_count)
+                       return 1;
        }
 
+       vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
        return 0;
 }
 
@@ -12684,8 +12559,10 @@ int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
                         unsigned int port, void *data,  unsigned int count,
                         int in)
 {
-       return in ? kvm_sev_es_ins(vcpu, size, port, data, count)
-                 : kvm_sev_es_outs(vcpu, size, port, data, count);
+       vcpu->arch.sev_pio_data = data;
+       vcpu->arch.sev_pio_count = count;
+       return in ? kvm_sev_es_ins(vcpu, size, port)
+                 : kvm_sev_es_outs(vcpu, size, port);
 }
 EXPORT_SYMBOL_GPL(kvm_sev_es_string_io);