KVM: PPC: Book3S 64: Move GUEST_MODE_SKIP test into KVM

Move the GUEST_MODE_SKIP logic into KVM code. This is quite a KVM
internal detail that has no real need to be in common handlers.

Add a comment explaining the what and why of KVM "skip" interrupts.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Reviewed-by: Daniel Axtens <dja@axtens.net>
Reviewed-by: Fabiano Rosas <farosas@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210528090752.3542186-3-npiggin@gmail.com

diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 868077f..192b927 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -133,7 +133,6 @@ name:
 #define IBRANCH_TO_COMMON      .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */
 #define IREALMODE_COMMON       .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */
 #define IMASK          .L_IMASK_\name\()       /* IRQ soft-mask bit */
-#define IKVM_SKIP      .L_IKVM_SKIP_\name\()   /* Generate KVM skip handler */
 #define IKVM_REAL      .L_IKVM_REAL_\name\()   /* Real entry tests KVM */
 #define __IKVM_REAL(name)      .L_IKVM_REAL_ ## name
 #define IKVM_VIRT      .L_IKVM_VIRT_\name\()   /* Virt entry tests KVM */
@@ -190,9 +189,6 @@ do_define_int n
        .ifndef IMASK
                IMASK=0
        .endif
-       .ifndef IKVM_SKIP
-               IKVM_SKIP=0
-       .endif
        .ifndef IKVM_REAL
                IKVM_REAL=0
        .endif
@@ -250,15 +246,10 @@ do_define_int n
        .balign IFETCH_ALIGN_BYTES
 \name\()_kvm:
 
-       .if IKVM_SKIP
-       cmpwi   r10,KVM_GUEST_MODE_SKIP
-       beq     89f
-       .else
 BEGIN_FTR_SECTION
        ld      r10,IAREA+EX_CFAR(r13)
        std     r10,HSTATE_CFAR(r13)
 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
-       .endif
 
        ld      r10,IAREA+EX_CTR(r13)
        mtctr   r10
@@ -285,27 +276,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
        ori     r12,r12,(IVEC)
        .endif
        b       kvmppc_interrupt
-
-       .if IKVM_SKIP
-89:    mtocrf  0x80,r9
-       ld      r10,IAREA+EX_CTR(r13)
-       mtctr   r10
-       ld      r9,IAREA+EX_R9(r13)
-       ld      r10,IAREA+EX_R10(r13)
-       ld      r11,IAREA+EX_R11(r13)
-       ld      r12,IAREA+EX_R12(r13)
-       .if IHSRR_IF_HVMODE
-       BEGIN_FTR_SECTION
-       b       kvmppc_skip_Hinterrupt
-       FTR_SECTION_ELSE
-       b       kvmppc_skip_interrupt
-       ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
-       .elseif IHSRR
-       b       kvmppc_skip_Hinterrupt
-       .else
-       b       kvmppc_skip_interrupt
-       .endif
-       .endif
 .endm
 
 #else
@@ -1064,7 +1034,6 @@ INT_DEFINE_BEGIN(machine_check)
        ISET_RI=0
        IDAR=1
        IDSISR=1
-       IKVM_SKIP=1
        IKVM_REAL=1
 INT_DEFINE_END(machine_check)
 
@@ -1336,7 +1305,6 @@ INT_DEFINE_BEGIN(data_access)
        IVEC=0x300
        IDAR=1
        IDSISR=1
-       IKVM_SKIP=1
        IKVM_REAL=1
 INT_DEFINE_END(data_access)
 
@@ -1390,7 +1358,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
 INT_DEFINE_BEGIN(data_access_slb)
        IVEC=0x380
        IDAR=1
-       IKVM_SKIP=1
        IKVM_REAL=1
 INT_DEFINE_END(data_access_slb)
 
@@ -2057,7 +2024,6 @@ INT_DEFINE_BEGIN(h_data_storage)
        IHSRR=1
        IDAR=1
        IDSISR=1
-       IKVM_SKIP=1
        IKVM_REAL=1
        IKVM_VIRT=1
 INT_DEFINE_END(h_data_storage)
@@ -3003,32 +2969,6 @@ EXPORT_SYMBOL(do_uaccess_flush)
 MASKED_INTERRUPT
 MASKED_INTERRUPT hsrr=1
 
-#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
-kvmppc_skip_interrupt:
-       /*
-        * Here all GPRs are unchanged from when the interrupt happened
-        * except for r13, which is saved in SPRG_SCRATCH0.
-        */
-       mfspr   r13, SPRN_SRR0
-       addi    r13, r13, 4
-       mtspr   SPRN_SRR0, r13
-       GET_SCRATCH0(r13)
-       RFI_TO_KERNEL
-       b       .
-
-kvmppc_skip_Hinterrupt:
-       /*
-        * Here all GPRs are unchanged from when the interrupt happened
-        * except for r13, which is saved in SPRG_SCRATCH0.
-        */
-       mfspr   r13, SPRN_HSRR0
-       addi    r13, r13, 4
-       mtspr   SPRN_HSRR0, r13
-       GET_SCRATCH0(r13)
-       HRFI_TO_KERNEL
-       b       .
-#endif
-
        /*
         * Relocation-on interrupts: A subset of the interrupts can be delivered
         * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering

diff --git a/arch/powerpc/kvm/book3s_64_entry.S b/arch/powerpc/kvm/book3s_64_entry.S
index 7a039ea..bf927e7 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_entry.S
+++ b/arch/powerpc/kvm/book3s_64_entry.S
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 #include <asm/asm-offsets.h>
 #include <asm/cache.h>
+#include <asm/exception-64s.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_book3s_asm.h>
 #include <asm/ppc_asm.h>
@@ -20,9 +21,12 @@ kvmppc_interrupt:
         * guest R12 saved in shadow VCPU SCRATCH0
         * guest R13 saved in SPRN_SCRATCH0
         */
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
        std     r9,HSTATE_SCRATCH2(r13)
        lbz     r9,HSTATE_IN_GUEST(r13)
+       cmpwi   r9,KVM_GUEST_MODE_SKIP
+       beq-    .Lmaybe_skip
+.Lno_skip:
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
        cmpwi   r9,KVM_GUEST_MODE_HOST_HV
        beq     kvmppc_bad_host_intr
 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
@@ -32,5 +36,58 @@ kvmppc_interrupt:
 #endif
        b       kvmppc_interrupt_hv
 #else
+       ld      r9,HSTATE_SCRATCH2(r13)
        b       kvmppc_interrupt_pr
 #endif
+
+/*
+ * "Skip" interrupts are part of a trick KVM uses a with hash guests to load
+ * the faulting instruction in guest memory from the the hypervisor without
+ * walking page tables.
+ *
+ * When the guest takes a fault that requires the hypervisor to load the
+ * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
+ * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
+ * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
+ * but loads and stores will access the guest context. This is used to load
+ * the faulting instruction using the faulting guest effective address.
+ *
+ * However the guest context may not be able to translate, or it may cause a
+ * machine check or other issue, which results in a fault in the host
+ * (even with KVM-HV).
+ *
+ * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
+ * are (or are likely) caused by that load, the instruction is skipped by
+ * just returning with the PC advanced +4, where it is noticed the load did
+ * not execute and it goes to the slow path which walks the page tables to
+ * read guest memory.
+ */
+.Lmaybe_skip:
+       cmpwi   r12,BOOK3S_INTERRUPT_MACHINE_CHECK
+       beq     1f
+       cmpwi   r12,BOOK3S_INTERRUPT_DATA_STORAGE
+       beq     1f
+       cmpwi   r12,BOOK3S_INTERRUPT_DATA_SEGMENT
+       beq     1f
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+       /* HSRR interrupts get 2 added to interrupt number */
+       cmpwi   r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
+       beq     2f
+#endif
+       b       .Lno_skip
+1:     mfspr   r9,SPRN_SRR0
+       addi    r9,r9,4
+       mtspr   SPRN_SRR0,r9
+       ld      r12,HSTATE_SCRATCH0(r13)
+       ld      r9,HSTATE_SCRATCH2(r13)
+       GET_SCRATCH0(r13)
+       RFI_TO_KERNEL
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+2:     mfspr   r9,SPRN_HSRR0
+       addi    r9,r9,4
+       mtspr   SPRN_HSRR0,r9
+       ld      r12,HSTATE_SCRATCH0(r13)
+       ld      r9,HSTATE_SCRATCH2(r13)
+       GET_SCRATCH0(r13)
+       HRFI_TO_KERNEL
+#endif