KVM: arm64: Add kvm_extable for vaxorcism code

KVM has a one instruction window where it will allow an SError exception
to be consumed by the hypervisor without treating it as a hypervisor bug.
This is used to consume asynchronous external abort that were caused by
the guest.

As we are about to add another location that survives unexpected exceptions,
generalise this code to make it behave like the host's extable.

KVM's version has to be mapped to EL2 to be accessible on nVHE systems.

The SError vaxorcism code is a one instruction window, so has two entries
in the extable. Because the KVM code is copied for VHE and nVHE, we end up
with four entries, half of which correspond with code that isn't mapped.

Signed-off-by: James Morse <james.morse@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index fb1a922b31ba2167d0d51467a17312f25037f436..63350f0e345386a1f32d725d42fb419bef263b93 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -193,6 +193,21 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ];
        ldr     \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
 .endm
 
+/*
+ * KVM extable for unexpected exceptions.
+ * In the same format _asm_extable, but output to a different section so that
+ * it can be mapped to EL2. The KVM version is not sorted. The caller must
+ * ensure:
+ * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
+ * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
+ */
+.macro _kvm_extable, from, to
+       .pushsection    __kvm_ex_table, "a"
+       .align          3
+       .long           (\from - .), (\to - .)
+       .popsection
+.endm
+
 #endif
 
 #endif /* __ARM_KVM_ASM_H__ */

diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 9e897c500237dfe449abdeba47257a9de5eba3b9..8982b68289b79f2d8177ed9e8eb5d861e14df0ef 100644 (file)
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -103,6 +103,10 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
 KVM_NVHE_ALIAS(gic_pmr_sync);
 #endif
 
+/* EL2 exception handling */
+KVM_NVHE_ALIAS(__start___kvm_ex_table);
+KVM_NVHE_ALIAS(__stop___kvm_ex_table);
+
 #endif /* CONFIG_KVM */
 
 #endif /* __ARM64_KERNEL_IMAGE_VARS_H */

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index ec8e894684a78eee91df855949da3a5eb309528b..7cba7623fcec75a7411c3388a3d5b516e46c49cc 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -20,6 +20,13 @@ ENTRY(_text)
 
 jiffies = jiffies_64;
 
+
+#define HYPERVISOR_EXTABLE                                     \
+       . = ALIGN(SZ_8);                                        \
+       __start___kvm_ex_table = .;                             \
+       *(__kvm_ex_table)                                       \
+       __stop___kvm_ex_table = .;
+
 #define HYPERVISOR_TEXT                                        \
        /*                                              \
         * Align to 4 KB so that                        \
@@ -35,6 +42,7 @@ jiffies = jiffies_64;
        __hyp_idmap_text_end = .;                       \
        __hyp_text_start = .;                           \
        *(.hyp.text)                                    \
+       HYPERVISOR_EXTABLE                              \
        __hyp_text_end = .;
 
 #define IDMAP_TEXT                                     \

diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index ee32a77433898da3e24bafcbee5d8c64ea2ab4d4..76e7eaf4675eb4b7a148ab06c4db62c33dc2db96 100644 (file)
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -196,20 +196,23 @@ alternative_endif
        // This is our single instruction exception window. A pending
        // SError is guaranteed to occur at the earliest when we unmask
        // it, and at the latest just after the ISB.
-       .global abort_guest_exit_start
 abort_guest_exit_start:
 
        isb
 
-       .global abort_guest_exit_end
 abort_guest_exit_end:
 
        msr     daifset, #4     // Mask aborts
+       ret
+
+       _kvm_extable    abort_guest_exit_start, 9997f
+       _kvm_extable    abort_guest_exit_end, 9997f
+9997:
+       msr     daifset, #4     // Mask aborts
+       mov     x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
 
-       // If the exception took place, restore the EL1 exception
-       // context so that we can report some information.
-       // Merge the exception code with the SError pending bit.
-       tbz     x0, #ARM_EXIT_WITH_SERROR_BIT, 1f
+       // restore the EL1 exception context so that we can report some
+       // information. Merge the exception code with the SError pending bit.
        msr     elr_el2, x2
        msr     esr_el2, x3
        msr     spsr_el2, x4

diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 689fccbc9de72d84894f0383b469dfeaba8894b8..e47547e276d8d2ddebf365811eeacd9891f7d920 100644 (file)
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -15,6 +15,30 @@
 #include <asm/kvm_mmu.h>
 #include <asm/mmu.h>
 
+.macro save_caller_saved_regs_vect
+       /* x0 and x1 were saved in the vector entry */
+       stp     x2, x3,   [sp, #-16]!
+       stp     x4, x5,   [sp, #-16]!
+       stp     x6, x7,   [sp, #-16]!
+       stp     x8, x9,   [sp, #-16]!
+       stp     x10, x11, [sp, #-16]!
+       stp     x12, x13, [sp, #-16]!
+       stp     x14, x15, [sp, #-16]!
+       stp     x16, x17, [sp, #-16]!
+.endm
+
+.macro restore_caller_saved_regs_vect
+       ldp     x16, x17, [sp], #16
+       ldp     x14, x15, [sp], #16
+       ldp     x12, x13, [sp], #16
+       ldp     x10, x11, [sp], #16
+       ldp     x8, x9,   [sp], #16
+       ldp     x6, x7,   [sp], #16
+       ldp     x4, x5,   [sp], #16
+       ldp     x2, x3,   [sp], #16
+       ldp     x0, x1,   [sp], #16
+.endm
+
        .text
 
 .macro do_el2_call
@@ -157,27 +181,14 @@ el2_sync:
 
 
 el2_error:
-       ldp     x0, x1, [sp], #16
+       save_caller_saved_regs_vect
+       stp     x29, x30, [sp, #-16]!
+
+       bl      kvm_unexpected_el2_exception
+
+       ldp     x29, x30, [sp], #16
+       restore_caller_saved_regs_vect
 
-       /*
-        * Only two possibilities:
-        * 1) Either we come from the exit path, having just unmasked
-        *    PSTATE.A: change the return code to an EL2 fault, and
-        *    carry on, as we're already in a sane state to handle it.
-        * 2) Or we come from anywhere else, and that's a bug: we panic.
-        *
-        * For (1), x0 contains the original return code and x1 doesn't
-        * contain anything meaningful at that stage. We can reuse them
-        * as temp registers.
-        * For (2), who cares?
-        */
-       mrs     x0, elr_el2
-       adr     x1, abort_guest_exit_start
-       cmp     x0, x1
-       adr     x1, abort_guest_exit_end
-       ccmp    x0, x1, #4, ne
-       b.ne    __hyp_panic
-       mov     x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
        eret
        sb
 

diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 426ef65601dd9c82cc71828012bd0ac110514e8b..6d8d4ed4287c0a034a33e5228c7af9495ff79ab2 100644 (file)
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -17,6 +17,7 @@
 
 #include <asm/barrier.h>
 #include <asm/cpufeature.h>
+#include <asm/extable.h>
 #include <asm/kprobes.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
@@ -29,6 +30,9 @@
 
 extern const char __hyp_panic_string[];
 
+extern struct exception_table_entry __start___kvm_ex_table;
+extern struct exception_table_entry __stop___kvm_ex_table;
+
 /* Check whether the FP regs were dirtied while in the host-side run loop: */
 static inline bool update_fp_enabled(struct kvm_vcpu *vcpu)
 {
@@ -508,4 +512,31 @@ static inline void __set_host_arch_workaround_state(struct kvm_vcpu *vcpu)
 #endif
 }
 
+static inline void __kvm_unexpected_el2_exception(void)
+{
+       unsigned long addr, fixup;
+       struct kvm_cpu_context *host_ctxt;
+       struct exception_table_entry *entry, *end;
+       unsigned long elr_el2 = read_sysreg(elr_el2);
+
+       entry = hyp_symbol_addr(__start___kvm_ex_table);
+       end = hyp_symbol_addr(__stop___kvm_ex_table);
+       host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
+
+       while (entry < end) {
+               addr = (unsigned long)&entry->insn + entry->insn;
+               fixup = (unsigned long)&entry->fixup + entry->fixup;
+
+               if (addr != elr_el2) {
+                       entry++;
+                       continue;
+               }
+
+               write_sysreg(fixup, elr_el2);
+               return;
+       }
+
+       hyp_panic(host_ctxt);
+}
+
 #endif /* __ARM64_KVM_HYP_SWITCH_H__ */

diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 341be2f2f31261c36563b67d818d7f52b0c2ec05..0970442d2dbcf779c44d5f88155c65c3705712b8 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -270,3 +270,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
                       read_sysreg(hpfar_el2), par, vcpu);
        unreachable();
 }
+
+asmlinkage void kvm_unexpected_el2_exception(void)
+{
+       return __kvm_unexpected_el2_exception();
+}

diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index c52d714e0d7592962011c709d7d728eb87d25375..c1da4f86ccacac9c65b0525265b2568f94217a57 100644 (file)
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -217,3 +217,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
        __hyp_call_panic(spsr, elr, par, host_ctxt);
        unreachable();
 }
+
+asmlinkage void kvm_unexpected_el2_exception(void)
+{
+       return __kvm_unexpected_el2_exception();
+}