arm64: Fix HCR.TGE status for NMI contexts

commit 5870970b9a828d8693aa6d15742573289d7dbcd0 upstream.

When using VHE, the host needs to clear HCR_EL2.TGE bit in order
to interact with guest TLBs, switching from EL2&0 translation regime
to EL1&0.

However, some non-maskable asynchronous event could happen while TGE is
cleared like SDEI. Because of this address translation operations
relying on EL2&0 translation regime could fail (tlb invalidation,
userspace access, ...).

Fix this by properly setting HCR_EL2.TGE when entering NMI context and
clear it if necessary when returning to the interrupted context.

Signed-off-by: Julien Thierry <julien.thierry@arm.com>
Suggested-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: James Morse <james.morse@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: linux-arch@vger.kernel.org
Cc: stable@vger.kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index 1473fc2..89691c8 100644 (file)
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -17,8 +17,12 @@
 #define __ASM_HARDIRQ_H
 
 #include <linux/cache.h>
 #define __ASM_HARDIRQ_H
 
 #include <linux/cache.h>

+#include <linux/percpu.h>

 #include <linux/threads.h>
 #include <linux/threads.h>

+#include <asm/barrier.h>

 #include <asm/irq.h>
 #include <asm/irq.h>

+#include <asm/kvm_arm.h>
+#include <asm/sysreg.h>

 
 #define NR_IPI 7
 
 
 #define NR_IPI 7
 


@@ -37,6 +41,33 @@ u64 smp_irq_stat_cpu(unsigned int cpu);
 
 #define __ARCH_IRQ_EXIT_IRQS_DISABLED  1
 
 
 #define __ARCH_IRQ_EXIT_IRQS_DISABLED  1
 

+struct nmi_ctx {
+       u64 hcr;
+};
+
+DECLARE_PER_CPU(struct nmi_ctx, nmi_contexts);
+
+#define arch_nmi_enter()                                                       \
+       do {                                                                    \
+               if (is_kernel_in_hyp_mode()) {                                  \
+                       struct nmi_ctx *nmi_ctx = this_cpu_ptr(&nmi_contexts);  \
+                       nmi_ctx->hcr = read_sysreg(hcr_el2);                    \
+                       if (!(nmi_ctx->hcr & HCR_TGE)) {                        \
+                               write_sysreg(nmi_ctx->hcr | HCR_TGE, hcr_el2);  \
+                               isb();                                          \
+                       }                                                       \
+               }                                                               \
+       } while (0)
+
+#define arch_nmi_exit()                                                                \
+       do {                                                                    \
+               if (is_kernel_in_hyp_mode()) {                                  \
+                       struct nmi_ctx *nmi_ctx = this_cpu_ptr(&nmi_contexts);  \
+                       if (!(nmi_ctx->hcr & HCR_TGE))                          \
+                               write_sysreg(nmi_ctx->hcr, hcr_el2);            \
+               }                                                               \
+       } while (0)
+

 static inline void ack_bad_irq(unsigned int irq)
 {
        extern unsigned long irq_err_count;
 static inline void ack_bad_irq(unsigned int irq)
 {
        extern unsigned long irq_err_count;

diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 780a12f..92fa817 100644 (file)
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -33,6 +33,9 @@
 
 unsigned long irq_err_count;
 
 
 unsigned long irq_err_count;
 

+/* Only access this in an NMI enter/exit */
+DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
+

 DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
 
 int arch_show_interrupts(struct seq_file *p, int prec)
 DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
 
 int arch_show_interrupts(struct seq_file *p, int prec)

diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 0fbbcdf..da0af63 100644 (file)
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -60,8 +60,14 @@ extern void irq_enter(void);
  */
 extern void irq_exit(void);
 
  */
 extern void irq_exit(void);
 

+#ifndef arch_nmi_enter
+#define arch_nmi_enter()       do { } while (0)
+#define arch_nmi_exit()                do { } while (0)
+#endif
+

 #define nmi_enter()                                            \
        do {                                                    \
 #define nmi_enter()                                            \
        do {                                                    \

+               arch_nmi_enter();                               \

                printk_nmi_enter();                             \
                lockdep_off();                                  \
                ftrace_nmi_enter();                             \
                printk_nmi_enter();                             \
                lockdep_off();                                  \
                ftrace_nmi_enter();                             \


@@ -80,6 +86,7 @@ extern void irq_exit(void);
                ftrace_nmi_exit();                              \
                lockdep_on();                                   \
                printk_nmi_exit();                              \
                ftrace_nmi_exit();                              \
                lockdep_on();                                   \
                printk_nmi_exit();                              \

+               arch_nmi_exit();                                \

        } while (0)
 
 #endif /* LINUX_HARDIRQ_H */
        } while (0)
 
 #endif /* LINUX_HARDIRQ_H */