arm64: entry: move el1 irq/nmi logic to C

In preparation for reworking the EL1 irq/nmi entry code, move the
existing logic to C. We no longer need the asm_nmi_enter() and
asm_nmi_exit() wrappers, so these are removed. The new C functions are
marked noinstr, which prevents compiler instrumentation and runtime
probing.

In subsequent patches we'll want the new C helpers to be called in all
cases, so we don't bother wrapping the calls with ifdeferry. Even when
the new C functions are stubs the trivial calls are unlikely to have a
measurable impact on the IRQ or NMI paths anyway.

Prototypes are added to <asm/exception.h> as otherwise (in some
configurations) GCC will complain about the lack of a forward
declaration. We already do this for existing function, e.g.
enter_from_user_mode().

The new helpers are marked as noinstr (which prevents all
instrumentation, tracing, and kprobes). Otherwise, there should be no
functional change as a result of this patch.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201130115950.22492-7-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 99b9383cd036d7e3744d6f1b4bae9c9c3cd366e8..d69d53dd7be78388eed37f0a0ce24e26e524a8ea 100644 (file)
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -31,6 +31,8 @@ static inline u32 disr_to_esr(u64 disr)
        return esr;
 }
 
+asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs);
+asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs);
 asmlinkage void enter_from_user_mode(void);
 void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
 void do_undefinstr(struct pt_regs *regs);

diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 9a685e7686feae54b017568efa82b99e646868a1..920da254be1da35c7df631afc08c04e13df46ac5 100644 (file)
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -17,6 +17,22 @@
 #include <asm/mmu.h>
 #include <asm/sysreg.h>
 
+asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
+{
+       if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
+               nmi_enter();
+
+       trace_hardirqs_off();
+}
+
+asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
+{
+       if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
+               nmi_exit();
+       else
+               trace_hardirqs_on();
+}
+
 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
 {
        unsigned long far = read_sysreg(far_el1);

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 84aec600eeed4ccee569f379ee0d4f7ab8bb0e84..53e30750fc2886cafef128053be90830533ba668 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -637,16 +637,8 @@ SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
        gic_prio_irq_setup pmr=x20, tmp=x1
        enable_da_f
 
-#ifdef CONFIG_ARM64_PSEUDO_NMI
-       test_irqs_unmasked      res=x0, pmr=x20
-       cbz     x0, 1f
-       bl      asm_nmi_enter
-1:
-#endif
-
-#ifdef CONFIG_TRACE_IRQFLAGS
-       bl      trace_hardirqs_off
-#endif
+       mov     x0, sp
+       bl      enter_el1_irq_or_nmi
 
        irq_handler
 
@@ -665,26 +657,8 @@ alternative_else_nop_endif
 1:
 #endif
 
-#ifdef CONFIG_ARM64_PSEUDO_NMI
-       /*
-        * When using IRQ priority masking, we can get spurious interrupts while
-        * PMR is set to GIC_PRIO_IRQOFF. An NMI might also have occurred in a
-        * section with interrupts disabled. Skip tracing in those cases.
-        */
-       test_irqs_unmasked      res=x0, pmr=x20
-       cbz     x0, 1f
-       bl      asm_nmi_exit
-1:
-#endif
-
-#ifdef CONFIG_TRACE_IRQFLAGS
-#ifdef CONFIG_ARM64_PSEUDO_NMI
-       test_irqs_unmasked      res=x0, pmr=x20
-       cbnz    x0, 1f
-#endif
-       bl      trace_hardirqs_on
-1:
-#endif
+       mov     x0, sp
+       bl      exit_el1_irq_or_nmi
 
        kernel_exit 1
 SYM_CODE_END(el1_irq)

diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 9cf2fb87584aed83d64c1c38468c1dd9ea8a2e67..60456a62da11660f3e4316e097501eddadcd7542 100644 (file)
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -67,18 +67,3 @@ void __init init_IRQ(void)
                local_daif_restore(DAIF_PROCCTX_NOIRQ);
        }
 }
-
-/*
- * Stubs to make nmi_enter/exit() code callable from ASM
- */
-asmlinkage void notrace asm_nmi_enter(void)
-{
-       nmi_enter();
-}
-NOKPROBE_SYMBOL(asm_nmi_enter);
-
-asmlinkage void notrace asm_nmi_exit(void)
-{
-       nmi_exit();
-}
-NOKPROBE_SYMBOL(asm_nmi_exit);