arm64: entry: clarify entry/exit helpers

When entering an exception, we must perform irq/context state management
before we can use instrumentable C code. Similarly, when exiting an
exception we cannot use instrumentable C code after we perform
irq/context state management.

Originally, we'd intended that the enter_from_*() and exit_to_*()
helpers would enforce this by virtue of being the first and last
functions called, respectively, in an exception handler. However, as
they now call instrumentable code themselves, this is not as clearly
true.

To make this more robust, this patch splits the irq/context state
management into separate helpers, with all the helpers commented to make
their intended purpose more obvious.

In exit_to_kernel_mode() we'll now check TFSR_EL1 before we assert that
IRQs are disabled, but this ordering is not important, and other than
this there should be no functional change as a result of this patch.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Joey Gouly <joey.gouly@arm.com>
Link: https://lore.kernel.org/r/20210802140733.52716-3-mark.rutland@arm.com
[catalin.marinas@arm.com: comment typos fix-up]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 6f7a98d8d60f7deb56145bbbe01b06bc4e54f621..c4923e3ba0657fff971368330c76556f4bf25e4c 100644 (file)
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -26,10 +26,14 @@
 #include <asm/system_misc.h>
 
 /*
+ * Handle IRQ/context state management when entering from kernel mode.
+ * Before this function is called it is not safe to call regular kernel code,
+ * intrumentable code, or any code which may trigger an exception.
+ *
  * This is intended to match the logic in irqentry_enter(), handling the kernel
  * mode transitions only.
  */
-static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
+static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)
 {
        regs->exit_rcu = false;
 
@@ -45,20 +49,26 @@ static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
        lockdep_hardirqs_off(CALLER_ADDR0);
        rcu_irq_enter_check_tick();
        trace_hardirqs_off_finish();
+}
 
+static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
+{
+       __enter_from_kernel_mode(regs);
        mte_check_tfsr_entry();
 }
 
 /*
+ * Handle IRQ/context state management when exiting to kernel mode.
+ * After this function returns it is not safe to call regular kernel code,
+ * intrumentable code, or any code which may trigger an exception.
+ *
  * This is intended to match the logic in irqentry_exit(), handling the kernel
  * mode transitions only, and with preemption handled elsewhere.
  */
-static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
+static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)
 {
        lockdep_assert_irqs_disabled();
 
-       mte_check_tfsr_exit();
-
        if (interrupts_enabled(regs)) {
                if (regs->exit_rcu) {
                        trace_hardirqs_on_prepare();
@@ -75,7 +85,18 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
        }
 }
 
-asmlinkage void noinstr enter_from_user_mode(void)
+static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
+{
+       mte_check_tfsr_exit();
+       __exit_to_kernel_mode(regs);
+}
+
+/*
+ * Handle IRQ/context state management when entering from user mode.
+ * Before this function is called it is not safe to call regular kernel code,
+ * intrumentable code, or any code which may trigger an exception.
+ */
+static __always_inline void __enter_from_user_mode(void)
 {
        lockdep_hardirqs_off(CALLER_ADDR0);
        CT_WARN_ON(ct_state() != CONTEXT_USER);
@@ -83,9 +104,18 @@ asmlinkage void noinstr enter_from_user_mode(void)
        trace_hardirqs_off_finish();
 }
 
-asmlinkage void noinstr exit_to_user_mode(void)
+asmlinkage void noinstr enter_from_user_mode(void)
+{
+       __enter_from_user_mode();
+}
+
+/*
+ * Handle IRQ/context state management when exiting to user mode.
+ * After this function returns it is not safe to call regular kernel code,
+ * intrumentable code, or any code which may trigger an exception.
+ */
+static __always_inline void __exit_to_user_mode(void)
 {
-       mte_check_tfsr_exit();
 
        trace_hardirqs_on_prepare();
        lockdep_hardirqs_on_prepare(CALLER_ADDR0);
@@ -93,6 +123,17 @@ asmlinkage void noinstr exit_to_user_mode(void)
        lockdep_hardirqs_on(CALLER_ADDR0);
 }
 
+asmlinkage void noinstr exit_to_user_mode(void)
+{
+       mte_check_tfsr_exit();
+       __exit_to_user_mode();
+}
+
+/*
+ * Handle IRQ/context state management when entering an NMI from user/kernel
+ * mode. Before this function is called it is not safe to call regular kernel
+ * code, intrumentable code, or any code which may trigger an exception.
+ */
 static void noinstr arm64_enter_nmi(struct pt_regs *regs)
 {
        regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
@@ -106,6 +147,11 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs)
        ftrace_nmi_enter();
 }
 
+/*
+ * Handle IRQ/context state management when exiting an NMI from user/kernel
+ * mode. After this function returns it is not safe to call regular kernel
+ * code, intrumentable code, or any code which may trigger an exception.
+ */
 static void noinstr arm64_exit_nmi(struct pt_regs *regs)
 {
        bool restore = regs->lockdep_hardirqs;
@@ -123,6 +169,11 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs)
        __nmi_exit();
 }
 
+/*
+ * Handle IRQ/context state management when entering a debug exception from
+ * kernel mode. Before this function is called it is not safe to call regular
+ * kernel code, intrumentable code, or any code which may trigger an exception.
+ */
 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
 {
        regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
@@ -133,6 +184,11 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
        trace_hardirqs_off_finish();
 }
 
+/*
+ * Handle IRQ/context state management when exiting a debug exception from
+ * kernel mode. After this function returns it is not safe to call regular
+ * kernel code, intrumentable code, or any code which may trigger an exception.
+ */
 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
 {
        bool restore = regs->lockdep_hardirqs;