arm64: kasan: mte: move GCR_EL1 switch to task switch when KASAN disabled

It is not necessary to write to GCR_EL1 on every kernel entry and
exit when HW tag-based KASAN is disabled because the kernel will not
execute any IRG instructions in that mode. Since accessing GCR_EL1
can be expensive on some microarchitectures, avoid doing so by moving
the access to task switch when HW tag-based KASAN is disabled.

Signed-off-by: Peter Collingbourne <pcc@google.com>
Acked-by: Andrey Konovalov <andreyknvl@gmail.com>
Link: https://linux-review.googlesource.com/id/I78e90d60612a94c24344526f476ac4ff216e10d2
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20210924010655.2886918-1-pcc@google.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index bc6d5a9..2f69ae4 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -168,9 +168,9 @@ alternative_else_nop_endif
 
        .macro mte_set_kernel_gcr, tmp, tmp2
 #ifdef CONFIG_KASAN_HW_TAGS
-alternative_if_not ARM64_MTE
+alternative_cb kasan_hw_tags_enable
        b       1f
-alternative_else_nop_endif
+alternative_cb_end
        mov     \tmp, KERNEL_GCR_EL1
        msr_s   SYS_GCR_EL1, \tmp
 1:
@@ -178,10 +178,10 @@ alternative_else_nop_endif
        .endm
 
        .macro mte_set_user_gcr, tsk, tmp, tmp2
-#ifdef CONFIG_ARM64_MTE
-alternative_if_not ARM64_MTE
+#ifdef CONFIG_KASAN_HW_TAGS
+alternative_cb kasan_hw_tags_enable
        b       1f
-alternative_else_nop_endif
+alternative_cb_end
        ldr     \tmp, [\tsk, #THREAD_MTE_CTRL]
 
        mte_set_gcr \tmp, \tmp2

diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index e5e801b..0cdae08 100644 (file)
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -179,6 +179,30 @@ static void mte_update_sctlr_user(struct task_struct *task)
        task->thread.sctlr_user = sctlr;
 }
 
+static void mte_update_gcr_excl(struct task_struct *task)
+{
+       /*
+        * SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by
+        * mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled.
+        */
+       if (kasan_hw_tags_enabled())
+               return;
+
+       write_sysreg_s(
+               ((task->thread.mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
+                SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND,
+               SYS_GCR_EL1);
+}
+
+void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
+                                __le32 *updptr, int nr_inst)
+{
+       BUG_ON(nr_inst != 1); /* Branch -> NOP */
+
+       if (kasan_hw_tags_enabled())
+               *updptr = cpu_to_le32(aarch64_insn_gen_nop());
+}
+
 void mte_thread_init_user(void)
 {
        if (!system_supports_mte())
@@ -198,6 +222,7 @@ void mte_thread_switch(struct task_struct *next)
                return;
 
        mte_update_sctlr_user(next);
+       mte_update_gcr_excl(next);
 
        /*
         * Check if an async tag exception occurred at EL1.
@@ -243,6 +268,7 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg)
        if (task == current) {
                preempt_disable();
                mte_update_sctlr_user(task);
+               mte_update_gcr_excl(task);
                update_sctlr_el1(task->thread.sctlr_user);
                preempt_enable();
        }

diff --git a/include/linux/kasan.h b/include/linux/kasan.h
index dd874a1..de5f591 100644 (file)
--- a/include/linux/kasan.h
+++ b/include/linux/kasan.h
@@ -89,7 +89,7 @@ static __always_inline bool kasan_enabled(void)
        return static_branch_likely(&kasan_flag_enabled);
 }
 
-static inline bool kasan_has_integrated_init(void)
+static inline bool kasan_hw_tags_enabled(void)
 {
        return kasan_enabled();
 }
@@ -104,7 +104,7 @@ static inline bool kasan_enabled(void)
        return IS_ENABLED(CONFIG_KASAN);
 }
 
-static inline bool kasan_has_integrated_init(void)
+static inline bool kasan_hw_tags_enabled(void)
 {
        return false;
 }
@@ -125,6 +125,11 @@ static __always_inline void kasan_free_pages(struct page *page,
 
 #endif /* CONFIG_KASAN_HW_TAGS */
 
+static inline bool kasan_has_integrated_init(void)
+{
+       return kasan_hw_tags_enabled();
+}
+
 #ifdef CONFIG_KASAN
 
 struct kasan_cache {