arm64: mte: DC {GVA,GZVA} shouldn't be used when DCZID_EL0.DZP == 1

Currently, mte_set_mem_tag_range() and mte_zero_clear_page_tags() use
DC {GVA,GZVA} unconditionally.  But, they should make sure that
DCZID_EL0.DZP, which indicates whether or not use of those instructions
is prohibited, is zero when using those instructions.
Use ST{G,ZG,Z2G} instead when DCZID_EL0.DZP == 1.

Fixes: 013bb59dbb7c ("arm64: mte: handle tags zeroing at page allocation time")
Fixes: 3d0cca0b02ac ("kasan: speed up mte_set_mem_tag_range")
Signed-off-by: Reiji Watanabe <reijiw@google.com>
Link: https://lore.kernel.org/r/20211206004736.1520989-3-reijiw@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
index 478b9bc..e4704a4 100644 (file)
--- a/arch/arm64/include/asm/mte-kasan.h
+++ b/arch/arm64/include/asm/mte-kasan.h
@@ -84,10 +84,12 @@ static inline void __dc_gzva(u64 p)
 static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
                                         bool init)
 {
-       u64 curr, mask, dczid_bs, end1, end2, end3;
+       u64 curr, mask, dczid, dczid_bs, dczid_dzp, end1, end2, end3;
 
        /* Read DC G(Z)VA block size from the system register. */
-       dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf);
+       dczid = read_cpuid(DCZID_EL0);
+       dczid_bs = 4ul << (dczid & 0xf);
+       dczid_dzp = (dczid >> 4) & 1;
 
        curr = (u64)__tag_set(addr, tag);
        mask = dczid_bs - 1;
@@ -106,7 +108,7 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
         */
 #define SET_MEMTAG_RANGE(stg_post, dc_gva)             \
        do {                                            \
-               if (size >= 2 * dczid_bs) {             \
+               if (!dczid_dzp && size >= 2 * dczid_bs) {\
                        do {                            \
                                curr = stg_post(curr);  \
                        } while (curr < end1);          \

diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index e83643b..f531dcb 100644 (file)
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -43,17 +43,23 @@ SYM_FUNC_END(mte_clear_page_tags)
  *     x0 - address to the beginning of the page
  */
 SYM_FUNC_START(mte_zero_clear_page_tags)
+       and     x0, x0, #(1 << MTE_TAG_SHIFT) - 1       // clear the tag
        mrs     x1, dczid_el0
+       tbnz    x1, #4, 2f      // Branch if DC GZVA is prohibited
        and     w1, w1, #0xf
        mov     x2, #4
        lsl     x1, x2, x1
-       and     x0, x0, #(1 << MTE_TAG_SHIFT) - 1       // clear the tag
 
 1:     dc      gzva, x0
        add     x0, x0, x1
        tst     x0, #(PAGE_SIZE - 1)
        b.ne    1b
        ret
+
+2:     stz2g   x0, [x0], #(MTE_GRANULE_SIZE * 2)
+       tst     x0, #(PAGE_SIZE - 1)
+       b.ne    2b
+       ret
 SYM_FUNC_END(mte_zero_clear_page_tags)
 
 /*