arm64: Add level-hinted TLB invalidation helper

Add a level-hinted TLB invalidation helper that only gets used if
ARMv8.4-TTL gets detected.

Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/stage2_pgtable.h b/arch/arm64/include/asm/stage2_pgtable.h
index b767904..996bf98 100644 (file)
--- a/arch/arm64/include/asm/stage2_pgtable.h
+++ b/arch/arm64/include/asm/stage2_pgtable.h
@@ -256,4 +256,13 @@ stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
        return (boundary - 1 < end - 1) ? boundary : end;
 }
 
+/*
+ * Level values for the ARMv8.4-TTL extension, mapping PUD/PMD/PTE and
+ * the architectural page-table level.
+ */
+#define S2_NO_LEVEL_HINT       0
+#define S2_PUD_LEVEL           1
+#define S2_PMD_LEVEL           2
+#define S2_PTE_LEVEL           3
+
 #endif /* __ARM64_S2_PGTABLE_H_ */

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index bc39490..3353f26 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -10,6 +10,7 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/bitfield.h>
 #include <linux/mm_types.h>
 #include <linux/sched.h>
 #include <asm/cputype.h>
@@ -60,6 +61,50 @@
        })
 
 /*
+ * Level-based TLBI operations.
+ *
+ * When ARMv8.4-TTL exists, TLBI operations take an additional hint for
+ * the level at which the invalidation must take place. If the level is
+ * wrong, no invalidation may take place. In the case where the level
+ * cannot be easily determined, a 0 value for the level parameter will
+ * perform a non-hinted invalidation.
+ *
+ * For Stage-2 invalidation, use the level values provided to that effect
+ * in asm/stage2_pgtable.h.
+ */
+#define TLBI_TTL_MASK          GENMASK_ULL(47, 44)
+#define TLBI_TTL_TG_4K         1
+#define TLBI_TTL_TG_16K                2
+#define TLBI_TTL_TG_64K                3
+
+#define __tlbi_level(op, addr, level)                                  \
+       do {                                                            \
+               u64 arg = addr;                                         \
+                                                                       \
+               if (cpus_have_const_cap(ARM64_HAS_ARMv8_4_TTL) &&       \
+                   level) {                                            \
+                       u64 ttl = level & 3;                            \
+                                                                       \
+                       switch (PAGE_SIZE) {                            \
+                       case SZ_4K:                                     \
+                               ttl |= TLBI_TTL_TG_4K << 2;             \
+                               break;                                  \
+                       case SZ_16K:                                    \
+                               ttl |= TLBI_TTL_TG_16K << 2;            \
+                               break;                                  \
+                       case SZ_64K:                                    \
+                               ttl |= TLBI_TTL_TG_64K << 2;            \
+                               break;                                  \
+                       }                                               \
+                                                                       \
+                       arg &= ~TLBI_TTL_MASK;                          \
+                       arg |= FIELD_PREP(TLBI_TTL_MASK, ttl);          \
+               }                                                       \
+                                                                       \
+               __tlbi(op, arg);                                        \
+       } while(0)
+
+/*
  *     TLB Invalidation
  *     ================
  *