arm64: Implement pmem API support

Add a clean-to-point-of-persistence cache maintenance helper, and wire
up the basic architectural support for the pmem driver based on it.

Reviewed-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
[catalin.marinas@arm.com: move arch_*_pmem() functions to arch/arm64/mm/flush.c]
[catalin.marinas@arm.com: change dmb(sy) to dmb(osh)]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index dfd9086..0b0576a 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -960,6 +960,17 @@ config ARM64_UAO
          regular load/store instructions if the cpu does not implement the
          feature.
 
+config ARM64_PMEM
+       bool "Enable support for persistent memory"
+       select ARCH_HAS_PMEM_API
+       help
+         Say Y to enable support for the persistent memory API based on the
+         ARMv8.2 DCPoP feature.
+
+         The feature is detected at runtime, and the kernel will use DC CVAC
+         operations if DC CVAP is not supported (following the behaviour of
+         DC CVAP itself if the system does not define a point of persistence).
+
 endmenu
 
 config ARM64_MODULE_CMODEL_LARGE

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 1b67c37..5d8903c 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -353,6 +353,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
 alternative_else
        dc      civac, \kaddr
 alternative_endif
+       .elseif (\op == cvap)
+alternative_if ARM64_HAS_DCPOP
+       sys 3, c7, c12, 1, \kaddr       // dc cvap
+alternative_else
+       dc      cvac, \kaddr
+alternative_endif
        .else
        dc      \op, \kaddr
        .endif

diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index b4b43a9..76d1cc8 100644 (file)
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -69,6 +69,7 @@ extern void flush_icache_range(unsigned long start, unsigned long end);
 extern void __flush_dcache_area(void *addr, size_t len);
 extern void __inval_dcache_area(void *addr, size_t len);
 extern void __clean_dcache_area_poc(void *addr, size_t len);
+extern void __clean_dcache_area_pop(void *addr, size_t len);
 extern void __clean_dcache_area_pou(void *addr, size_t len);
 extern long __flush_cache_user_range(unsigned long start, unsigned long end);
 extern void sync_icache_aliases(void *kaddr, unsigned long len);

diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 8d2272c..8da6216 100644 (file)
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -39,7 +39,8 @@
 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003     18
 #define ARM64_WORKAROUND_858921                        19
 #define ARM64_WORKAROUND_CAVIUM_30115          20
+#define ARM64_HAS_DCPOP                                21
 
-#define ARM64_NCAPS                            21
+#define ARM64_NCAPS                            22
 
 #endif /* __ASM_CPUCAPS_H */

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index a2542ef..cd52d36 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -889,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .min_field_value = 0,
                .matches = has_no_fpsimd,
        },
+#ifdef CONFIG_ARM64_PMEM
+       {
+               .desc = "Data cache clean to Point of Persistence",
+               .capability = ARM64_HAS_DCPOP,
+               .def_scope = SCOPE_SYSTEM,
+               .matches = has_cpuid_feature,
+               .sys_reg = SYS_ID_AA64ISAR1_EL1,
+               .field_pos = ID_AA64ISAR1_DPB_SHIFT,
+               .min_field_value = 1,
+       },
+#endif
        {},
 };
 

diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index ed47fbb..7f1dbe9 100644 (file)
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -172,6 +172,20 @@ ENDPIPROC(__clean_dcache_area_poc)
 ENDPROC(__dma_clean_area)
 
 /*
+ *     __clean_dcache_area_pop(kaddr, size)
+ *
+ *     Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ *     are cleaned to the PoP.
+ *
+ *     - kaddr   - kernel address
+ *     - size    - size in question
+ */
+ENTRY(__clean_dcache_area_pop)
+       dcache_by_line_op cvap, sy, x0, x1, x2, x3
+       ret
+ENDPIPROC(__clean_dcache_area_pop)
+
+/*
  *     __dma_flush_area(start, size)
  *
  *     clean & invalidate D / U line

diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index 21a8d82..280f90f 100644 (file)
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page);
  * Additional functions defined in assembly.
  */
 EXPORT_SYMBOL(flush_icache_range);
+
+#ifdef CONFIG_ARCH_HAS_PMEM_API
+static inline void arch_wb_cache_pmem(void *addr, size_t size)
+{
+       /* Ensure order against any prior non-cacheable writes */
+       dmb(osh);
+       __clean_dcache_area_pop(addr, size);
+}
+EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
+
+static inline void arch_invalidate_pmem(void *addr, size_t size)
+{
+       __inval_dcache_area(addr, size);
+}
+EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
+#endif