mm: Introduce flush_cache_vmap_early()

[ Upstream commit 7a92fc8b4d20680e4c20289a670d8fca2d1f2c1b ]

The pcpu setup when using the page allocator sets up a new vmalloc
mapping very early in the boot process, so early that it cannot use the
flush_cache_vmap() function which may depend on structures not yet
initialized (for example in riscv, we currently send an IPI to flush
other cpus TLB).

But on some architectures, we must call flush_cache_vmap(): for example,
in riscv, some uarchs can cache invalid TLB entries so we need to flush
the new established mapping to avoid taking an exception.

So fix this by introducing a new function flush_cache_vmap_early() which
is called right after setting the new page table entry and before
accessing this new mapping. This new function implements a local flush
tlb on riscv and is no-op for other architectures (same as today).

Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Signed-off-by: Dennis Zhou <dennis@kernel.org>
Stable-dep-of: d9807d60c145 ("riscv: mm: execute local TLB flush after populating vmemmap")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/arc/include/asm/cacheflush.h b/arch/arc/include/asm/cacheflush.h
index bd5b1a9a054402993cddbced4ea16bf396599279..6fc74500a9f52ae8b73fc3ce3529af5d5b2702f3 100644 (file)
--- a/arch/arc/include/asm/cacheflush.h
+++ b/arch/arc/include/asm/cacheflush.h
@@ -40,6 +40,7 @@ void dma_cache_wback(phys_addr_t start, unsigned long sz);
 
 /* TBD: optimize this */
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 #define flush_cache_dup_mm(mm)                 /* called on fork (VIVT only) */

diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index f6181f69577fe538dc4a13a85cc479784ad9ebef..1075534b0a2eeba73be7d0e83b9d0c995e1f8cf9 100644 (file)
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -340,6 +340,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
                dsb(ishst);
 }
 
+#define flush_cache_vmap_early(start, end)     do { } while (0)
+
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 {
        if (!cache_is_vipt_nonaliasing())

diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h
index 908d8b0bc4fdc645f6ff5b06965df704a97f49cf..d011a81575d21e08a85621e8a58c59ba8e5e7dd8 100644 (file)
--- a/arch/csky/abiv1/inc/abi/cacheflush.h
+++ b/arch/csky/abiv1/inc/abi/cacheflush.h
@@ -43,6 +43,7 @@ static inline void flush_anon_page(struct vm_area_struct *vma,
  */
 extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 #define flush_cache_vmap(start, end)           cache_wbinv_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         cache_wbinv_all()
 
 #define flush_icache_range(start, end)         cache_wbinv_range(start, end)

diff --git a/arch/csky/abiv2/inc/abi/cacheflush.h b/arch/csky/abiv2/inc/abi/cacheflush.h
index 40be16907267d673581278512742567213187a38..6513ac5d257888fbd41385c9263305dfefd18de6 100644 (file)
--- a/arch/csky/abiv2/inc/abi/cacheflush.h
+++ b/arch/csky/abiv2/inc/abi/cacheflush.h
@@ -41,6 +41,7 @@ void flush_icache_mm_range(struct mm_struct *mm,
 void flush_icache_deferred(struct mm_struct *mm);
 
 #define flush_cache_vmap(start, end)           do { } while (0)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         do { } while (0)
 
 #define copy_to_user_page(vma, page, vaddr, dst, src, len) \

diff --git a/arch/m68k/include/asm/cacheflush_mm.h b/arch/m68k/include/asm/cacheflush_mm.h
index ed12358c4783b468ae834106925732ef5875c772..9a71b0148461a4551fe4aae49ca9cf8fea6d46fe 100644 (file)
--- a/arch/m68k/include/asm/cacheflush_mm.h
+++ b/arch/m68k/include/asm/cacheflush_mm.h
@@ -191,6 +191,7 @@ extern void cache_push_v(unsigned long vaddr, int len);
 #define flush_cache_all() __flush_cache_all()
 
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 static inline void flush_cache_mm(struct mm_struct *mm)

diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h
index f36c2519ed9768b8eb570e9b8feb0f1ad0bf8de5..1f14132b3fc98afb6c44de0b7efda4d820978278 100644 (file)
--- a/arch/mips/include/asm/cacheflush.h
+++ b/arch/mips/include/asm/cacheflush.h
@@ -97,6 +97,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
                __flush_cache_vmap();
 }
 
+#define flush_cache_vmap_early(start, end)     do { } while (0)
+
 extern void (*__flush_cache_vunmap)(void);
 
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)

diff --git a/arch/nios2/include/asm/cacheflush.h b/arch/nios2/include/asm/cacheflush.h
index 348cea0977927a523022217bc3637b498a8e4185..81484a776b333a2d9c9b402461f296b3b091e219 100644 (file)
--- a/arch/nios2/include/asm/cacheflush.h
+++ b/arch/nios2/include/asm/cacheflush.h
@@ -38,6 +38,7 @@ void flush_icache_pages(struct vm_area_struct *vma, struct page *page,
 #define flush_icache_pages flush_icache_pages
 
 #define flush_cache_vmap(start, end)           flush_dcache_range(start, end)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_dcache_range(start, end)
 
 extern void copy_to_user_page(struct vm_area_struct *vma, struct page *page,

diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h
index b4006f2a97052da67eaf3d5dcbac95197dc0fd69..ba4c05bc24d6901124deb89152c82a62aa3f5e4d 100644 (file)
--- a/arch/parisc/include/asm/cacheflush.h
+++ b/arch/parisc/include/asm/cacheflush.h
@@ -41,6 +41,7 @@ void flush_kernel_vmap_range(void *vaddr, int size);
 void invalidate_kernel_vmap_range(void *vaddr, int size);
 
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 void flush_dcache_folio(struct folio *folio);

diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h
index 3cb53c4df27cfe4772f2f74623fc861f6a72360f..a129dac4521d35d69af22f713f5cdad27edf7f86 100644 (file)
--- a/arch/riscv/include/asm/cacheflush.h
+++ b/arch/riscv/include/asm/cacheflush.h
@@ -37,7 +37,8 @@ static inline void flush_dcache_page(struct page *page)
        flush_icache_mm(vma->vm_mm, 0)
 
 #ifdef CONFIG_64BIT
-#define flush_cache_vmap(start, end)   flush_tlb_kernel_range(start, end)
+#define flush_cache_vmap(start, end)           flush_tlb_kernel_range(start, end)
+#define flush_cache_vmap_early(start, end)     local_flush_tlb_kernel_range(start, end)
 #endif
 
 #ifndef CONFIG_SMP

diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 8f3418c5f1724ba45e412ca52e0ef59ba0140638..a60416bbe19046a4aa2c303d6eb29d9bf1c0f2b6 100644 (file)
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -41,6 +41,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr);
 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
                     unsigned long end);
 void flush_tlb_kernel_range(unsigned long start, unsigned long end);
+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end);
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,

diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 8723adc884c7c18d77f700c107fd6b5e5dd2e161..b1ab6cf78e9ec4bc7c229b045d8379762cd5a816 100644 (file)
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -65,6 +65,11 @@ static inline void local_flush_tlb_range_asid(unsigned long start,
                local_flush_tlb_range_threshold_asid(start, size, stride, asid);
 }
 
+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+       local_flush_tlb_range_asid(start, end, PAGE_SIZE, FLUSH_TLB_NO_ASID);
+}
+
 static void __ipi_flush_tlb_all(void *info)
 {
        local_flush_tlb_all();

diff --git a/arch/sh/include/asm/cacheflush.h b/arch/sh/include/asm/cacheflush.h
index 878b6b551bd2d0119dd17f2918b73d3ae4120a77..51112f54552b329a307577a5a047f97172d56381 100644 (file)
--- a/arch/sh/include/asm/cacheflush.h
+++ b/arch/sh/include/asm/cacheflush.h
@@ -90,6 +90,7 @@ extern void copy_from_user_page(struct vm_area_struct *vma,
        unsigned long len);
 
 #define flush_cache_vmap(start, end)           local_flush_cache_all(NULL)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         local_flush_cache_all(NULL)
 
 #define flush_dcache_mmap_lock(mapping)                do { } while (0)

diff --git a/arch/sparc/include/asm/cacheflush_32.h b/arch/sparc/include/asm/cacheflush_32.h
index f3b7270bf71b26ae7dcf77378d2b336363f307f5..9fee0ccfccb8e1b95a9d21ab293774fd6797eeb7 100644 (file)
--- a/arch/sparc/include/asm/cacheflush_32.h
+++ b/arch/sparc/include/asm/cacheflush_32.h
@@ -48,6 +48,7 @@ static inline void flush_dcache_page(struct page *page)
 #define flush_dcache_mmap_unlock(mapping)      do { } while (0)
 
 #define flush_cache_vmap(start, end)           flush_cache_all()
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         flush_cache_all()
 
 /* When a context switch happens we must flush all user windows so that

diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h
index 0e879004efff16e69afadb5867731960f20f9781..2b1261b77ecd1b9f93bdecbd1fa08a1985b08171 100644 (file)
--- a/arch/sparc/include/asm/cacheflush_64.h
+++ b/arch/sparc/include/asm/cacheflush_64.h
@@ -75,6 +75,7 @@ void flush_ptrace_access(struct vm_area_struct *, struct page *,
 #define flush_dcache_mmap_unlock(mapping)      do { } while (0)
 
 #define flush_cache_vmap(start, end)           do { } while (0)
+#define flush_cache_vmap_early(start, end)     do { } while (0)
 #define flush_cache_vunmap(start, end)         do { } while (0)
 
 #endif /* !__ASSEMBLY__ */

diff --git a/arch/xtensa/include/asm/cacheflush.h b/arch/xtensa/include/asm/cacheflush.h
index 785a00ce83c11e8bbfa8e02b131315606060c35c..38bcecb0e457d9741c142cada4d38ec65ff0f88b 100644 (file)
--- a/arch/xtensa/include/asm/cacheflush.h
+++ b/arch/xtensa/include/asm/cacheflush.h
@@ -116,8 +116,9 @@ void flush_cache_page(struct vm_area_struct*,
 #define flush_cache_mm(mm)             flush_cache_all()
 #define flush_cache_dup_mm(mm)         flush_cache_mm(mm)
 
-#define flush_cache_vmap(start,end)    flush_cache_all()
-#define flush_cache_vunmap(start,end)  flush_cache_all()
+#define flush_cache_vmap(start,end)            flush_cache_all()
+#define flush_cache_vmap_early(start,end)      do { } while (0)
+#define flush_cache_vunmap(start,end)          flush_cache_all()
 
 void flush_dcache_folio(struct folio *folio);
 #define flush_dcache_folio flush_dcache_folio
@@ -140,6 +141,7 @@ void local_flush_cache_page(struct vm_area_struct *vma,
 #define flush_cache_dup_mm(mm)                         do { } while (0)
 
 #define flush_cache_vmap(start,end)                    do { } while (0)
+#define flush_cache_vmap_early(start,end)              do { } while (0)
 #define flush_cache_vunmap(start,end)                  do { } while (0)
 
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0

diff --git a/include/asm-generic/cacheflush.h b/include/asm-generic/cacheflush.h
index 84ec53ccc450296f264cad5952b4cc26e73dad40..7ee8a179d1036e1d8010b8b18a8f3022e41c1695 100644 (file)
--- a/include/asm-generic/cacheflush.h
+++ b/include/asm-generic/cacheflush.h
@@ -91,6 +91,12 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end)
 }
 #endif
 
+#ifndef flush_cache_vmap_early
+static inline void flush_cache_vmap_early(unsigned long start, unsigned long end)
+{
+}
+#endif
+
 #ifndef flush_cache_vunmap
 static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 {

diff --git a/mm/percpu.c b/mm/percpu.c
index a7665de8485fd9594c523c68ad579b1303e1c4ba..d287cebd58caa31cc2bed534a10bffa807f80048 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -3306,13 +3306,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t
                if (rc < 0)
                        panic("failed to map percpu area, err=%d\n", rc);
 
-               /*
-                * FIXME: Archs with virtual cache should flush local
-                * cache for the linear mapping here - something
-                * equivalent to flush_cache_vmap() on the local cpu.
-                * flush_cache_vmap() can't be used as most supporting
-                * data structures are not set up yet.
-                */
+               flush_cache_vmap_early(unit_addr, unit_addr + ai->unit_size);
 
                /* copy static data */
                memcpy((void *)unit_addr, __per_cpu_load, ai->static_size);