riscv: Split early and final KASAN population functions

This is a preliminary work that allows to make the code more
understandable.

Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Reviewed-by: Björn Töpel <bjorn@rivosinc.com>
Link: https://lore.kernel.org/r/20230203075232.274282-2-alexghiti@rivosinc.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index e122670..2a48eba 100644 (file)
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -95,23 +95,13 @@ static void __init kasan_populate_pmd(pud_t *pud, unsigned long vaddr, unsigned
 }
 
 static void __init kasan_populate_pud(pgd_t *pgd,
-                                     unsigned long vaddr, unsigned long end,
-                                     bool early)
+                                     unsigned long vaddr, unsigned long end)
 {
        phys_addr_t phys_addr;
        pud_t *pudp, *base_pud;
        unsigned long next;
 
-       if (early) {
-               /*
-                * We can't use pgd_page_vaddr here as it would return a linear
-                * mapping address but it is not mapped yet, but when populating
-                * early_pg_dir, we need the physical address and when populating
-                * swapper_pg_dir, we need the kernel virtual address so use
-                * pt_ops facility.
-                */
-               base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd)));
-       } else if (pgd_none(*pgd)) {
+       if (pgd_none(*pgd)) {
                base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE);
                memcpy(base_pud, (void *)kasan_early_shadow_pud,
                        sizeof(pud_t) * PTRS_PER_PUD);
@@ -130,16 +120,10 @@ static void __init kasan_populate_pud(pgd_t *pgd,
                next = pud_addr_end(vaddr, end);
 
                if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) {
-                       if (early) {
-                               phys_addr = __pa(((uintptr_t)kasan_early_shadow_pmd));
-                               set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE));
+                       phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE);
+                       if (phys_addr) {
+                               set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL));
                                continue;
-                       } else {
-                               phys_addr = memblock_phys_alloc(PUD_SIZE, PUD_SIZE);
-                               if (phys_addr) {
-                                       set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL));
-                                       continue;
-                               }
                        }
                }
 
@@ -152,34 +136,21 @@ static void __init kasan_populate_pud(pgd_t *pgd,
         * it entirely, memblock could allocate a page at a physical address
         * where KASAN is not populated yet and then we'd get a page fault.
         */
-       if (!early)
-               set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
+       set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
 }
 
 static void __init kasan_populate_p4d(pgd_t *pgd,
-                                     unsigned long vaddr, unsigned long end,
-                                     bool early)
+                                     unsigned long vaddr, unsigned long end)
 {
        phys_addr_t phys_addr;
        p4d_t *p4dp, *base_p4d;
        unsigned long next;
 
-       if (early) {
-               /*
-                * We can't use pgd_page_vaddr here as it would return a linear
-                * mapping address but it is not mapped yet, but when populating
-                * early_pg_dir, we need the physical address and when populating
-                * swapper_pg_dir, we need the kernel virtual address so use
-                * pt_ops facility.
-                */
-               base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd)));
-       } else {
-               base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
-               if (base_p4d == lm_alias(kasan_early_shadow_p4d)) {
-                       base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
-                       memcpy(base_p4d, (void *)kasan_early_shadow_p4d,
-                               sizeof(p4d_t) * PTRS_PER_P4D);
-               }
+       base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
+       if (base_p4d == lm_alias(kasan_early_shadow_p4d)) {
+               base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
+               memcpy(base_p4d, (void *)kasan_early_shadow_p4d,
+                      sizeof(p4d_t) * PTRS_PER_P4D);
        }
 
        p4dp = base_p4d + p4d_index(vaddr);
@@ -188,20 +159,14 @@ static void __init kasan_populate_p4d(pgd_t *pgd,
                next = p4d_addr_end(vaddr, end);
 
                if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) {
-                       if (early) {
-                               phys_addr = __pa(((uintptr_t)kasan_early_shadow_pud));
-                               set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
+                       phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
+                       if (phys_addr) {
+                               set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
                                continue;
-                       } else {
-                               phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
-                               if (phys_addr) {
-                                       set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
-                                       continue;
-                               }
                        }
                }
 
-               kasan_populate_pud((pgd_t *)p4dp, vaddr, next, early);
+               kasan_populate_pud((pgd_t *)p4dp, vaddr, next);
        } while (p4dp++, vaddr = next, vaddr != end);
 
        /*
@@ -210,8 +175,7 @@ static void __init kasan_populate_p4d(pgd_t *pgd,
         * it entirely, memblock could allocate a page at a physical address
         * where KASAN is not populated yet and then we'd get a page fault.
         */
-       if (!early)
-               set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
+       set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
 }
 
 #define kasan_early_shadow_pgd_next                    (pgtable_l5_enabled ?   \
@@ -219,16 +183,15 @@ static void __init kasan_populate_p4d(pgd_t *pgd,
                                                        (pgtable_l4_enabled ?   \
                                (uintptr_t)kasan_early_shadow_pud :             \
                                (uintptr_t)kasan_early_shadow_pmd))
-#define kasan_populate_pgd_next(pgdp, vaddr, next, early)                      \
+#define kasan_populate_pgd_next(pgdp, vaddr, next)                             \
                (pgtable_l5_enabled ?                                           \
-               kasan_populate_p4d(pgdp, vaddr, next, early) :                  \
+               kasan_populate_p4d(pgdp, vaddr, next) :                         \
                (pgtable_l4_enabled ?                                           \
-                       kasan_populate_pud(pgdp, vaddr, next, early) :          \
+                       kasan_populate_pud(pgdp, vaddr, next) :                 \
                        kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))
 
 static void __init kasan_populate_pgd(pgd_t *pgdp,
-                                     unsigned long vaddr, unsigned long end,
-                                     bool early)
+                                     unsigned long vaddr, unsigned long end)
 {
        phys_addr_t phys_addr;
        unsigned long next;
@@ -237,11 +200,7 @@ static void __init kasan_populate_pgd(pgd_t *pgdp,
                next = pgd_addr_end(vaddr, end);
 
                if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) {
-                       if (early) {
-                               phys_addr = __pa((uintptr_t)kasan_early_shadow_pgd_next);
-                               set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE));
-                               continue;
-                       } else if (pgd_page_vaddr(*pgdp) ==
+                       if (pgd_page_vaddr(*pgdp) ==
                                   (unsigned long)lm_alias(kasan_early_shadow_pgd_next)) {
                                /*
                                 * pgdp can't be none since kasan_early_init
@@ -258,7 +217,95 @@ static void __init kasan_populate_pgd(pgd_t *pgdp,
                        }
                }
 
-               kasan_populate_pgd_next(pgdp, vaddr, next, early);
+               kasan_populate_pgd_next(pgdp, vaddr, next);
+       } while (pgdp++, vaddr = next, vaddr != end);
+}
+
+static void __init kasan_early_populate_pud(p4d_t *p4dp,
+                                           unsigned long vaddr,
+                                           unsigned long end)
+{
+       pud_t *pudp, *base_pud;
+       phys_addr_t phys_addr;
+       unsigned long next;
+
+       if (!pgtable_l4_enabled) {
+               pudp = (pud_t *)p4dp;
+       } else {
+               base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(*p4dp)));
+               pudp = base_pud + pud_index(vaddr);
+       }
+
+       do {
+               next = pud_addr_end(vaddr, end);
+
+               if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) &&
+                   (next - vaddr) >= PUD_SIZE) {
+                       phys_addr = __pa((uintptr_t)kasan_early_shadow_pmd);
+                       set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_TABLE));
+                       continue;
+               }
+
+               BUG();
+       } while (pudp++, vaddr = next, vaddr != end);
+}
+
+static void __init kasan_early_populate_p4d(pgd_t *pgdp,
+                                           unsigned long vaddr,
+                                           unsigned long end)
+{
+       p4d_t *p4dp, *base_p4d;
+       phys_addr_t phys_addr;
+       unsigned long next;
+
+       /*
+        * We can't use pgd_page_vaddr here as it would return a linear
+        * mapping address but it is not mapped yet, but when populating
+        * early_pg_dir, we need the physical address and when populating
+        * swapper_pg_dir, we need the kernel virtual address so use
+        * pt_ops facility.
+        * Note that this test is then completely equivalent to
+        * p4dp = p4d_offset(pgdp, vaddr)
+        */
+       if (!pgtable_l5_enabled) {
+               p4dp = (p4d_t *)pgdp;
+       } else {
+               base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgdp)));
+               p4dp = base_p4d + p4d_index(vaddr);
+       }
+
+       do {
+               next = p4d_addr_end(vaddr, end);
+
+               if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) &&
+                   (next - vaddr) >= P4D_SIZE) {
+                       phys_addr = __pa((uintptr_t)kasan_early_shadow_pud);
+                       set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
+                       continue;
+               }
+
+               kasan_early_populate_pud(p4dp, vaddr, next);
+       } while (p4dp++, vaddr = next, vaddr != end);
+}
+
+static void __init kasan_early_populate_pgd(pgd_t *pgdp,
+                                           unsigned long vaddr,
+                                           unsigned long end)
+{
+       phys_addr_t phys_addr;
+       unsigned long next;
+
+       do {
+               next = pgd_addr_end(vaddr, end);
+
+               if (pgd_none(*pgdp) && IS_ALIGNED(vaddr, PGDIR_SIZE) &&
+                   (next - vaddr) >= PGDIR_SIZE) {
+                       phys_addr = __pa((uintptr_t)kasan_early_shadow_p4d);
+                       set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_TABLE));
+                       continue;
+               }
+
+               kasan_early_populate_p4d(pgdp, vaddr, next);
        } while (pgdp++, vaddr = next, vaddr != end);
 }
 
@@ -295,16 +342,16 @@ asmlinkage void __init kasan_early_init(void)
                                        PAGE_TABLE));
        }
 
-       kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
-                          KASAN_SHADOW_START, KASAN_SHADOW_END, true);
+       kasan_early_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
+                                KASAN_SHADOW_START, KASAN_SHADOW_END);
 
        local_flush_tlb_all();
 }
 
 void __init kasan_swapper_init(void)
 {
-       kasan_populate_pgd(pgd_offset_k(KASAN_SHADOW_START),
-                          KASAN_SHADOW_START, KASAN_SHADOW_END, true);
+       kasan_early_populate_pgd(pgd_offset_k(KASAN_SHADOW_START),
+                                KASAN_SHADOW_START, KASAN_SHADOW_END);
 
        local_flush_tlb_all();
 }
@@ -314,7 +361,7 @@ static void __init kasan_populate(void *start, void *end)
        unsigned long vaddr = (unsigned long)start & PAGE_MASK;
        unsigned long vend = PAGE_ALIGN((unsigned long)end);
 
-       kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend, false);
+       kasan_populate_pgd(pgd_offset_k(vaddr), vaddr, vend);
 
        local_flush_tlb_all();
        memset(start, KASAN_SHADOW_INIT, end - start);