riscv: mm: Support kasan for sv57

This patchset add kasan_populate and kasan_shallow_populate for sv57,
and is tested on both qemu and unmatched with CONFIG_KASAN and
CONFIG_KASAN_VMALLOC on.

Signed-off-by: Qinglin Pan <panqinglin2020@iscas.ac.cn>
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index f61f7ca..3c99b24 100644 (file)
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -111,6 +111,8 @@ static void __init kasan_populate_pud(pgd_t *pgd,
                 * pt_ops facility.
                 */
                base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd)));
+       } else if (pgd_none(*pgd)) {
+               base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE);
        } else {
                base_pud = (pud_t *)pgd_page_vaddr(*pgd);
                if (base_pud == lm_alias(kasan_early_shadow_pud))
@@ -149,13 +151,72 @@ static void __init kasan_populate_pud(pgd_t *pgd,
                set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
 }
 
-#define kasan_early_shadow_pgd_next                    (pgtable_l4_enabled ?   \
+static void __init kasan_populate_p4d(pgd_t *pgd,
+                                     unsigned long vaddr, unsigned long end,
+                                     bool early)
+{
+       phys_addr_t phys_addr;
+       p4d_t *p4dp, *base_p4d;
+       unsigned long next;
+
+       if (early) {
+               /*
+                * We can't use pgd_page_vaddr here as it would return a linear
+                * mapping address but it is not mapped yet, but when populating
+                * early_pg_dir, we need the physical address and when populating
+                * swapper_pg_dir, we need the kernel virtual address so use
+                * pt_ops facility.
+                */
+               base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd)));
+       } else {
+               base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
+               if (base_p4d == lm_alias(kasan_early_shadow_p4d))
+                       base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
+       }
+
+       p4dp = base_p4d + p4d_index(vaddr);
+
+       do {
+               next = p4d_addr_end(vaddr, end);
+
+               if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) {
+                       if (early) {
+                               phys_addr = __pa(((uintptr_t)kasan_early_shadow_pud));
+                               set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
+                               continue;
+                       } else {
+                               phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
+                               if (phys_addr) {
+                                       set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
+                                       continue;
+                               }
+                       }
+               }
+
+               kasan_populate_pud((pgd_t *)p4dp, vaddr, next, early);
+       } while (p4dp++, vaddr = next, vaddr != end);
+
+       /*
+        * Wait for the whole P4D to be populated before setting the P4D in
+        * the page table, otherwise, if we did set the P4D before populating
+        * it entirely, memblock could allocate a page at a physical address
+        * where KASAN is not populated yet and then we'd get a page fault.
+        */
+       if (!early)
+               set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
+}
+
+#define kasan_early_shadow_pgd_next                    (pgtable_l5_enabled ?   \
+                               (uintptr_t)kasan_early_shadow_p4d :             \
+                                                       (pgtable_l4_enabled ?   \
                                (uintptr_t)kasan_early_shadow_pud :             \
-                               (uintptr_t)kasan_early_shadow_pmd)
+                               (uintptr_t)kasan_early_shadow_pmd))
 #define kasan_populate_pgd_next(pgdp, vaddr, next, early)                      \
+               (pgtable_l5_enabled ?                                           \
+               kasan_populate_p4d(pgdp, vaddr, next, early) :                  \
                (pgtable_l4_enabled ?                                           \
                        kasan_populate_pud(pgdp, vaddr, next, early) :          \
-                       kasan_populate_pmd((pud_t *)pgdp, vaddr, next))
+                       kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))
 
 static void __init kasan_populate_pgd(pgd_t *pgdp,
                                      unsigned long vaddr, unsigned long end,
@@ -219,6 +280,14 @@ asmlinkage void __init kasan_early_init(void)
                                        PAGE_TABLE));
        }
 
+       if (pgtable_l5_enabled) {
+               for (i = 0; i < PTRS_PER_P4D; ++i)
+                       set_p4d(kasan_early_shadow_p4d + i,
+                               pfn_p4d(PFN_DOWN
+                                       (__pa(((uintptr_t)kasan_early_shadow_pud))),
+                                       PAGE_TABLE));
+       }
+
        kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
                           KASAN_SHADOW_START, KASAN_SHADOW_END, true);
 
@@ -244,9 +313,27 @@ static void __init kasan_populate(void *start, void *end)
        memset(start, KASAN_SHADOW_INIT, end - start);
 }
 
+static void __init kasan_shallow_populate_pmd(pgd_t *pgdp,
+                                             unsigned long vaddr, unsigned long end)
+{
+       unsigned long next;
+       pmd_t *pmdp, *base_pmd;
+       bool is_kasan_pte;
+
+       base_pmd = (pmd_t *)pgd_page_vaddr(*pgdp);
+       pmdp = base_pmd + pmd_index(vaddr);
+
+       do {
+               next = pmd_addr_end(vaddr, end);
+               is_kasan_pte = (pmd_pgtable(*pmdp) == lm_alias(kasan_early_shadow_pte));
+
+               if (is_kasan_pte)
+                       pmd_clear(pmdp);
+       } while (pmdp++, vaddr = next, vaddr != end);
+}
+
 static void __init kasan_shallow_populate_pud(pgd_t *pgdp,
-                                             unsigned long vaddr, unsigned long end,
-                                             bool kasan_populate)
+                                             unsigned long vaddr, unsigned long end)
 {
        unsigned long next;
        pud_t *pudp, *base_pud;
@@ -256,21 +343,60 @@ static void __init kasan_shallow_populate_pud(pgd_t *pgdp,
        base_pud = (pud_t *)pgd_page_vaddr(*pgdp);
        pudp = base_pud + pud_index(vaddr);
 
-       if (kasan_populate)
-               memcpy(base_pud, (void *)kasan_early_shadow_pgd_next,
-                      sizeof(pud_t) * PTRS_PER_PUD);
-
        do {
                next = pud_addr_end(vaddr, end);
                is_kasan_pmd = (pud_pgtable(*pudp) == lm_alias(kasan_early_shadow_pmd));
 
-               if (is_kasan_pmd) {
-                       base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
-                       set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE));
-               }
+               if (!is_kasan_pmd)
+                       continue;
+
+               base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+               set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE));
+
+               if (IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE)
+                       continue;
+
+               memcpy(base_pmd, (void *)kasan_early_shadow_pmd, PAGE_SIZE);
+               kasan_shallow_populate_pmd((pgd_t *)pudp, vaddr, next);
        } while (pudp++, vaddr = next, vaddr != end);
 }
 
+static void __init kasan_shallow_populate_p4d(pgd_t *pgdp,
+                                             unsigned long vaddr, unsigned long end)
+{
+       unsigned long next;
+       p4d_t *p4dp, *base_p4d;
+       pud_t *base_pud;
+       bool is_kasan_pud;
+
+       base_p4d = (p4d_t *)pgd_page_vaddr(*pgdp);
+       p4dp = base_p4d + p4d_index(vaddr);
+
+       do {
+               next = p4d_addr_end(vaddr, end);
+               is_kasan_pud = (p4d_pgtable(*p4dp) == lm_alias(kasan_early_shadow_pud));
+
+               if (!is_kasan_pud)
+                       continue;
+
+               base_pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+               set_p4d(p4dp, pfn_p4d(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
+
+               if (IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE)
+                       continue;
+
+               memcpy(base_pud, (void *)kasan_early_shadow_pud, PAGE_SIZE);
+               kasan_shallow_populate_pud((pgd_t *)p4dp, vaddr, next);
+       } while (p4dp++, vaddr = next, vaddr != end);
+}
+
+#define kasan_shallow_populate_pgd_next(pgdp, vaddr, next)                     \
+               (pgtable_l5_enabled ?                                           \
+               kasan_shallow_populate_p4d(pgdp, vaddr, next) :                 \
+               (pgtable_l4_enabled ?                                           \
+               kasan_shallow_populate_pud(pgdp, vaddr, next) :                 \
+               kasan_shallow_populate_pmd(pgdp, vaddr, next)))
+
 static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end)
 {
        unsigned long next;
@@ -291,7 +417,8 @@ static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long
                if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE)
                        continue;
 
-               kasan_shallow_populate_pud(pgd_k, vaddr, next, is_kasan_pgd_next);
+               memcpy(p, (void *)kasan_early_shadow_pgd_next, PAGE_SIZE);
+               kasan_shallow_populate_pgd_next(pgd_k, vaddr, next);
        } while (pgd_k++, vaddr = next, vaddr != end);
 }