arch/riscv/mm/pageattr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2019 SiFive
   4  */
   5
   6 #include <linux/pagewalk.h>
   7 #include <linux/pgtable.h>
   8 #include <asm/tlbflush.h>
   9 #include <asm/bitops.h>
  10 #include <asm/set_memory.h>
  11
  12 struct pageattr_masks {
  13         pgprot_t set_mask;
  14         pgprot_t clear_mask;
  15 };
  16
  17 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
  18 {
  19         struct pageattr_masks *masks = walk->private;
  20         unsigned long new_val = val;
  21
  22         new_val &= ~(pgprot_val(masks->clear_mask));
  23         new_val |= (pgprot_val(masks->set_mask));
  24
  25         return new_val;
  26 }
  27
  28 static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr,
  29                               unsigned long next, struct mm_walk *walk)
  30 {
  31         pgd_t val = READ_ONCE(*pgd);
  32
  33         if (pgd_leaf(val)) {
  34                 val = __pgd(set_pageattr_masks(pgd_val(val), walk));
  35                 set_pgd(pgd, val);
  36         }
  37
  38         return 0;
  39 }
  40
  41 static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
  42                               unsigned long next, struct mm_walk *walk)
  43 {
  44         p4d_t val = READ_ONCE(*p4d);
  45
  46         if (p4d_leaf(val)) {
  47                 val = __p4d(set_pageattr_masks(p4d_val(val), walk));
  48                 set_p4d(p4d, val);
  49         }
  50
  51         return 0;
  52 }
  53
  54 static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
  55                               unsigned long next, struct mm_walk *walk)
  56 {
  57         pud_t val = READ_ONCE(*pud);
  58
  59         if (pud_leaf(val)) {
  60                 val = __pud(set_pageattr_masks(pud_val(val), walk));
  61                 set_pud(pud, val);
  62         }
  63
  64         return 0;
  65 }
  66
  67 static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
  68                               unsigned long next, struct mm_walk *walk)
  69 {
  70         pmd_t val = READ_ONCE(*pmd);
  71
  72         if (pmd_leaf(val)) {
  73                 val = __pmd(set_pageattr_masks(pmd_val(val), walk));
  74                 set_pmd(pmd, val);
  75         }
  76
  77         return 0;
  78 }
  79
  80 static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
  81                               unsigned long next, struct mm_walk *walk)
  82 {
  83         pte_t val = READ_ONCE(*pte);
  84
  85         val = __pte(set_pageattr_masks(pte_val(val), walk));
  86         set_pte(pte, val);
  87
  88         return 0;
  89 }
  90
  91 static int pageattr_pte_hole(unsigned long addr, unsigned long next,
  92                              int depth, struct mm_walk *walk)
  93 {
  94         /* Nothing to do here */
  95         return 0;
  96 }
  97
  98 static const struct mm_walk_ops pageattr_ops = {
  99         .pgd_entry = pageattr_pgd_entry,
 100         .p4d_entry = pageattr_p4d_entry,
 101         .pud_entry = pageattr_pud_entry,
 102         .pmd_entry = pageattr_pmd_entry,
 103         .pte_entry = pageattr_pte_entry,
 104         .pte_hole = pageattr_pte_hole,
 105         .walk_lock = PGWALK_RDLOCK,
 106 };
 107
 108 static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
 109                         pgprot_t clear_mask)
 110 {
 111         int ret;
 112         unsigned long start = addr;
 113         unsigned long end = start + PAGE_SIZE * numpages;
 114         struct pageattr_masks masks = {
 115                 .set_mask = set_mask,
 116                 .clear_mask = clear_mask
 117         };
 118
 119         if (!numpages)
 120                 return 0;
 121
 122         mmap_write_lock(&init_mm);
 123         ret =  walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
 124                                      &masks);
 125         mmap_write_unlock(&init_mm);
 126
 127         flush_tlb_kernel_range(start, end);
 128
 129         return ret;
 130 }
 131
 132 int set_memory_rw_nx(unsigned long addr, int numpages)
 133 {
 134         return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
 135                             __pgprot(_PAGE_EXEC));
 136 }
 137
 138 int set_memory_ro(unsigned long addr, int numpages)
 139 {
 140         return __set_memory(addr, numpages, __pgprot(_PAGE_READ),
 141                             __pgprot(_PAGE_WRITE));
 142 }
 143
 144 int set_memory_rw(unsigned long addr, int numpages)
 145 {
 146         return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
 147                             __pgprot(0));
 148 }
 149
 150 int set_memory_x(unsigned long addr, int numpages)
 151 {
 152         return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));
 153 }
 154
 155 int set_memory_nx(unsigned long addr, int numpages)
 156 {
 157         return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
 158 }
 159
 160 int set_direct_map_invalid_noflush(struct page *page)
 161 {
 162         int ret;
 163         unsigned long start = (unsigned long)page_address(page);
 164         unsigned long end = start + PAGE_SIZE;
 165         struct pageattr_masks masks = {
 166                 .set_mask = __pgprot(0),
 167                 .clear_mask = __pgprot(_PAGE_PRESENT)
 168         };
 169
 170         mmap_read_lock(&init_mm);
 171         ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks);
 172         mmap_read_unlock(&init_mm);
 173
 174         return ret;
 175 }
 176
 177 int set_direct_map_default_noflush(struct page *page)
 178 {
 179         int ret;
 180         unsigned long start = (unsigned long)page_address(page);
 181         unsigned long end = start + PAGE_SIZE;
 182         struct pageattr_masks masks = {
 183                 .set_mask = PAGE_KERNEL,
 184                 .clear_mask = __pgprot(0)
 185         };
 186
 187         mmap_read_lock(&init_mm);
 188         ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks);
 189         mmap_read_unlock(&init_mm);
 190
 191         return ret;
 192 }
 193
 194 #ifdef CONFIG_DEBUG_PAGEALLOC
 195 void __kernel_map_pages(struct page *page, int numpages, int enable)
 196 {
 197         if (!debug_pagealloc_enabled())
 198                 return;
 199
 200         if (enable)
 201                 __set_memory((unsigned long)page_address(page), numpages,
 202                              __pgprot(_PAGE_PRESENT), __pgprot(0));
 203         else
 204                 __set_memory((unsigned long)page_address(page), numpages,
 205                              __pgprot(0), __pgprot(_PAGE_PRESENT));
 206 }
 207 #endif
 208
 209 bool kernel_page_present(struct page *page)
 210 {
 211         unsigned long addr = (unsigned long)page_address(page);
 212         pgd_t *pgd;
 213         pud_t *pud;
 214         p4d_t *p4d;
 215         pmd_t *pmd;
 216         pte_t *pte;
 217
 218         pgd = pgd_offset_k(addr);
 219         if (!pgd_present(*pgd))
 220                 return false;
 221         if (pgd_leaf(*pgd))
 222                 return true;
 223
 224         p4d = p4d_offset(pgd, addr);
 225         if (!p4d_present(*p4d))
 226                 return false;
 227         if (p4d_leaf(*p4d))
 228                 return true;
 229
 230         pud = pud_offset(p4d, addr);
 231         if (!pud_present(*pud))
 232                 return false;
 233         if (pud_leaf(*pud))
 234                 return true;
 235
 236         pmd = pmd_offset(pud, addr);
 237         if (!pmd_present(*pmd))
 238                 return false;
 239         if (pmd_leaf(*pmd))
 240                 return true;
 241
 242         pte = pte_offset_kernel(pmd, addr);
 243         return pte_present(*pte);
 244 }