mm/page_table_check.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright (c) 2021, Google LLC.
   5  * Pasha Tatashin <pasha.tatashin@soleen.com>
   6  */
   7 #include <linux/mm.h>
   8 #include <linux/page_table_check.h>
   9
  10 #undef pr_fmt
  11 #define pr_fmt(fmt)     "page_table_check: " fmt
  12
  13 struct page_table_check {
  14         atomic_t anon_map_count;
  15         atomic_t file_map_count;
  16 };
  17
  18 static bool __page_table_check_enabled __initdata =
  19                                 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
  20
  21 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
  22 EXPORT_SYMBOL(page_table_check_disabled);
  23
  24 static int __init early_page_table_check_param(char *buf)
  25 {
  26         return strtobool(buf, &__page_table_check_enabled);
  27 }
  28
  29 early_param("page_table_check", early_page_table_check_param);
  30
  31 static bool __init need_page_table_check(void)
  32 {
  33         return __page_table_check_enabled;
  34 }
  35
  36 static void __init init_page_table_check(void)
  37 {
  38         if (!__page_table_check_enabled)
  39                 return;
  40         static_branch_disable(&page_table_check_disabled);
  41 }
  42
  43 struct page_ext_operations page_table_check_ops = {
  44         .size = sizeof(struct page_table_check),
  45         .need = need_page_table_check,
  46         .init = init_page_table_check,
  47 };
  48
  49 static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
  50 {
  51         BUG_ON(!page_ext);
  52         return (void *)(page_ext) + page_table_check_ops.offset;
  53 }
  54
  55 /*
  56  * An entry is removed from the page table, decrement the counters for that page
  57  * verify that it is of correct type and counters do not become negative.
  58  */
  59 static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
  60                                    unsigned long pfn, unsigned long pgcnt)
  61 {
  62         struct page_ext *page_ext;
  63         struct page *page;
  64         unsigned long i;
  65         bool anon;
  66
  67         if (!pfn_valid(pfn))
  68                 return;
  69
  70         page = pfn_to_page(pfn);
  71         page_ext = page_ext_get(page);
  72
  73         BUG_ON(PageSlab(page));
  74         anon = PageAnon(page);
  75
  76         for (i = 0; i < pgcnt; i++) {
  77                 struct page_table_check *ptc = get_page_table_check(page_ext);
  78
  79                 if (anon) {
  80                         BUG_ON(atomic_read(&ptc->file_map_count));
  81                         BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
  82                 } else {
  83                         BUG_ON(atomic_read(&ptc->anon_map_count));
  84                         BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
  85                 }
  86                 page_ext = page_ext_next(page_ext);
  87         }
  88         page_ext_put(page_ext);
  89 }
  90
  91 /*
  92  * A new entry is added to the page table, increment the counters for that page
  93  * verify that it is of correct type and is not being mapped with a different
  94  * type to a different process.
  95  */
  96 static void page_table_check_set(struct mm_struct *mm, unsigned long addr,
  97                                  unsigned long pfn, unsigned long pgcnt,
  98                                  bool rw)
  99 {
 100         struct page_ext *page_ext;
 101         struct page *page;
 102         unsigned long i;
 103         bool anon;
 104
 105         if (!pfn_valid(pfn))
 106                 return;
 107
 108         page = pfn_to_page(pfn);
 109         page_ext = page_ext_get(page);
 110
 111         BUG_ON(PageSlab(page));
 112         anon = PageAnon(page);
 113
 114         for (i = 0; i < pgcnt; i++) {
 115                 struct page_table_check *ptc = get_page_table_check(page_ext);
 116
 117                 if (anon) {
 118                         BUG_ON(atomic_read(&ptc->file_map_count));
 119                         BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
 120                 } else {
 121                         BUG_ON(atomic_read(&ptc->anon_map_count));
 122                         BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
 123                 }
 124                 page_ext = page_ext_next(page_ext);
 125         }
 126         page_ext_put(page_ext);
 127 }
 128
 129 /*
 130  * page is on free list, or is being allocated, verify that counters are zeroes
 131  * crash if they are not.
 132  */
 133 void __page_table_check_zero(struct page *page, unsigned int order)
 134 {
 135         struct page_ext *page_ext;
 136         unsigned long i;
 137
 138         BUG_ON(PageSlab(page));
 139
 140         page_ext = page_ext_get(page);
 141         BUG_ON(!page_ext);
 142         for (i = 0; i < (1ul << order); i++) {
 143                 struct page_table_check *ptc = get_page_table_check(page_ext);
 144
 145                 BUG_ON(atomic_read(&ptc->anon_map_count));
 146                 BUG_ON(atomic_read(&ptc->file_map_count));
 147                 page_ext = page_ext_next(page_ext);
 148         }
 149         page_ext_put(page_ext);
 150 }
 151
 152 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
 153                                   pte_t pte)
 154 {
 155         if (&init_mm == mm)
 156                 return;
 157
 158         if (pte_user_accessible_page(pte)) {
 159                 page_table_check_clear(mm, addr, pte_pfn(pte),
 160                                        PAGE_SIZE >> PAGE_SHIFT);
 161         }
 162 }
 163 EXPORT_SYMBOL(__page_table_check_pte_clear);
 164
 165 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
 166                                   pmd_t pmd)
 167 {
 168         if (&init_mm == mm)
 169                 return;
 170
 171         if (pmd_user_accessible_page(pmd)) {
 172                 page_table_check_clear(mm, addr, pmd_pfn(pmd),
 173                                        PMD_SIZE >> PAGE_SHIFT);
 174         }
 175 }
 176 EXPORT_SYMBOL(__page_table_check_pmd_clear);
 177
 178 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
 179                                   pud_t pud)
 180 {
 181         if (&init_mm == mm)
 182                 return;
 183
 184         if (pud_user_accessible_page(pud)) {
 185                 page_table_check_clear(mm, addr, pud_pfn(pud),
 186                                        PUD_SIZE >> PAGE_SHIFT);
 187         }
 188 }
 189 EXPORT_SYMBOL(__page_table_check_pud_clear);
 190
 191 void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
 192                                 pte_t *ptep, pte_t pte)
 193 {
 194         if (&init_mm == mm)
 195                 return;
 196
 197         __page_table_check_pte_clear(mm, addr, *ptep);
 198         if (pte_user_accessible_page(pte)) {
 199                 page_table_check_set(mm, addr, pte_pfn(pte),
 200                                      PAGE_SIZE >> PAGE_SHIFT,
 201                                      pte_write(pte));
 202         }
 203 }
 204 EXPORT_SYMBOL(__page_table_check_pte_set);
 205
 206 void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
 207                                 pmd_t *pmdp, pmd_t pmd)
 208 {
 209         if (&init_mm == mm)
 210                 return;
 211
 212         __page_table_check_pmd_clear(mm, addr, *pmdp);
 213         if (pmd_user_accessible_page(pmd)) {
 214                 page_table_check_set(mm, addr, pmd_pfn(pmd),
 215                                      PMD_SIZE >> PAGE_SHIFT,
 216                                      pmd_write(pmd));
 217         }
 218 }
 219 EXPORT_SYMBOL(__page_table_check_pmd_set);
 220
 221 void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
 222                                 pud_t *pudp, pud_t pud)
 223 {
 224         if (&init_mm == mm)
 225                 return;
 226
 227         __page_table_check_pud_clear(mm, addr, *pudp);
 228         if (pud_user_accessible_page(pud)) {
 229                 page_table_check_set(mm, addr, pud_pfn(pud),
 230                                      PUD_SIZE >> PAGE_SHIFT,
 231                                      pud_write(pud));
 232         }
 233 }
 234 EXPORT_SYMBOL(__page_table_check_pud_set);
 235
 236 void __page_table_check_pte_clear_range(struct mm_struct *mm,
 237                                         unsigned long addr,
 238                                         pmd_t pmd)
 239 {
 240         if (&init_mm == mm)
 241                 return;
 242
 243         if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
 244                 pte_t *ptep = pte_offset_map(&pmd, addr);
 245                 unsigned long i;
 246
 247                 for (i = 0; i < PTRS_PER_PTE; i++) {
 248                         __page_table_check_pte_clear(mm, addr, *ptep);
 249                         addr += PAGE_SIZE;
 250                         ptep++;
 251                 }
 252                 pte_unmap(ptep - PTRS_PER_PTE);
 253         }
 254 }