mm/page_table_check.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright (c) 2021, Google LLC.
   5  * Pasha Tatashin <pasha.tatashin@soleen.com>
   6  */
   7 #include <linux/kstrtox.h>
   8 #include <linux/mm.h>
   9 #include <linux/page_table_check.h>
  10
  11 #undef pr_fmt
  12 #define pr_fmt(fmt)     "page_table_check: " fmt
  13
  14 struct page_table_check {
  15         atomic_t anon_map_count;
  16         atomic_t file_map_count;
  17 };
  18
  19 static bool __page_table_check_enabled __initdata =
  20                                 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
  21
  22 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
  23 EXPORT_SYMBOL(page_table_check_disabled);
  24
  25 static int __init early_page_table_check_param(char *buf)
  26 {
  27         return kstrtobool(buf, &__page_table_check_enabled);
  28 }
  29
  30 early_param("page_table_check", early_page_table_check_param);
  31
  32 static bool __init need_page_table_check(void)
  33 {
  34         return __page_table_check_enabled;
  35 }
  36
  37 static void __init init_page_table_check(void)
  38 {
  39         if (!__page_table_check_enabled)
  40                 return;
  41         static_branch_disable(&page_table_check_disabled);
  42 }
  43
  44 struct page_ext_operations page_table_check_ops = {
  45         .size = sizeof(struct page_table_check),
  46         .need = need_page_table_check,
  47         .init = init_page_table_check,
  48         .need_shared_flags = false,
  49 };
  50
  51 static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
  52 {
  53         BUG_ON(!page_ext);
  54         return (void *)(page_ext) + page_table_check_ops.offset;
  55 }
  56
  57 /*
  58  * An entry is removed from the page table, decrement the counters for that page
  59  * verify that it is of correct type and counters do not become negative.
  60  */
  61 static void page_table_check_clear(struct mm_struct *mm, unsigned long addr,
  62                                    unsigned long pfn, unsigned long pgcnt)
  63 {
  64         struct page_ext *page_ext;
  65         struct page *page;
  66         unsigned long i;
  67         bool anon;
  68
  69         if (!pfn_valid(pfn))
  70                 return;
  71
  72         page = pfn_to_page(pfn);
  73         page_ext = page_ext_get(page);
  74
  75         BUG_ON(PageSlab(page));
  76         anon = PageAnon(page);
  77
  78         for (i = 0; i < pgcnt; i++) {
  79                 struct page_table_check *ptc = get_page_table_check(page_ext);
  80
  81                 if (anon) {
  82                         BUG_ON(atomic_read(&ptc->file_map_count));
  83                         BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
  84                 } else {
  85                         BUG_ON(atomic_read(&ptc->anon_map_count));
  86                         BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
  87                 }
  88                 page_ext = page_ext_next(page_ext);
  89         }
  90         page_ext_put(page_ext);
  91 }
  92
  93 /*
  94  * A new entry is added to the page table, increment the counters for that page
  95  * verify that it is of correct type and is not being mapped with a different
  96  * type to a different process.
  97  */
  98 static void page_table_check_set(struct mm_struct *mm, unsigned long addr,
  99                                  unsigned long pfn, unsigned long pgcnt,
 100                                  bool rw)
 101 {
 102         struct page_ext *page_ext;
 103         struct page *page;
 104         unsigned long i;
 105         bool anon;
 106
 107         if (!pfn_valid(pfn))
 108                 return;
 109
 110         page = pfn_to_page(pfn);
 111         page_ext = page_ext_get(page);
 112
 113         BUG_ON(PageSlab(page));
 114         anon = PageAnon(page);
 115
 116         for (i = 0; i < pgcnt; i++) {
 117                 struct page_table_check *ptc = get_page_table_check(page_ext);
 118
 119                 if (anon) {
 120                         BUG_ON(atomic_read(&ptc->file_map_count));
 121                         BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
 122                 } else {
 123                         BUG_ON(atomic_read(&ptc->anon_map_count));
 124                         BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
 125                 }
 126                 page_ext = page_ext_next(page_ext);
 127         }
 128         page_ext_put(page_ext);
 129 }
 130
 131 /*
 132  * page is on free list, or is being allocated, verify that counters are zeroes
 133  * crash if they are not.
 134  */
 135 void __page_table_check_zero(struct page *page, unsigned int order)
 136 {
 137         struct page_ext *page_ext;
 138         unsigned long i;
 139
 140         BUG_ON(PageSlab(page));
 141
 142         page_ext = page_ext_get(page);
 143         BUG_ON(!page_ext);
 144         for (i = 0; i < (1ul << order); i++) {
 145                 struct page_table_check *ptc = get_page_table_check(page_ext);
 146
 147                 BUG_ON(atomic_read(&ptc->anon_map_count));
 148                 BUG_ON(atomic_read(&ptc->file_map_count));
 149                 page_ext = page_ext_next(page_ext);
 150         }
 151         page_ext_put(page_ext);
 152 }
 153
 154 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
 155                                   pte_t pte)
 156 {
 157         if (&init_mm == mm)
 158                 return;
 159
 160         if (pte_user_accessible_page(pte)) {
 161                 page_table_check_clear(mm, addr, pte_pfn(pte),
 162                                        PAGE_SIZE >> PAGE_SHIFT);
 163         }
 164 }
 165 EXPORT_SYMBOL(__page_table_check_pte_clear);
 166
 167 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
 168                                   pmd_t pmd)
 169 {
 170         if (&init_mm == mm)
 171                 return;
 172
 173         if (pmd_user_accessible_page(pmd)) {
 174                 page_table_check_clear(mm, addr, pmd_pfn(pmd),
 175                                        PMD_SIZE >> PAGE_SHIFT);
 176         }
 177 }
 178 EXPORT_SYMBOL(__page_table_check_pmd_clear);
 179
 180 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
 181                                   pud_t pud)
 182 {
 183         if (&init_mm == mm)
 184                 return;
 185
 186         if (pud_user_accessible_page(pud)) {
 187                 page_table_check_clear(mm, addr, pud_pfn(pud),
 188                                        PUD_SIZE >> PAGE_SHIFT);
 189         }
 190 }
 191 EXPORT_SYMBOL(__page_table_check_pud_clear);
 192
 193 void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
 194                                 pte_t *ptep, pte_t pte)
 195 {
 196         if (&init_mm == mm)
 197                 return;
 198
 199         __page_table_check_pte_clear(mm, addr, ptep_get(ptep));
 200         if (pte_user_accessible_page(pte)) {
 201                 page_table_check_set(mm, addr, pte_pfn(pte),
 202                                      PAGE_SIZE >> PAGE_SHIFT,
 203                                      pte_write(pte));
 204         }
 205 }
 206 EXPORT_SYMBOL(__page_table_check_pte_set);
 207
 208 void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
 209                                 pmd_t *pmdp, pmd_t pmd)
 210 {
 211         if (&init_mm == mm)
 212                 return;
 213
 214         __page_table_check_pmd_clear(mm, addr, *pmdp);
 215         if (pmd_user_accessible_page(pmd)) {
 216                 page_table_check_set(mm, addr, pmd_pfn(pmd),
 217                                      PMD_SIZE >> PAGE_SHIFT,
 218                                      pmd_write(pmd));
 219         }
 220 }
 221 EXPORT_SYMBOL(__page_table_check_pmd_set);
 222
 223 void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
 224                                 pud_t *pudp, pud_t pud)
 225 {
 226         if (&init_mm == mm)
 227                 return;
 228
 229         __page_table_check_pud_clear(mm, addr, *pudp);
 230         if (pud_user_accessible_page(pud)) {
 231                 page_table_check_set(mm, addr, pud_pfn(pud),
 232                                      PUD_SIZE >> PAGE_SHIFT,
 233                                      pud_write(pud));
 234         }
 235 }
 236 EXPORT_SYMBOL(__page_table_check_pud_set);
 237
 238 void __page_table_check_pte_clear_range(struct mm_struct *mm,
 239                                         unsigned long addr,
 240                                         pmd_t pmd)
 241 {
 242         if (&init_mm == mm)
 243                 return;
 244
 245         if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
 246                 pte_t *ptep = pte_offset_map(&pmd, addr);
 247                 unsigned long i;
 248
 249                 if (WARN_ON(!ptep))
 250                         return;
 251                 for (i = 0; i < PTRS_PER_PTE; i++) {
 252                         __page_table_check_pte_clear(mm, addr, ptep_get(ptep));
 253                         addr += PAGE_SIZE;
 254                         ptep++;
 255                 }
 256                 pte_unmap(ptep - PTRS_PER_PTE);
 257         }
 258 }