mm/kasan/shadow.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains KASAN runtime code that manages shadow memory for
   4  * generic and software tag-based KASAN modes.
   5  *
   6  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
   7  * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
   8  *
   9  * Some code borrowed from https://github.com/xairy/kasan-prototype by
  10  *        Andrey Konovalov <andreyknvl@gmail.com>
  11  */
  12
  13 #include <linux/init.h>
  14 #include <linux/kasan.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kmemleak.h>
  17 #include <linux/memory.h>
  18 #include <linux/mm.h>
  19 #include <linux/string.h>
  20 #include <linux/types.h>
  21 #include <linux/vmalloc.h>
  22
  23 #include <asm/cacheflush.h>
  24 #include <asm/tlbflush.h>
  25
  26 #include "kasan.h"
  27
  28 bool __kasan_check_read(const volatile void *p, unsigned int size)
  29 {
  30         return check_memory_region((unsigned long)p, size, false, _RET_IP_);
  31 }
  32 EXPORT_SYMBOL(__kasan_check_read);
  33
  34 bool __kasan_check_write(const volatile void *p, unsigned int size)
  35 {
  36         return check_memory_region((unsigned long)p, size, true, _RET_IP_);
  37 }
  38 EXPORT_SYMBOL(__kasan_check_write);
  39
  40 #undef memset
  41 void *memset(void *addr, int c, size_t len)
  42 {
  43         if (!check_memory_region((unsigned long)addr, len, true, _RET_IP_))
  44                 return NULL;
  45
  46         return __memset(addr, c, len);
  47 }
  48
  49 #ifdef __HAVE_ARCH_MEMMOVE
  50 #undef memmove
  51 void *memmove(void *dest, const void *src, size_t len)
  52 {
  53         if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) ||
  54             !check_memory_region((unsigned long)dest, len, true, _RET_IP_))
  55                 return NULL;
  56
  57         return __memmove(dest, src, len);
  58 }
  59 #endif
  60
  61 #undef memcpy
  62 void *memcpy(void *dest, const void *src, size_t len)
  63 {
  64         if (!check_memory_region((unsigned long)src, len, false, _RET_IP_) ||
  65             !check_memory_region((unsigned long)dest, len, true, _RET_IP_))
  66                 return NULL;
  67
  68         return __memcpy(dest, src, len);
  69 }
  70
  71 /*
  72  * Poisons the shadow memory for 'size' bytes starting from 'addr'.
  73  * Memory addresses should be aligned to KASAN_GRANULE_SIZE.
  74  */
  75 void poison_range(const void *address, size_t size, u8 value)
  76 {
  77         void *shadow_start, *shadow_end;
  78
  79         /*
  80          * Perform shadow offset calculation based on untagged address, as
  81          * some of the callers (e.g. kasan_poison_object_data) pass tagged
  82          * addresses to this function.
  83          */
  84         address = reset_tag(address);
  85
  86         shadow_start = kasan_mem_to_shadow(address);
  87         shadow_end = kasan_mem_to_shadow(address + size);
  88
  89         __memset(shadow_start, value, shadow_end - shadow_start);
  90 }
  91
  92 void unpoison_range(const void *address, size_t size)
  93 {
  94         u8 tag = get_tag(address);
  95
  96         /*
  97          * Perform shadow offset calculation based on untagged address, as
  98          * some of the callers (e.g. kasan_unpoison_object_data) pass tagged
  99          * addresses to this function.
 100          */
 101         address = reset_tag(address);
 102
 103         poison_range(address, size, tag);
 104
 105         if (size & KASAN_GRANULE_MASK) {
 106                 u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size);
 107
 108                 if (IS_ENABLED(CONFIG_KASAN_SW_TAGS))
 109                         *shadow = tag;
 110                 else
 111                         *shadow = size & KASAN_GRANULE_MASK;
 112         }
 113 }
 114
 115 #ifdef CONFIG_MEMORY_HOTPLUG
 116 static bool shadow_mapped(unsigned long addr)
 117 {
 118         pgd_t *pgd = pgd_offset_k(addr);
 119         p4d_t *p4d;
 120         pud_t *pud;
 121         pmd_t *pmd;
 122         pte_t *pte;
 123
 124         if (pgd_none(*pgd))
 125                 return false;
 126         p4d = p4d_offset(pgd, addr);
 127         if (p4d_none(*p4d))
 128                 return false;
 129         pud = pud_offset(p4d, addr);
 130         if (pud_none(*pud))
 131                 return false;
 132
 133         /*
 134          * We can't use pud_large() or pud_huge(), the first one is
 135          * arch-specific, the last one depends on HUGETLB_PAGE.  So let's abuse
 136          * pud_bad(), if pud is bad then it's bad because it's huge.
 137          */
 138         if (pud_bad(*pud))
 139                 return true;
 140         pmd = pmd_offset(pud, addr);
 141         if (pmd_none(*pmd))
 142                 return false;
 143
 144         if (pmd_bad(*pmd))
 145                 return true;
 146         pte = pte_offset_kernel(pmd, addr);
 147         return !pte_none(*pte);
 148 }
 149
 150 static int __meminit kasan_mem_notifier(struct notifier_block *nb,
 151                         unsigned long action, void *data)
 152 {
 153         struct memory_notify *mem_data = data;
 154         unsigned long nr_shadow_pages, start_kaddr, shadow_start;
 155         unsigned long shadow_end, shadow_size;
 156
 157         nr_shadow_pages = mem_data->nr_pages >> KASAN_SHADOW_SCALE_SHIFT;
 158         start_kaddr = (unsigned long)pfn_to_kaddr(mem_data->start_pfn);
 159         shadow_start = (unsigned long)kasan_mem_to_shadow((void *)start_kaddr);
 160         shadow_size = nr_shadow_pages << PAGE_SHIFT;
 161         shadow_end = shadow_start + shadow_size;
 162
 163         if (WARN_ON(mem_data->nr_pages % KASAN_GRANULE_SIZE) ||
 164                 WARN_ON(start_kaddr % KASAN_MEMORY_PER_SHADOW_PAGE))
 165                 return NOTIFY_BAD;
 166
 167         switch (action) {
 168         case MEM_GOING_ONLINE: {
 169                 void *ret;
 170
 171                 /*
 172                  * If shadow is mapped already than it must have been mapped
 173                  * during the boot. This could happen if we onlining previously
 174                  * offlined memory.
 175                  */
 176                 if (shadow_mapped(shadow_start))
 177                         return NOTIFY_OK;
 178
 179                 ret = __vmalloc_node_range(shadow_size, PAGE_SIZE, shadow_start,
 180                                         shadow_end, GFP_KERNEL,
 181                                         PAGE_KERNEL, VM_NO_GUARD,
 182                                         pfn_to_nid(mem_data->start_pfn),
 183                                         __builtin_return_address(0));
 184                 if (!ret)
 185                         return NOTIFY_BAD;
 186
 187                 kmemleak_ignore(ret);
 188                 return NOTIFY_OK;
 189         }
 190         case MEM_CANCEL_ONLINE:
 191         case MEM_OFFLINE: {
 192                 struct vm_struct *vm;
 193
 194                 /*
 195                  * shadow_start was either mapped during boot by kasan_init()
 196                  * or during memory online by __vmalloc_node_range().
 197                  * In the latter case we can use vfree() to free shadow.
 198                  * Non-NULL result of the find_vm_area() will tell us if
 199                  * that was the second case.
 200                  *
 201                  * Currently it's not possible to free shadow mapped
 202                  * during boot by kasan_init(). It's because the code
 203                  * to do that hasn't been written yet. So we'll just
 204                  * leak the memory.
 205                  */
 206                 vm = find_vm_area((void *)shadow_start);
 207                 if (vm)
 208                         vfree((void *)shadow_start);
 209         }
 210         }
 211
 212         return NOTIFY_OK;
 213 }
 214
 215 static int __init kasan_memhotplug_init(void)
 216 {
 217         hotplug_memory_notifier(kasan_mem_notifier, 0);
 218
 219         return 0;
 220 }
 221
 222 core_initcall(kasan_memhotplug_init);
 223 #endif
 224
 225 #ifdef CONFIG_KASAN_VMALLOC
 226
 227 static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
 228                                       void *unused)
 229 {
 230         unsigned long page;
 231         pte_t pte;
 232
 233         if (likely(!pte_none(*ptep)))
 234                 return 0;
 235
 236         page = __get_free_page(GFP_KERNEL);
 237         if (!page)
 238                 return -ENOMEM;
 239
 240         memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE);
 241         pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL);
 242
 243         spin_lock(&init_mm.page_table_lock);
 244         if (likely(pte_none(*ptep))) {
 245                 set_pte_at(&init_mm, addr, ptep, pte);
 246                 page = 0;
 247         }
 248         spin_unlock(&init_mm.page_table_lock);
 249         if (page)
 250                 free_page(page);
 251         return 0;
 252 }
 253
 254 int kasan_populate_vmalloc(unsigned long addr, unsigned long size)
 255 {
 256         unsigned long shadow_start, shadow_end;
 257         int ret;
 258
 259         if (!is_vmalloc_or_module_addr((void *)addr))
 260                 return 0;
 261
 262         shadow_start = (unsigned long)kasan_mem_to_shadow((void *)addr);
 263         shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE);
 264         shadow_end = (unsigned long)kasan_mem_to_shadow((void *)addr + size);
 265         shadow_end = ALIGN(shadow_end, PAGE_SIZE);
 266
 267         ret = apply_to_page_range(&init_mm, shadow_start,
 268                                   shadow_end - shadow_start,
 269                                   kasan_populate_vmalloc_pte, NULL);
 270         if (ret)
 271                 return ret;
 272
 273         flush_cache_vmap(shadow_start, shadow_end);
 274
 275         /*
 276          * We need to be careful about inter-cpu effects here. Consider:
 277          *
 278          *   CPU#0                                CPU#1
 279          * WRITE_ONCE(p, vmalloc(100));         while (x = READ_ONCE(p)) ;
 280          *                                      p[99] = 1;
 281          *
 282          * With compiler instrumentation, that ends up looking like this:
 283          *
 284          *   CPU#0                                CPU#1
 285          * // vmalloc() allocates memory
 286          * // let a = area->addr
 287          * // we reach kasan_populate_vmalloc
 288          * // and call unpoison_range:
 289          * STORE shadow(a), unpoison_val
 290          * ...
 291          * STORE shadow(a+99), unpoison_val     x = LOAD p
 292          * // rest of vmalloc process           <data dependency>
 293          * STORE p, a                           LOAD shadow(x+99)
 294          *
 295          * If there is no barrier between the end of unpoisioning the shadow
 296          * and the store of the result to p, the stores could be committed
 297          * in a different order by CPU#0, and CPU#1 could erroneously observe
 298          * poison in the shadow.
 299          *
 300          * We need some sort of barrier between the stores.
 301          *
 302          * In the vmalloc() case, this is provided by a smp_wmb() in
 303          * clear_vm_uninitialized_flag(). In the per-cpu allocator and in
 304          * get_vm_area() and friends, the caller gets shadow allocated but
 305          * doesn't have any pages mapped into the virtual address space that
 306          * has been reserved. Mapping those pages in will involve taking and
 307          * releasing a page-table lock, which will provide the barrier.
 308          */
 309
 310         return 0;
 311 }
 312
 313 /*
 314  * Poison the shadow for a vmalloc region. Called as part of the
 315  * freeing process at the time the region is freed.
 316  */
 317 void kasan_poison_vmalloc(const void *start, unsigned long size)
 318 {
 319         if (!is_vmalloc_or_module_addr(start))
 320                 return;
 321
 322         size = round_up(size, KASAN_GRANULE_SIZE);
 323         poison_range(start, size, KASAN_VMALLOC_INVALID);
 324 }
 325
 326 void kasan_unpoison_vmalloc(const void *start, unsigned long size)
 327 {
 328         if (!is_vmalloc_or_module_addr(start))
 329                 return;
 330
 331         unpoison_range(start, size);
 332 }
 333
 334 static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
 335                                         void *unused)
 336 {
 337         unsigned long page;
 338
 339         page = (unsigned long)__va(pte_pfn(*ptep) << PAGE_SHIFT);
 340
 341         spin_lock(&init_mm.page_table_lock);
 342
 343         if (likely(!pte_none(*ptep))) {
 344                 pte_clear(&init_mm, addr, ptep);
 345                 free_page(page);
 346         }
 347         spin_unlock(&init_mm.page_table_lock);
 348
 349         return 0;
 350 }
 351
 352 /*
 353  * Release the backing for the vmalloc region [start, end), which
 354  * lies within the free region [free_region_start, free_region_end).
 355  *
 356  * This can be run lazily, long after the region was freed. It runs
 357  * under vmap_area_lock, so it's not safe to interact with the vmalloc/vmap
 358  * infrastructure.
 359  *
 360  * How does this work?
 361  * -------------------
 362  *
 363  * We have a region that is page aligned, labelled as A.
 364  * That might not map onto the shadow in a way that is page-aligned:
 365  *
 366  *                    start                     end
 367  *                    v                         v
 368  * |????????|????????|AAAAAAAA|AA....AA|AAAAAAAA|????????| < vmalloc
 369  *  -------- -------- --------          -------- --------
 370  *      |        |       |                 |        |
 371  *      |        |       |         /-------/        |
 372  *      \-------\|/------/         |/---------------/
 373  *              |||                ||
 374  *             |??AAAAAA|AAAAAAAA|AA??????|                < shadow
 375  *                 (1)      (2)      (3)
 376  *
 377  * First we align the start upwards and the end downwards, so that the
 378  * shadow of the region aligns with shadow page boundaries. In the
 379  * example, this gives us the shadow page (2). This is the shadow entirely
 380  * covered by this allocation.
 381  *
 382  * Then we have the tricky bits. We want to know if we can free the
 383  * partially covered shadow pages - (1) and (3) in the example. For this,
 384  * we are given the start and end of the free region that contains this
 385  * allocation. Extending our previous example, we could have:
 386  *
 387  *  free_region_start                                    free_region_end
 388  *  |                 start                     end      |
 389  *  v                 v                         v        v
 390  * |FFFFFFFF|FFFFFFFF|AAAAAAAA|AA....AA|AAAAAAAA|FFFFFFFF| < vmalloc
 391  *  -------- -------- --------          -------- --------
 392  *      |        |       |                 |        |
 393  *      |        |       |         /-------/        |
 394  *      \-------\|/------/         |/---------------/
 395  *              |||                ||
 396  *             |FFAAAAAA|AAAAAAAA|AAF?????|                < shadow
 397  *                 (1)      (2)      (3)
 398  *
 399  * Once again, we align the start of the free region up, and the end of
 400  * the free region down so that the shadow is page aligned. So we can free
 401  * page (1) - we know no allocation currently uses anything in that page,
 402  * because all of it is in the vmalloc free region. But we cannot free
 403  * page (3), because we can't be sure that the rest of it is unused.
 404  *
 405  * We only consider pages that contain part of the original region for
 406  * freeing: we don't try to free other pages from the free region or we'd
 407  * end up trying to free huge chunks of virtual address space.
 408  *
 409  * Concurrency
 410  * -----------
 411  *
 412  * How do we know that we're not freeing a page that is simultaneously
 413  * being used for a fresh allocation in kasan_populate_vmalloc(_pte)?
 414  *
 415  * We _can_ have kasan_release_vmalloc and kasan_populate_vmalloc running
 416  * at the same time. While we run under free_vmap_area_lock, the population
 417  * code does not.
 418  *
 419  * free_vmap_area_lock instead operates to ensure that the larger range
 420  * [free_region_start, free_region_end) is safe: because __alloc_vmap_area and
 421  * the per-cpu region-finding algorithm both run under free_vmap_area_lock,
 422  * no space identified as free will become used while we are running. This
 423  * means that so long as we are careful with alignment and only free shadow
 424  * pages entirely covered by the free region, we will not run in to any
 425  * trouble - any simultaneous allocations will be for disjoint regions.
 426  */
 427 void kasan_release_vmalloc(unsigned long start, unsigned long end,
 428                            unsigned long free_region_start,
 429                            unsigned long free_region_end)
 430 {
 431         void *shadow_start, *shadow_end;
 432         unsigned long region_start, region_end;
 433         unsigned long size;
 434
 435         region_start = ALIGN(start, KASAN_MEMORY_PER_SHADOW_PAGE);
 436         region_end = ALIGN_DOWN(end, KASAN_MEMORY_PER_SHADOW_PAGE);
 437
 438         free_region_start = ALIGN(free_region_start, KASAN_MEMORY_PER_SHADOW_PAGE);
 439
 440         if (start != region_start &&
 441             free_region_start < region_start)
 442                 region_start -= KASAN_MEMORY_PER_SHADOW_PAGE;
 443
 444         free_region_end = ALIGN_DOWN(free_region_end, KASAN_MEMORY_PER_SHADOW_PAGE);
 445
 446         if (end != region_end &&
 447             free_region_end > region_end)
 448                 region_end += KASAN_MEMORY_PER_SHADOW_PAGE;
 449
 450         shadow_start = kasan_mem_to_shadow((void *)region_start);
 451         shadow_end = kasan_mem_to_shadow((void *)region_end);
 452
 453         if (shadow_end > shadow_start) {
 454                 size = shadow_end - shadow_start;
 455                 apply_to_existing_page_range(&init_mm,
 456                                              (unsigned long)shadow_start,
 457                                              size, kasan_depopulate_vmalloc_pte,
 458                                              NULL);
 459                 flush_tlb_kernel_range((unsigned long)shadow_start,
 460                                        (unsigned long)shadow_end);
 461         }
 462 }
 463
 464 #else /* CONFIG_KASAN_VMALLOC */
 465
 466 int kasan_module_alloc(void *addr, size_t size)
 467 {
 468         void *ret;
 469         size_t scaled_size;
 470         size_t shadow_size;
 471         unsigned long shadow_start;
 472
 473         shadow_start = (unsigned long)kasan_mem_to_shadow(addr);
 474         scaled_size = (size + KASAN_GRANULE_SIZE - 1) >>
 475                                 KASAN_SHADOW_SCALE_SHIFT;
 476         shadow_size = round_up(scaled_size, PAGE_SIZE);
 477
 478         if (WARN_ON(!PAGE_ALIGNED(shadow_start)))
 479                 return -EINVAL;
 480
 481         ret = __vmalloc_node_range(shadow_size, 1, shadow_start,
 482                         shadow_start + shadow_size,
 483                         GFP_KERNEL,
 484                         PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE,
 485                         __builtin_return_address(0));
 486
 487         if (ret) {
 488                 __memset(ret, KASAN_SHADOW_INIT, shadow_size);
 489                 find_vm_area(addr)->flags |= VM_KASAN;
 490                 kmemleak_ignore(ret);
 491                 return 0;
 492         }
 493
 494         return -ENOMEM;
 495 }
 496
 497 void kasan_free_shadow(const struct vm_struct *vm)
 498 {
 499         if (vm->flags & VM_KASAN)
 500                 vfree(kasan_mem_to_shadow(vm->addr));
 501 }
 502
 503 #endif