1 /* SPDX-License-Identifier: GPL-2.0 */
5 #include <linux/static_key.h>
6 #include <linux/types.h>
15 #include <linux/linkage.h>
16 #include <asm/kasan.h>
18 /* kasan_data struct is used in KUnit tests for KASAN expected failures */
19 struct kunit_kasan_expectation {
26 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
28 #include <linux/pgtable.h>
30 /* Software KASAN implementations use shadow memory. */
32 #ifdef CONFIG_KASAN_SW_TAGS
33 /* This matches KASAN_TAG_INVALID. */
34 #define KASAN_SHADOW_INIT 0xFE
36 #define KASAN_SHADOW_INIT 0
39 #ifndef PTE_HWTABLE_PTRS
40 #define PTE_HWTABLE_PTRS 0
43 extern unsigned char kasan_early_shadow_page[PAGE_SIZE];
44 extern pte_t kasan_early_shadow_pte[PTRS_PER_PTE + PTE_HWTABLE_PTRS];
45 extern pmd_t kasan_early_shadow_pmd[PTRS_PER_PMD];
46 extern pud_t kasan_early_shadow_pud[PTRS_PER_PUD];
47 extern p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D];
49 int kasan_populate_early_shadow(const void *shadow_start,
50 const void *shadow_end);
52 static inline void *kasan_mem_to_shadow(const void *addr)
54 return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
55 + KASAN_SHADOW_OFFSET;
58 int kasan_add_zero_shadow(void *start, unsigned long size);
59 void kasan_remove_zero_shadow(void *start, unsigned long size);
61 /* Enable reporting bugs after kasan_disable_current() */
62 extern void kasan_enable_current(void);
64 /* Disable reporting bugs for current task */
65 extern void kasan_disable_current(void);
67 #else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
69 static inline int kasan_add_zero_shadow(void *start, unsigned long size)
73 static inline void kasan_remove_zero_shadow(void *start,
77 static inline void kasan_enable_current(void) {}
78 static inline void kasan_disable_current(void) {}
80 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
85 int alloc_meta_offset;
90 #ifdef CONFIG_KASAN_HW_TAGS
92 DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);
94 static __always_inline bool kasan_enabled(void)
96 return static_branch_likely(&kasan_flag_enabled);
99 static inline bool kasan_has_integrated_init(void)
101 return kasan_enabled();
104 #else /* CONFIG_KASAN_HW_TAGS */
106 static inline bool kasan_enabled(void)
111 static inline bool kasan_has_integrated_init(void)
116 #endif /* CONFIG_KASAN_HW_TAGS */
118 slab_flags_t __kasan_never_merge(void);
119 static __always_inline slab_flags_t kasan_never_merge(void)
122 return __kasan_never_merge();
126 void __kasan_unpoison_range(const void *addr, size_t size);
127 static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
130 __kasan_unpoison_range(addr, size);
133 void __kasan_alloc_pages(struct page *page, unsigned int order, bool init);
134 static __always_inline void kasan_alloc_pages(struct page *page,
135 unsigned int order, bool init)
138 __kasan_alloc_pages(page, order, init);
141 void __kasan_free_pages(struct page *page, unsigned int order, bool init);
142 static __always_inline void kasan_free_pages(struct page *page,
143 unsigned int order, bool init)
146 __kasan_free_pages(page, order, init);
149 void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
150 slab_flags_t *flags);
151 static __always_inline void kasan_cache_create(struct kmem_cache *cache,
152 unsigned int *size, slab_flags_t *flags)
155 __kasan_cache_create(cache, size, flags);
158 void __kasan_cache_create_kmalloc(struct kmem_cache *cache);
159 static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache)
162 __kasan_cache_create_kmalloc(cache);
165 size_t __kasan_metadata_size(struct kmem_cache *cache);
166 static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache)
169 return __kasan_metadata_size(cache);
173 void __kasan_poison_slab(struct page *page);
174 static __always_inline void kasan_poison_slab(struct page *page)
177 __kasan_poison_slab(page);
180 void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
181 static __always_inline void kasan_unpoison_object_data(struct kmem_cache *cache,
185 __kasan_unpoison_object_data(cache, object);
188 void __kasan_poison_object_data(struct kmem_cache *cache, void *object);
189 static __always_inline void kasan_poison_object_data(struct kmem_cache *cache,
193 __kasan_poison_object_data(cache, object);
196 void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
198 static __always_inline void * __must_check kasan_init_slab_obj(
199 struct kmem_cache *cache, const void *object)
202 return __kasan_init_slab_obj(cache, object);
203 return (void *)object;
206 bool __kasan_slab_free(struct kmem_cache *s, void *object, unsigned long ip);
207 static __always_inline bool kasan_slab_free(struct kmem_cache *s, void *object)
210 return __kasan_slab_free(s, object, _RET_IP_);
214 void __kasan_kfree_large(void *ptr, unsigned long ip);
215 static __always_inline void kasan_kfree_large(void *ptr)
218 __kasan_kfree_large(ptr, _RET_IP_);
221 void __kasan_slab_free_mempool(void *ptr, unsigned long ip);
222 static __always_inline void kasan_slab_free_mempool(void *ptr)
225 __kasan_slab_free_mempool(ptr, _RET_IP_);
228 void * __must_check __kasan_slab_alloc(struct kmem_cache *s,
229 void *object, gfp_t flags, bool init);
230 static __always_inline void * __must_check kasan_slab_alloc(
231 struct kmem_cache *s, void *object, gfp_t flags, bool init)
234 return __kasan_slab_alloc(s, object, flags, init);
238 void * __must_check __kasan_kmalloc(struct kmem_cache *s, const void *object,
239 size_t size, gfp_t flags);
240 static __always_inline void * __must_check kasan_kmalloc(struct kmem_cache *s,
241 const void *object, size_t size, gfp_t flags)
244 return __kasan_kmalloc(s, object, size, flags);
245 return (void *)object;
248 void * __must_check __kasan_kmalloc_large(const void *ptr,
249 size_t size, gfp_t flags);
250 static __always_inline void * __must_check kasan_kmalloc_large(const void *ptr,
251 size_t size, gfp_t flags)
254 return __kasan_kmalloc_large(ptr, size, flags);
258 void * __must_check __kasan_krealloc(const void *object,
259 size_t new_size, gfp_t flags);
260 static __always_inline void * __must_check kasan_krealloc(const void *object,
261 size_t new_size, gfp_t flags)
264 return __kasan_krealloc(object, new_size, flags);
265 return (void *)object;
269 * Unlike kasan_check_read/write(), kasan_check_byte() is performed even for
270 * the hardware tag-based mode that doesn't rely on compiler instrumentation.
272 bool __kasan_check_byte(const void *addr, unsigned long ip);
273 static __always_inline bool kasan_check_byte(const void *addr)
276 return __kasan_check_byte(addr, _RET_IP_);
281 bool kasan_save_enable_multi_shot(void);
282 void kasan_restore_multi_shot(bool enabled);
284 #else /* CONFIG_KASAN */
286 static inline bool kasan_enabled(void)
290 static inline bool kasan_has_integrated_init(void)
294 static inline slab_flags_t kasan_never_merge(void)
298 static inline void kasan_unpoison_range(const void *address, size_t size) {}
299 static inline void kasan_alloc_pages(struct page *page, unsigned int order, bool init) {}
300 static inline void kasan_free_pages(struct page *page, unsigned int order, bool init) {}
301 static inline void kasan_cache_create(struct kmem_cache *cache,
303 slab_flags_t *flags) {}
304 static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {}
305 static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; }
306 static inline void kasan_poison_slab(struct page *page) {}
307 static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
309 static inline void kasan_poison_object_data(struct kmem_cache *cache,
311 static inline void *kasan_init_slab_obj(struct kmem_cache *cache,
314 return (void *)object;
316 static inline bool kasan_slab_free(struct kmem_cache *s, void *object)
320 static inline void kasan_kfree_large(void *ptr) {}
321 static inline void kasan_slab_free_mempool(void *ptr) {}
322 static inline void *kasan_slab_alloc(struct kmem_cache *s, void *object,
323 gfp_t flags, bool init)
327 static inline void *kasan_kmalloc(struct kmem_cache *s, const void *object,
328 size_t size, gfp_t flags)
330 return (void *)object;
332 static inline void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
336 static inline void *kasan_krealloc(const void *object, size_t new_size,
339 return (void *)object;
341 static inline bool kasan_check_byte(const void *address)
346 #endif /* CONFIG_KASAN */
348 #if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK)
349 void kasan_unpoison_task_stack(struct task_struct *task);
351 static inline void kasan_unpoison_task_stack(struct task_struct *task) {}
354 #ifdef CONFIG_KASAN_GENERIC
356 void kasan_cache_shrink(struct kmem_cache *cache);
357 void kasan_cache_shutdown(struct kmem_cache *cache);
358 void kasan_record_aux_stack(void *ptr);
360 #else /* CONFIG_KASAN_GENERIC */
362 static inline void kasan_cache_shrink(struct kmem_cache *cache) {}
363 static inline void kasan_cache_shutdown(struct kmem_cache *cache) {}
364 static inline void kasan_record_aux_stack(void *ptr) {}
366 #endif /* CONFIG_KASAN_GENERIC */
368 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
370 static inline void *kasan_reset_tag(const void *addr)
372 return (void *)arch_kasan_reset_tag(addr);
376 * kasan_report - print a report about a bad memory access detected by KASAN
377 * @addr: address of the bad access
378 * @size: size of the bad access
379 * @is_write: whether the bad access is a write or a read
380 * @ip: instruction pointer for the accessibility check or the bad access itself
382 bool kasan_report(unsigned long addr, size_t size,
383 bool is_write, unsigned long ip);
385 #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
387 static inline void *kasan_reset_tag(const void *addr)
392 #endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS*/
394 #ifdef CONFIG_KASAN_HW_TAGS
396 void kasan_report_async(void);
398 #endif /* CONFIG_KASAN_HW_TAGS */
400 #ifdef CONFIG_KASAN_SW_TAGS
401 void __init kasan_init_sw_tags(void);
403 static inline void kasan_init_sw_tags(void) { }
406 #ifdef CONFIG_KASAN_HW_TAGS
407 void kasan_init_hw_tags_cpu(void);
408 void __init kasan_init_hw_tags(void);
410 static inline void kasan_init_hw_tags_cpu(void) { }
411 static inline void kasan_init_hw_tags(void) { }
414 #ifdef CONFIG_KASAN_VMALLOC
416 int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
417 void kasan_poison_vmalloc(const void *start, unsigned long size);
418 void kasan_unpoison_vmalloc(const void *start, unsigned long size);
419 void kasan_release_vmalloc(unsigned long start, unsigned long end,
420 unsigned long free_region_start,
421 unsigned long free_region_end);
423 #else /* CONFIG_KASAN_VMALLOC */
425 static inline int kasan_populate_vmalloc(unsigned long start,
431 static inline void kasan_poison_vmalloc(const void *start, unsigned long size)
433 static inline void kasan_unpoison_vmalloc(const void *start, unsigned long size)
435 static inline void kasan_release_vmalloc(unsigned long start,
437 unsigned long free_region_start,
438 unsigned long free_region_end) {}
440 #endif /* CONFIG_KASAN_VMALLOC */
442 #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
443 !defined(CONFIG_KASAN_VMALLOC)
446 * These functions provide a special case to support backing module
447 * allocations with real shadow memory. With KASAN vmalloc, the special
448 * case is unnecessary, as the work is handled in the generic case.
450 int kasan_module_alloc(void *addr, size_t size);
451 void kasan_free_shadow(const struct vm_struct *vm);
453 #else /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
455 static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
456 static inline void kasan_free_shadow(const struct vm_struct *vm) {}
458 #endif /* (CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS) && !CONFIG_KASAN_VMALLOC */
460 #ifdef CONFIG_KASAN_INLINE
461 void kasan_non_canonical_hook(unsigned long addr);
462 #else /* CONFIG_KASAN_INLINE */
463 static inline void kasan_non_canonical_hook(unsigned long addr) { }
464 #endif /* CONFIG_KASAN_INLINE */
466 #endif /* LINUX_KASAN_H */