Merge tag 'kvmarm-fixes-6.1-3' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmar...
[platform/kernel/linux-starfive.git] / mm / kmemleak.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * mm/kmemleak.c
4  *
5  * Copyright (C) 2008 ARM Limited
6  * Written by Catalin Marinas <catalin.marinas@arm.com>
7  *
8  * For more information on the algorithm and kmemleak usage, please see
9  * Documentation/dev-tools/kmemleak.rst.
10  *
11  * Notes on locking
12  * ----------------
13  *
14  * The following locks and mutexes are used by kmemleak:
15  *
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
17  *   accesses to the object_tree_root (or object_phys_tree_root). The
18  *   object_list is the main list holding the metadata (struct kmemleak_object)
19  *   for the allocated memory blocks. The object_tree_root and object_phys_tree_root
20  *   are red black trees used to look-up metadata based on a pointer to the
21  *   corresponding memory block. The object_phys_tree_root is for objects
22  *   allocated with physical address. The kmemleak_object structures are
23  *   added to the object_list and object_tree_root (or object_phys_tree_root)
24  *   in the create_object() function called from the kmemleak_alloc() (or
25  *   kmemleak_alloc_phys()) callback and removed in delete_object() called from
26  *   the kmemleak_free() callback
27  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
28  *   Accesses to the metadata (e.g. count) are protected by this lock. Note
29  *   that some members of this structure may be protected by other means
30  *   (atomic or kmemleak_lock). This lock is also held when scanning the
31  *   corresponding memory block to avoid the kernel freeing it via the
32  *   kmemleak_free() callback. This is less heavyweight than holding a global
33  *   lock like kmemleak_lock during scanning.
34  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
35  *   unreferenced objects at a time. The gray_list contains the objects which
36  *   are already referenced or marked as false positives and need to be
37  *   scanned. This list is only modified during a scanning episode when the
38  *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
39  *   Note that the kmemleak_object.use_count is incremented when an object is
40  *   added to the gray_list and therefore cannot be freed. This mutex also
41  *   prevents multiple users of the "kmemleak" debugfs file together with
42  *   modifications to the memory scanning parameters including the scan_thread
43  *   pointer
44  *
45  * Locks and mutexes are acquired/nested in the following order:
46  *
47  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48  *
49  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
50  * regions.
51  *
52  * The kmemleak_object structures have a use_count incremented or decremented
53  * using the get_object()/put_object() functions. When the use_count becomes
54  * 0, this count can no longer be incremented and put_object() schedules the
55  * kmemleak_object freeing via an RCU callback. All calls to the get_object()
56  * function must be protected by rcu_read_lock() to avoid accessing a freed
57  * structure.
58  */
59
60 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
61
62 #include <linux/init.h>
63 #include <linux/kernel.h>
64 #include <linux/list.h>
65 #include <linux/sched/signal.h>
66 #include <linux/sched/task.h>
67 #include <linux/sched/task_stack.h>
68 #include <linux/jiffies.h>
69 #include <linux/delay.h>
70 #include <linux/export.h>
71 #include <linux/kthread.h>
72 #include <linux/rbtree.h>
73 #include <linux/fs.h>
74 #include <linux/debugfs.h>
75 #include <linux/seq_file.h>
76 #include <linux/cpumask.h>
77 #include <linux/spinlock.h>
78 #include <linux/module.h>
79 #include <linux/mutex.h>
80 #include <linux/rcupdate.h>
81 #include <linux/stacktrace.h>
82 #include <linux/cache.h>
83 #include <linux/percpu.h>
84 #include <linux/memblock.h>
85 #include <linux/pfn.h>
86 #include <linux/mmzone.h>
87 #include <linux/slab.h>
88 #include <linux/thread_info.h>
89 #include <linux/err.h>
90 #include <linux/uaccess.h>
91 #include <linux/string.h>
92 #include <linux/nodemask.h>
93 #include <linux/mm.h>
94 #include <linux/workqueue.h>
95 #include <linux/crc32.h>
96
97 #include <asm/sections.h>
98 #include <asm/processor.h>
99 #include <linux/atomic.h>
100
101 #include <linux/kasan.h>
102 #include <linux/kfence.h>
103 #include <linux/kmemleak.h>
104 #include <linux/memory_hotplug.h>
105
106 /*
107  * Kmemleak configuration and common defines.
108  */
109 #define MAX_TRACE               16      /* stack trace length */
110 #define MSECS_MIN_AGE           5000    /* minimum object age for reporting */
111 #define SECS_FIRST_SCAN         60      /* delay before the first scan */
112 #define SECS_SCAN_WAIT          600     /* subsequent auto scanning delay */
113 #define MAX_SCAN_SIZE           4096    /* maximum size of a scanned block */
114
115 #define BYTES_PER_POINTER       sizeof(void *)
116
117 /* GFP bitmask for kmemleak internal allocations */
118 #define gfp_kmemleak_mask(gfp)  (((gfp) & (GFP_KERNEL | GFP_ATOMIC | \
119                                            __GFP_NOLOCKDEP)) | \
120                                  __GFP_NORETRY | __GFP_NOMEMALLOC | \
121                                  __GFP_NOWARN)
122
123 /* scanning area inside a memory block */
124 struct kmemleak_scan_area {
125         struct hlist_node node;
126         unsigned long start;
127         size_t size;
128 };
129
130 #define KMEMLEAK_GREY   0
131 #define KMEMLEAK_BLACK  -1
132
133 /*
134  * Structure holding the metadata for each allocated memory block.
135  * Modifications to such objects should be made while holding the
136  * object->lock. Insertions or deletions from object_list, gray_list or
137  * rb_node are already protected by the corresponding locks or mutex (see
138  * the notes on locking above). These objects are reference-counted
139  * (use_count) and freed using the RCU mechanism.
140  */
141 struct kmemleak_object {
142         raw_spinlock_t lock;
143         unsigned int flags;             /* object status flags */
144         struct list_head object_list;
145         struct list_head gray_list;
146         struct rb_node rb_node;
147         struct rcu_head rcu;            /* object_list lockless traversal */
148         /* object usage count; object freed when use_count == 0 */
149         atomic_t use_count;
150         unsigned long pointer;
151         size_t size;
152         /* pass surplus references to this pointer */
153         unsigned long excess_ref;
154         /* minimum number of a pointers found before it is considered leak */
155         int min_count;
156         /* the total number of pointers found pointing to this object */
157         int count;
158         /* checksum for detecting modified objects */
159         u32 checksum;
160         /* memory ranges to be scanned inside an object (empty for all) */
161         struct hlist_head area_list;
162         unsigned long trace[MAX_TRACE];
163         unsigned int trace_len;
164         unsigned long jiffies;          /* creation timestamp */
165         pid_t pid;                      /* pid of the current task */
166         char comm[TASK_COMM_LEN];       /* executable name */
167 };
168
169 /* flag representing the memory block allocation status */
170 #define OBJECT_ALLOCATED        (1 << 0)
171 /* flag set after the first reporting of an unreference object */
172 #define OBJECT_REPORTED         (1 << 1)
173 /* flag set to not scan the object */
174 #define OBJECT_NO_SCAN          (1 << 2)
175 /* flag set to fully scan the object when scan_area allocation failed */
176 #define OBJECT_FULL_SCAN        (1 << 3)
177 /* flag set for object allocated with physical address */
178 #define OBJECT_PHYS             (1 << 4)
179
180 #define HEX_PREFIX              "    "
181 /* number of bytes to print per line; must be 16 or 32 */
182 #define HEX_ROW_SIZE            16
183 /* number of bytes to print at a time (1, 2, 4, 8) */
184 #define HEX_GROUP_SIZE          1
185 /* include ASCII after the hex output */
186 #define HEX_ASCII               1
187 /* max number of lines to be printed */
188 #define HEX_MAX_LINES           2
189
190 /* the list of all allocated objects */
191 static LIST_HEAD(object_list);
192 /* the list of gray-colored objects (see color_gray comment below) */
193 static LIST_HEAD(gray_list);
194 /* memory pool allocation */
195 static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
196 static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
197 static LIST_HEAD(mem_pool_free_list);
198 /* search tree for object boundaries */
199 static struct rb_root object_tree_root = RB_ROOT;
200 /* search tree for object (with OBJECT_PHYS flag) boundaries */
201 static struct rb_root object_phys_tree_root = RB_ROOT;
202 /* protecting the access to object_list, object_tree_root (or object_phys_tree_root) */
203 static DEFINE_RAW_SPINLOCK(kmemleak_lock);
204
205 /* allocation caches for kmemleak internal data */
206 static struct kmem_cache *object_cache;
207 static struct kmem_cache *scan_area_cache;
208
209 /* set if tracing memory operations is enabled */
210 static int kmemleak_enabled = 1;
211 /* same as above but only for the kmemleak_free() callback */
212 static int kmemleak_free_enabled = 1;
213 /* set in the late_initcall if there were no errors */
214 static int kmemleak_initialized;
215 /* set if a kmemleak warning was issued */
216 static int kmemleak_warning;
217 /* set if a fatal kmemleak error has occurred */
218 static int kmemleak_error;
219
220 /* minimum and maximum address that may be valid pointers */
221 static unsigned long min_addr = ULONG_MAX;
222 static unsigned long max_addr;
223
224 static struct task_struct *scan_thread;
225 /* used to avoid reporting of recently allocated objects */
226 static unsigned long jiffies_min_age;
227 static unsigned long jiffies_last_scan;
228 /* delay between automatic memory scannings */
229 static unsigned long jiffies_scan_wait;
230 /* enables or disables the task stacks scanning */
231 static int kmemleak_stack_scan = 1;
232 /* protects the memory scanning, parameters and debug/kmemleak file access */
233 static DEFINE_MUTEX(scan_mutex);
234 /* setting kmemleak=on, will set this var, skipping the disable */
235 static int kmemleak_skip_disable;
236 /* If there are leaks that can be reported */
237 static bool kmemleak_found_leaks;
238
239 static bool kmemleak_verbose;
240 module_param_named(verbose, kmemleak_verbose, bool, 0600);
241
242 static void kmemleak_disable(void);
243
244 /*
245  * Print a warning and dump the stack trace.
246  */
247 #define kmemleak_warn(x...)     do {            \
248         pr_warn(x);                             \
249         dump_stack();                           \
250         kmemleak_warning = 1;                   \
251 } while (0)
252
253 /*
254  * Macro invoked when a serious kmemleak condition occurred and cannot be
255  * recovered from. Kmemleak will be disabled and further allocation/freeing
256  * tracing no longer available.
257  */
258 #define kmemleak_stop(x...)     do {    \
259         kmemleak_warn(x);               \
260         kmemleak_disable();             \
261 } while (0)
262
263 #define warn_or_seq_printf(seq, fmt, ...)       do {    \
264         if (seq)                                        \
265                 seq_printf(seq, fmt, ##__VA_ARGS__);    \
266         else                                            \
267                 pr_warn(fmt, ##__VA_ARGS__);            \
268 } while (0)
269
270 static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
271                                  int rowsize, int groupsize, const void *buf,
272                                  size_t len, bool ascii)
273 {
274         if (seq)
275                 seq_hex_dump(seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
276                              buf, len, ascii);
277         else
278                 print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
279                                rowsize, groupsize, buf, len, ascii);
280 }
281
282 /*
283  * Printing of the objects hex dump to the seq file. The number of lines to be
284  * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
285  * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
286  * with the object->lock held.
287  */
288 static void hex_dump_object(struct seq_file *seq,
289                             struct kmemleak_object *object)
290 {
291         const u8 *ptr = (const u8 *)object->pointer;
292         size_t len;
293
294         if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
295                 return;
296
297         /* limit the number of lines to HEX_MAX_LINES */
298         len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
299
300         warn_or_seq_printf(seq, "  hex dump (first %zu bytes):\n", len);
301         kasan_disable_current();
302         warn_or_seq_hex_dump(seq, DUMP_PREFIX_NONE, HEX_ROW_SIZE,
303                              HEX_GROUP_SIZE, kasan_reset_tag((void *)ptr), len, HEX_ASCII);
304         kasan_enable_current();
305 }
306
307 /*
308  * Object colors, encoded with count and min_count:
309  * - white - orphan object, not enough references to it (count < min_count)
310  * - gray  - not orphan, not marked as false positive (min_count == 0) or
311  *              sufficient references to it (count >= min_count)
312  * - black - ignore, it doesn't contain references (e.g. text section)
313  *              (min_count == -1). No function defined for this color.
314  * Newly created objects don't have any color assigned (object->count == -1)
315  * before the next memory scan when they become white.
316  */
317 static bool color_white(const struct kmemleak_object *object)
318 {
319         return object->count != KMEMLEAK_BLACK &&
320                 object->count < object->min_count;
321 }
322
323 static bool color_gray(const struct kmemleak_object *object)
324 {
325         return object->min_count != KMEMLEAK_BLACK &&
326                 object->count >= object->min_count;
327 }
328
329 /*
330  * Objects are considered unreferenced only if their color is white, they have
331  * not be deleted and have a minimum age to avoid false positives caused by
332  * pointers temporarily stored in CPU registers.
333  */
334 static bool unreferenced_object(struct kmemleak_object *object)
335 {
336         return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
337                 time_before_eq(object->jiffies + jiffies_min_age,
338                                jiffies_last_scan);
339 }
340
341 /*
342  * Printing of the unreferenced objects information to the seq file. The
343  * print_unreferenced function must be called with the object->lock held.
344  */
345 static void print_unreferenced(struct seq_file *seq,
346                                struct kmemleak_object *object)
347 {
348         int i;
349         unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
350
351         warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
352                    object->pointer, object->size);
353         warn_or_seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
354                    object->comm, object->pid, object->jiffies,
355                    msecs_age / 1000, msecs_age % 1000);
356         hex_dump_object(seq, object);
357         warn_or_seq_printf(seq, "  backtrace:\n");
358
359         for (i = 0; i < object->trace_len; i++) {
360                 void *ptr = (void *)object->trace[i];
361                 warn_or_seq_printf(seq, "    [<%p>] %pS\n", ptr, ptr);
362         }
363 }
364
365 /*
366  * Print the kmemleak_object information. This function is used mainly for
367  * debugging special cases when kmemleak operations. It must be called with
368  * the object->lock held.
369  */
370 static void dump_object_info(struct kmemleak_object *object)
371 {
372         pr_notice("Object 0x%08lx (size %zu):\n",
373                   object->pointer, object->size);
374         pr_notice("  comm \"%s\", pid %d, jiffies %lu\n",
375                   object->comm, object->pid, object->jiffies);
376         pr_notice("  min_count = %d\n", object->min_count);
377         pr_notice("  count = %d\n", object->count);
378         pr_notice("  flags = 0x%x\n", object->flags);
379         pr_notice("  checksum = %u\n", object->checksum);
380         pr_notice("  backtrace:\n");
381         stack_trace_print(object->trace, object->trace_len, 4);
382 }
383
384 /*
385  * Look-up a memory block metadata (kmemleak_object) in the object search
386  * tree based on a pointer value. If alias is 0, only values pointing to the
387  * beginning of the memory block are allowed. The kmemleak_lock must be held
388  * when calling this function.
389  */
390 static struct kmemleak_object *__lookup_object(unsigned long ptr, int alias,
391                                                bool is_phys)
392 {
393         struct rb_node *rb = is_phys ? object_phys_tree_root.rb_node :
394                              object_tree_root.rb_node;
395         unsigned long untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
396
397         while (rb) {
398                 struct kmemleak_object *object;
399                 unsigned long untagged_objp;
400
401                 object = rb_entry(rb, struct kmemleak_object, rb_node);
402                 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
403
404                 if (untagged_ptr < untagged_objp)
405                         rb = object->rb_node.rb_left;
406                 else if (untagged_objp + object->size <= untagged_ptr)
407                         rb = object->rb_node.rb_right;
408                 else if (untagged_objp == untagged_ptr || alias)
409                         return object;
410                 else {
411                         kmemleak_warn("Found object by alias at 0x%08lx\n",
412                                       ptr);
413                         dump_object_info(object);
414                         break;
415                 }
416         }
417         return NULL;
418 }
419
420 /* Look-up a kmemleak object which allocated with virtual address. */
421 static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
422 {
423         return __lookup_object(ptr, alias, false);
424 }
425
426 /*
427  * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
428  * that once an object's use_count reached 0, the RCU freeing was already
429  * registered and the object should no longer be used. This function must be
430  * called under the protection of rcu_read_lock().
431  */
432 static int get_object(struct kmemleak_object *object)
433 {
434         return atomic_inc_not_zero(&object->use_count);
435 }
436
437 /*
438  * Memory pool allocation and freeing. kmemleak_lock must not be held.
439  */
440 static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
441 {
442         unsigned long flags;
443         struct kmemleak_object *object;
444
445         /* try the slab allocator first */
446         if (object_cache) {
447                 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
448                 if (object)
449                         return object;
450         }
451
452         /* slab allocation failed, try the memory pool */
453         raw_spin_lock_irqsave(&kmemleak_lock, flags);
454         object = list_first_entry_or_null(&mem_pool_free_list,
455                                           typeof(*object), object_list);
456         if (object)
457                 list_del(&object->object_list);
458         else if (mem_pool_free_count)
459                 object = &mem_pool[--mem_pool_free_count];
460         else
461                 pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
462         raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
463
464         return object;
465 }
466
467 /*
468  * Return the object to either the slab allocator or the memory pool.
469  */
470 static void mem_pool_free(struct kmemleak_object *object)
471 {
472         unsigned long flags;
473
474         if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
475                 kmem_cache_free(object_cache, object);
476                 return;
477         }
478
479         /* add the object to the memory pool free list */
480         raw_spin_lock_irqsave(&kmemleak_lock, flags);
481         list_add(&object->object_list, &mem_pool_free_list);
482         raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
483 }
484
485 /*
486  * RCU callback to free a kmemleak_object.
487  */
488 static void free_object_rcu(struct rcu_head *rcu)
489 {
490         struct hlist_node *tmp;
491         struct kmemleak_scan_area *area;
492         struct kmemleak_object *object =
493                 container_of(rcu, struct kmemleak_object, rcu);
494
495         /*
496          * Once use_count is 0 (guaranteed by put_object), there is no other
497          * code accessing this object, hence no need for locking.
498          */
499         hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
500                 hlist_del(&area->node);
501                 kmem_cache_free(scan_area_cache, area);
502         }
503         mem_pool_free(object);
504 }
505
506 /*
507  * Decrement the object use_count. Once the count is 0, free the object using
508  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
509  * delete_object() path, the delayed RCU freeing ensures that there is no
510  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
511  * is also possible.
512  */
513 static void put_object(struct kmemleak_object *object)
514 {
515         if (!atomic_dec_and_test(&object->use_count))
516                 return;
517
518         /* should only get here after delete_object was called */
519         WARN_ON(object->flags & OBJECT_ALLOCATED);
520
521         /*
522          * It may be too early for the RCU callbacks, however, there is no
523          * concurrent object_list traversal when !object_cache and all objects
524          * came from the memory pool. Free the object directly.
525          */
526         if (object_cache)
527                 call_rcu(&object->rcu, free_object_rcu);
528         else
529                 free_object_rcu(&object->rcu);
530 }
531
532 /*
533  * Look up an object in the object search tree and increase its use_count.
534  */
535 static struct kmemleak_object *__find_and_get_object(unsigned long ptr, int alias,
536                                                      bool is_phys)
537 {
538         unsigned long flags;
539         struct kmemleak_object *object;
540
541         rcu_read_lock();
542         raw_spin_lock_irqsave(&kmemleak_lock, flags);
543         object = __lookup_object(ptr, alias, is_phys);
544         raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
545
546         /* check whether the object is still available */
547         if (object && !get_object(object))
548                 object = NULL;
549         rcu_read_unlock();
550
551         return object;
552 }
553
554 /* Look up and get an object which allocated with virtual address. */
555 static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
556 {
557         return __find_and_get_object(ptr, alias, false);
558 }
559
560 /*
561  * Remove an object from the object_tree_root (or object_phys_tree_root)
562  * and object_list. Must be called with the kmemleak_lock held _if_ kmemleak
563  * is still enabled.
564  */
565 static void __remove_object(struct kmemleak_object *object)
566 {
567         rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?
568                                    &object_phys_tree_root :
569                                    &object_tree_root);
570         list_del_rcu(&object->object_list);
571 }
572
573 /*
574  * Look up an object in the object search tree and remove it from both
575  * object_tree_root (or object_phys_tree_root) and object_list. The
576  * returned object's use_count should be at least 1, as initially set
577  * by create_object().
578  */
579 static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias,
580                                                       bool is_phys)
581 {
582         unsigned long flags;
583         struct kmemleak_object *object;
584
585         raw_spin_lock_irqsave(&kmemleak_lock, flags);
586         object = __lookup_object(ptr, alias, is_phys);
587         if (object)
588                 __remove_object(object);
589         raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
590
591         return object;
592 }
593
594 /*
595  * Save stack trace to the given array of MAX_TRACE size.
596  */
597 static int __save_stack_trace(unsigned long *trace)
598 {
599         return stack_trace_save(trace, MAX_TRACE, 2);
600 }
601
602 /*
603  * Create the metadata (struct kmemleak_object) corresponding to an allocated
604  * memory block and add it to the object_list and object_tree_root (or
605  * object_phys_tree_root).
606  */
607 static void __create_object(unsigned long ptr, size_t size,
608                             int min_count, gfp_t gfp, bool is_phys)
609 {
610         unsigned long flags;
611         struct kmemleak_object *object, *parent;
612         struct rb_node **link, *rb_parent;
613         unsigned long untagged_ptr;
614         unsigned long untagged_objp;
615
616         object = mem_pool_alloc(gfp);
617         if (!object) {
618                 pr_warn("Cannot allocate a kmemleak_object structure\n");
619                 kmemleak_disable();
620                 return;
621         }
622
623         INIT_LIST_HEAD(&object->object_list);
624         INIT_LIST_HEAD(&object->gray_list);
625         INIT_HLIST_HEAD(&object->area_list);
626         raw_spin_lock_init(&object->lock);
627         atomic_set(&object->use_count, 1);
628         object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);
629         object->pointer = ptr;
630         object->size = kfence_ksize((void *)ptr) ?: size;
631         object->excess_ref = 0;
632         object->min_count = min_count;
633         object->count = 0;                      /* white color initially */
634         object->jiffies = jiffies;
635         object->checksum = 0;
636
637         /* task information */
638         if (in_hardirq()) {
639                 object->pid = 0;
640                 strncpy(object->comm, "hardirq", sizeof(object->comm));
641         } else if (in_serving_softirq()) {
642                 object->pid = 0;
643                 strncpy(object->comm, "softirq", sizeof(object->comm));
644         } else {
645                 object->pid = current->pid;
646                 /*
647                  * There is a small chance of a race with set_task_comm(),
648                  * however using get_task_comm() here may cause locking
649                  * dependency issues with current->alloc_lock. In the worst
650                  * case, the command line is not correct.
651                  */
652                 strncpy(object->comm, current->comm, sizeof(object->comm));
653         }
654
655         /* kernel backtrace */
656         object->trace_len = __save_stack_trace(object->trace);
657
658         raw_spin_lock_irqsave(&kmemleak_lock, flags);
659
660         untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
661         /*
662          * Only update min_addr and max_addr with object
663          * storing virtual address.
664          */
665         if (!is_phys) {
666                 min_addr = min(min_addr, untagged_ptr);
667                 max_addr = max(max_addr, untagged_ptr + size);
668         }
669         link = is_phys ? &object_phys_tree_root.rb_node :
670                 &object_tree_root.rb_node;
671         rb_parent = NULL;
672         while (*link) {
673                 rb_parent = *link;
674                 parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
675                 untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer);
676                 if (untagged_ptr + size <= untagged_objp)
677                         link = &parent->rb_node.rb_left;
678                 else if (untagged_objp + parent->size <= untagged_ptr)
679                         link = &parent->rb_node.rb_right;
680                 else {
681                         kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
682                                       ptr);
683                         /*
684                          * No need for parent->lock here since "parent" cannot
685                          * be freed while the kmemleak_lock is held.
686                          */
687                         dump_object_info(parent);
688                         kmem_cache_free(object_cache, object);
689                         goto out;
690                 }
691         }
692         rb_link_node(&object->rb_node, rb_parent, link);
693         rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :
694                                           &object_tree_root);
695
696         list_add_tail_rcu(&object->object_list, &object_list);
697 out:
698         raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
699 }
700
701 /* Create kmemleak object which allocated with virtual address. */
702 static void create_object(unsigned long ptr, size_t size,
703                           int min_count, gfp_t gfp)
704 {
705         __create_object(ptr, size, min_count, gfp, false);
706 }
707
708 /* Create kmemleak object which allocated with physical address. */
709 static void create_object_phys(unsigned long ptr, size_t size,
710                                int min_count, gfp_t gfp)
711 {
712         __create_object(ptr, size, min_count, gfp, true);
713 }
714
715 /*
716  * Mark the object as not allocated and schedule RCU freeing via put_object().
717  */
718 static void __delete_object(struct kmemleak_object *object)
719 {
720         unsigned long flags;
721
722         WARN_ON(!(object->flags & OBJECT_ALLOCATED));
723         WARN_ON(atomic_read(&object->use_count) < 1);
724
725         /*
726          * Locking here also ensures that the corresponding memory block
727          * cannot be freed when it is being scanned.
728          */
729         raw_spin_lock_irqsave(&object->lock, flags);
730         object->flags &= ~OBJECT_ALLOCATED;
731         raw_spin_unlock_irqrestore(&object->lock, flags);
732         put_object(object);
733 }
734
735 /*
736  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
737  * delete it.
738  */
739 static void delete_object_full(unsigned long ptr)
740 {
741         struct kmemleak_object *object;
742
743         object = find_and_remove_object(ptr, 0, false);
744         if (!object) {
745 #ifdef DEBUG
746                 kmemleak_warn("Freeing unknown object at 0x%08lx\n",
747                               ptr);
748 #endif
749                 return;
750         }
751         __delete_object(object);
752 }
753
754 /*
755  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
756  * delete it. If the memory block is partially freed, the function may create
757  * additional metadata for the remaining parts of the block.
758  */
759 static void delete_object_part(unsigned long ptr, size_t size, bool is_phys)
760 {
761         struct kmemleak_object *object;
762         unsigned long start, end;
763
764         object = find_and_remove_object(ptr, 1, is_phys);
765         if (!object) {
766 #ifdef DEBUG
767                 kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
768                               ptr, size);
769 #endif
770                 return;
771         }
772
773         /*
774          * Create one or two objects that may result from the memory block
775          * split. Note that partial freeing is only done by free_bootmem() and
776          * this happens before kmemleak_init() is called.
777          */
778         start = object->pointer;
779         end = object->pointer + object->size;
780         if (ptr > start)
781                 __create_object(start, ptr - start, object->min_count,
782                               GFP_KERNEL, is_phys);
783         if (ptr + size < end)
784                 __create_object(ptr + size, end - ptr - size, object->min_count,
785                               GFP_KERNEL, is_phys);
786
787         __delete_object(object);
788 }
789
790 static void __paint_it(struct kmemleak_object *object, int color)
791 {
792         object->min_count = color;
793         if (color == KMEMLEAK_BLACK)
794                 object->flags |= OBJECT_NO_SCAN;
795 }
796
797 static void paint_it(struct kmemleak_object *object, int color)
798 {
799         unsigned long flags;
800
801         raw_spin_lock_irqsave(&object->lock, flags);
802         __paint_it(object, color);
803         raw_spin_unlock_irqrestore(&object->lock, flags);
804 }
805
806 static void paint_ptr(unsigned long ptr, int color, bool is_phys)
807 {
808         struct kmemleak_object *object;
809
810         object = __find_and_get_object(ptr, 0, is_phys);
811         if (!object) {
812                 kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
813                               ptr,
814                               (color == KMEMLEAK_GREY) ? "Grey" :
815                               (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
816                 return;
817         }
818         paint_it(object, color);
819         put_object(object);
820 }
821
822 /*
823  * Mark an object permanently as gray-colored so that it can no longer be
824  * reported as a leak. This is used in general to mark a false positive.
825  */
826 static void make_gray_object(unsigned long ptr)
827 {
828         paint_ptr(ptr, KMEMLEAK_GREY, false);
829 }
830
831 /*
832  * Mark the object as black-colored so that it is ignored from scans and
833  * reporting.
834  */
835 static void make_black_object(unsigned long ptr, bool is_phys)
836 {
837         paint_ptr(ptr, KMEMLEAK_BLACK, is_phys);
838 }
839
840 /*
841  * Add a scanning area to the object. If at least one such area is added,
842  * kmemleak will only scan these ranges rather than the whole memory block.
843  */
844 static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
845 {
846         unsigned long flags;
847         struct kmemleak_object *object;
848         struct kmemleak_scan_area *area = NULL;
849         unsigned long untagged_ptr;
850         unsigned long untagged_objp;
851
852         object = find_and_get_object(ptr, 1);
853         if (!object) {
854                 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
855                               ptr);
856                 return;
857         }
858
859         untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
860         untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);
861
862         if (scan_area_cache)
863                 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
864
865         raw_spin_lock_irqsave(&object->lock, flags);
866         if (!area) {
867                 pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
868                 /* mark the object for full scan to avoid false positives */
869                 object->flags |= OBJECT_FULL_SCAN;
870                 goto out_unlock;
871         }
872         if (size == SIZE_MAX) {
873                 size = untagged_objp + object->size - untagged_ptr;
874         } else if (untagged_ptr + size > untagged_objp + object->size) {
875                 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
876                 dump_object_info(object);
877                 kmem_cache_free(scan_area_cache, area);
878                 goto out_unlock;
879         }
880
881         INIT_HLIST_NODE(&area->node);
882         area->start = ptr;
883         area->size = size;
884
885         hlist_add_head(&area->node, &object->area_list);
886 out_unlock:
887         raw_spin_unlock_irqrestore(&object->lock, flags);
888         put_object(object);
889 }
890
891 /*
892  * Any surplus references (object already gray) to 'ptr' are passed to
893  * 'excess_ref'. This is used in the vmalloc() case where a pointer to
894  * vm_struct may be used as an alternative reference to the vmalloc'ed object
895  * (see free_thread_stack()).
896  */
897 static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
898 {
899         unsigned long flags;
900         struct kmemleak_object *object;
901
902         object = find_and_get_object(ptr, 0);
903         if (!object) {
904                 kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
905                               ptr);
906                 return;
907         }
908
909         raw_spin_lock_irqsave(&object->lock, flags);
910         object->excess_ref = excess_ref;
911         raw_spin_unlock_irqrestore(&object->lock, flags);
912         put_object(object);
913 }
914
915 /*
916  * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
917  * pointer. Such object will not be scanned by kmemleak but references to it
918  * are searched.
919  */
920 static void object_no_scan(unsigned long ptr)
921 {
922         unsigned long flags;
923         struct kmemleak_object *object;
924
925         object = find_and_get_object(ptr, 0);
926         if (!object) {
927                 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
928                 return;
929         }
930
931         raw_spin_lock_irqsave(&object->lock, flags);
932         object->flags |= OBJECT_NO_SCAN;
933         raw_spin_unlock_irqrestore(&object->lock, flags);
934         put_object(object);
935 }
936
937 /**
938  * kmemleak_alloc - register a newly allocated object
939  * @ptr:        pointer to beginning of the object
940  * @size:       size of the object
941  * @min_count:  minimum number of references to this object. If during memory
942  *              scanning a number of references less than @min_count is found,
943  *              the object is reported as a memory leak. If @min_count is 0,
944  *              the object is never reported as a leak. If @min_count is -1,
945  *              the object is ignored (not scanned and not reported as a leak)
946  * @gfp:        kmalloc() flags used for kmemleak internal memory allocations
947  *
948  * This function is called from the kernel allocators when a new object
949  * (memory block) is allocated (kmem_cache_alloc, kmalloc etc.).
950  */
951 void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
952                           gfp_t gfp)
953 {
954         pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
955
956         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
957                 create_object((unsigned long)ptr, size, min_count, gfp);
958 }
959 EXPORT_SYMBOL_GPL(kmemleak_alloc);
960
961 /**
962  * kmemleak_alloc_percpu - register a newly allocated __percpu object
963  * @ptr:        __percpu pointer to beginning of the object
964  * @size:       size of the object
965  * @gfp:        flags used for kmemleak internal memory allocations
966  *
967  * This function is called from the kernel percpu allocator when a new object
968  * (memory block) is allocated (alloc_percpu).
969  */
970 void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
971                                  gfp_t gfp)
972 {
973         unsigned int cpu;
974
975         pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
976
977         /*
978          * Percpu allocations are only scanned and not reported as leaks
979          * (min_count is set to 0).
980          */
981         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
982                 for_each_possible_cpu(cpu)
983                         create_object((unsigned long)per_cpu_ptr(ptr, cpu),
984                                       size, 0, gfp);
985 }
986 EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
987
988 /**
989  * kmemleak_vmalloc - register a newly vmalloc'ed object
990  * @area:       pointer to vm_struct
991  * @size:       size of the object
992  * @gfp:        __vmalloc() flags used for kmemleak internal memory allocations
993  *
994  * This function is called from the vmalloc() kernel allocator when a new
995  * object (memory block) is allocated.
996  */
997 void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
998 {
999         pr_debug("%s(0x%p, %zu)\n", __func__, area, size);
1000
1001         /*
1002          * A min_count = 2 is needed because vm_struct contains a reference to
1003          * the virtual address of the vmalloc'ed block.
1004          */
1005         if (kmemleak_enabled) {
1006                 create_object((unsigned long)area->addr, size, 2, gfp);
1007                 object_set_excess_ref((unsigned long)area,
1008                                       (unsigned long)area->addr);
1009         }
1010 }
1011 EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
1012
1013 /**
1014  * kmemleak_free - unregister a previously registered object
1015  * @ptr:        pointer to beginning of the object
1016  *
1017  * This function is called from the kernel allocators when an object (memory
1018  * block) is freed (kmem_cache_free, kfree, vfree etc.).
1019  */
1020 void __ref kmemleak_free(const void *ptr)
1021 {
1022         pr_debug("%s(0x%p)\n", __func__, ptr);
1023
1024         if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1025                 delete_object_full((unsigned long)ptr);
1026 }
1027 EXPORT_SYMBOL_GPL(kmemleak_free);
1028
1029 /**
1030  * kmemleak_free_part - partially unregister a previously registered object
1031  * @ptr:        pointer to the beginning or inside the object. This also
1032  *              represents the start of the range to be freed
1033  * @size:       size to be unregistered
1034  *
1035  * This function is called when only a part of a memory block is freed
1036  * (usually from the bootmem allocator).
1037  */
1038 void __ref kmemleak_free_part(const void *ptr, size_t size)
1039 {
1040         pr_debug("%s(0x%p)\n", __func__, ptr);
1041
1042         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1043                 delete_object_part((unsigned long)ptr, size, false);
1044 }
1045 EXPORT_SYMBOL_GPL(kmemleak_free_part);
1046
1047 /**
1048  * kmemleak_free_percpu - unregister a previously registered __percpu object
1049  * @ptr:        __percpu pointer to beginning of the object
1050  *
1051  * This function is called from the kernel percpu allocator when an object
1052  * (memory block) is freed (free_percpu).
1053  */
1054 void __ref kmemleak_free_percpu(const void __percpu *ptr)
1055 {
1056         unsigned int cpu;
1057
1058         pr_debug("%s(0x%p)\n", __func__, ptr);
1059
1060         if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1061                 for_each_possible_cpu(cpu)
1062                         delete_object_full((unsigned long)per_cpu_ptr(ptr,
1063                                                                       cpu));
1064 }
1065 EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1066
1067 /**
1068  * kmemleak_update_trace - update object allocation stack trace
1069  * @ptr:        pointer to beginning of the object
1070  *
1071  * Override the object allocation stack trace for cases where the actual
1072  * allocation place is not always useful.
1073  */
1074 void __ref kmemleak_update_trace(const void *ptr)
1075 {
1076         struct kmemleak_object *object;
1077         unsigned long flags;
1078
1079         pr_debug("%s(0x%p)\n", __func__, ptr);
1080
1081         if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1082                 return;
1083
1084         object = find_and_get_object((unsigned long)ptr, 1);
1085         if (!object) {
1086 #ifdef DEBUG
1087                 kmemleak_warn("Updating stack trace for unknown object at %p\n",
1088                               ptr);
1089 #endif
1090                 return;
1091         }
1092
1093         raw_spin_lock_irqsave(&object->lock, flags);
1094         object->trace_len = __save_stack_trace(object->trace);
1095         raw_spin_unlock_irqrestore(&object->lock, flags);
1096
1097         put_object(object);
1098 }
1099 EXPORT_SYMBOL(kmemleak_update_trace);
1100
1101 /**
1102  * kmemleak_not_leak - mark an allocated object as false positive
1103  * @ptr:        pointer to beginning of the object
1104  *
1105  * Calling this function on an object will cause the memory block to no longer
1106  * be reported as leak and always be scanned.
1107  */
1108 void __ref kmemleak_not_leak(const void *ptr)
1109 {
1110         pr_debug("%s(0x%p)\n", __func__, ptr);
1111
1112         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1113                 make_gray_object((unsigned long)ptr);
1114 }
1115 EXPORT_SYMBOL(kmemleak_not_leak);
1116
1117 /**
1118  * kmemleak_ignore - ignore an allocated object
1119  * @ptr:        pointer to beginning of the object
1120  *
1121  * Calling this function on an object will cause the memory block to be
1122  * ignored (not scanned and not reported as a leak). This is usually done when
1123  * it is known that the corresponding block is not a leak and does not contain
1124  * any references to other allocated memory blocks.
1125  */
1126 void __ref kmemleak_ignore(const void *ptr)
1127 {
1128         pr_debug("%s(0x%p)\n", __func__, ptr);
1129
1130         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1131                 make_black_object((unsigned long)ptr, false);
1132 }
1133 EXPORT_SYMBOL(kmemleak_ignore);
1134
1135 /**
1136  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1137  * @ptr:        pointer to beginning or inside the object. This also
1138  *              represents the start of the scan area
1139  * @size:       size of the scan area
1140  * @gfp:        kmalloc() flags used for kmemleak internal memory allocations
1141  *
1142  * This function is used when it is known that only certain parts of an object
1143  * contain references to other objects. Kmemleak will only scan these areas
1144  * reducing the number false negatives.
1145  */
1146 void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1147 {
1148         pr_debug("%s(0x%p)\n", __func__, ptr);
1149
1150         if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1151                 add_scan_area((unsigned long)ptr, size, gfp);
1152 }
1153 EXPORT_SYMBOL(kmemleak_scan_area);
1154
1155 /**
1156  * kmemleak_no_scan - do not scan an allocated object
1157  * @ptr:        pointer to beginning of the object
1158  *
1159  * This function notifies kmemleak not to scan the given memory block. Useful
1160  * in situations where it is known that the given object does not contain any
1161  * references to other objects. Kmemleak will not scan such objects reducing
1162  * the number of false negatives.
1163  */
1164 void __ref kmemleak_no_scan(const void *ptr)
1165 {
1166         pr_debug("%s(0x%p)\n", __func__, ptr);
1167
1168         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1169                 object_no_scan((unsigned long)ptr);
1170 }
1171 EXPORT_SYMBOL(kmemleak_no_scan);
1172
1173 /**
1174  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1175  *                       address argument
1176  * @phys:       physical address of the object
1177  * @size:       size of the object
1178  * @gfp:        kmalloc() flags used for kmemleak internal memory allocations
1179  */
1180 void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, gfp_t gfp)
1181 {
1182         pr_debug("%s(0x%pa, %zu)\n", __func__, &phys, size);
1183
1184         if (kmemleak_enabled)
1185                 /*
1186                  * Create object with OBJECT_PHYS flag and
1187                  * assume min_count 0.
1188                  */
1189                 create_object_phys((unsigned long)phys, size, 0, gfp);
1190 }
1191 EXPORT_SYMBOL(kmemleak_alloc_phys);
1192
1193 /**
1194  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1195  *                           physical address argument
1196  * @phys:       physical address if the beginning or inside an object. This
1197  *              also represents the start of the range to be freed
1198  * @size:       size to be unregistered
1199  */
1200 void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1201 {
1202         pr_debug("%s(0x%pa)\n", __func__, &phys);
1203
1204         if (kmemleak_enabled)
1205                 delete_object_part((unsigned long)phys, size, true);
1206 }
1207 EXPORT_SYMBOL(kmemleak_free_part_phys);
1208
1209 /**
1210  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1211  *                        address argument
1212  * @phys:       physical address of the object
1213  */
1214 void __ref kmemleak_ignore_phys(phys_addr_t phys)
1215 {
1216         pr_debug("%s(0x%pa)\n", __func__, &phys);
1217
1218         if (kmemleak_enabled)
1219                 make_black_object((unsigned long)phys, true);
1220 }
1221 EXPORT_SYMBOL(kmemleak_ignore_phys);
1222
1223 /*
1224  * Update an object's checksum and return true if it was modified.
1225  */
1226 static bool update_checksum(struct kmemleak_object *object)
1227 {
1228         u32 old_csum = object->checksum;
1229
1230         if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))
1231                 return false;
1232
1233         kasan_disable_current();
1234         kcsan_disable_current();
1235         object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
1236         kasan_enable_current();
1237         kcsan_enable_current();
1238
1239         return object->checksum != old_csum;
1240 }
1241
1242 /*
1243  * Update an object's references. object->lock must be held by the caller.
1244  */
1245 static void update_refs(struct kmemleak_object *object)
1246 {
1247         if (!color_white(object)) {
1248                 /* non-orphan, ignored or new */
1249                 return;
1250         }
1251
1252         /*
1253          * Increase the object's reference count (number of pointers to the
1254          * memory block). If this count reaches the required minimum, the
1255          * object's color will become gray and it will be added to the
1256          * gray_list.
1257          */
1258         object->count++;
1259         if (color_gray(object)) {
1260                 /* put_object() called when removing from gray_list */
1261                 WARN_ON(!get_object(object));
1262                 list_add_tail(&object->gray_list, &gray_list);
1263         }
1264 }
1265
1266 /*
1267  * Memory scanning is a long process and it needs to be interruptible. This
1268  * function checks whether such interrupt condition occurred.
1269  */
1270 static int scan_should_stop(void)
1271 {
1272         if (!kmemleak_enabled)
1273                 return 1;
1274
1275         /*
1276          * This function may be called from either process or kthread context,
1277          * hence the need to check for both stop conditions.
1278          */
1279         if (current->mm)
1280                 return signal_pending(current);
1281         else
1282                 return kthread_should_stop();
1283
1284         return 0;
1285 }
1286
1287 /*
1288  * Scan a memory block (exclusive range) for valid pointers and add those
1289  * found to the gray list.
1290  */
1291 static void scan_block(void *_start, void *_end,
1292                        struct kmemleak_object *scanned)
1293 {
1294         unsigned long *ptr;
1295         unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1296         unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1297         unsigned long flags;
1298         unsigned long untagged_ptr;
1299
1300         raw_spin_lock_irqsave(&kmemleak_lock, flags);
1301         for (ptr = start; ptr < end; ptr++) {
1302                 struct kmemleak_object *object;
1303                 unsigned long pointer;
1304                 unsigned long excess_ref;
1305
1306                 if (scan_should_stop())
1307                         break;
1308
1309                 kasan_disable_current();
1310                 pointer = *(unsigned long *)kasan_reset_tag((void *)ptr);
1311                 kasan_enable_current();
1312
1313                 untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
1314                 if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
1315                         continue;
1316
1317                 /*
1318                  * No need for get_object() here since we hold kmemleak_lock.
1319                  * object->use_count cannot be dropped to 0 while the object
1320                  * is still present in object_tree_root and object_list
1321                  * (with updates protected by kmemleak_lock).
1322                  */
1323                 object = lookup_object(pointer, 1);
1324                 if (!object)
1325                         continue;
1326                 if (object == scanned)
1327                         /* self referenced, ignore */
1328                         continue;
1329
1330                 /*
1331                  * Avoid the lockdep recursive warning on object->lock being
1332                  * previously acquired in scan_object(). These locks are
1333                  * enclosed by scan_mutex.
1334                  */
1335                 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1336                 /* only pass surplus references (object already gray) */
1337                 if (color_gray(object)) {
1338                         excess_ref = object->excess_ref;
1339                         /* no need for update_refs() if object already gray */
1340                 } else {
1341                         excess_ref = 0;
1342                         update_refs(object);
1343                 }
1344                 raw_spin_unlock(&object->lock);
1345
1346                 if (excess_ref) {
1347                         object = lookup_object(excess_ref, 0);
1348                         if (!object)
1349                                 continue;
1350                         if (object == scanned)
1351                                 /* circular reference, ignore */
1352                                 continue;
1353                         raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1354                         update_refs(object);
1355                         raw_spin_unlock(&object->lock);
1356                 }
1357         }
1358         raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
1359 }
1360
1361 /*
1362  * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
1363  */
1364 #ifdef CONFIG_SMP
1365 static void scan_large_block(void *start, void *end)
1366 {
1367         void *next;
1368
1369         while (start < end) {
1370                 next = min(start + MAX_SCAN_SIZE, end);
1371                 scan_block(start, next, NULL);
1372                 start = next;
1373                 cond_resched();
1374         }
1375 }
1376 #endif
1377
1378 /*
1379  * Scan a memory block corresponding to a kmemleak_object. A condition is
1380  * that object->use_count >= 1.
1381  */
1382 static void scan_object(struct kmemleak_object *object)
1383 {
1384         struct kmemleak_scan_area *area;
1385         unsigned long flags;
1386         void *obj_ptr;
1387
1388         /*
1389          * Once the object->lock is acquired, the corresponding memory block
1390          * cannot be freed (the same lock is acquired in delete_object).
1391          */
1392         raw_spin_lock_irqsave(&object->lock, flags);
1393         if (object->flags & OBJECT_NO_SCAN)
1394                 goto out;
1395         if (!(object->flags & OBJECT_ALLOCATED))
1396                 /* already freed object */
1397                 goto out;
1398
1399         obj_ptr = object->flags & OBJECT_PHYS ?
1400                   __va((phys_addr_t)object->pointer) :
1401                   (void *)object->pointer;
1402
1403         if (hlist_empty(&object->area_list) ||
1404             object->flags & OBJECT_FULL_SCAN) {
1405                 void *start = obj_ptr;
1406                 void *end = obj_ptr + object->size;
1407                 void *next;
1408
1409                 do {
1410                         next = min(start + MAX_SCAN_SIZE, end);
1411                         scan_block(start, next, object);
1412
1413                         start = next;
1414                         if (start >= end)
1415                                 break;
1416
1417                         raw_spin_unlock_irqrestore(&object->lock, flags);
1418                         cond_resched();
1419                         raw_spin_lock_irqsave(&object->lock, flags);
1420                 } while (object->flags & OBJECT_ALLOCATED);
1421         } else
1422                 hlist_for_each_entry(area, &object->area_list, node)
1423                         scan_block((void *)area->start,
1424                                    (void *)(area->start + area->size),
1425                                    object);
1426 out:
1427         raw_spin_unlock_irqrestore(&object->lock, flags);
1428 }
1429
1430 /*
1431  * Scan the objects already referenced (gray objects). More objects will be
1432  * referenced and, if there are no memory leaks, all the objects are scanned.
1433  */
1434 static void scan_gray_list(void)
1435 {
1436         struct kmemleak_object *object, *tmp;
1437
1438         /*
1439          * The list traversal is safe for both tail additions and removals
1440          * from inside the loop. The kmemleak objects cannot be freed from
1441          * outside the loop because their use_count was incremented.
1442          */
1443         object = list_entry(gray_list.next, typeof(*object), gray_list);
1444         while (&object->gray_list != &gray_list) {
1445                 cond_resched();
1446
1447                 /* may add new objects to the list */
1448                 if (!scan_should_stop())
1449                         scan_object(object);
1450
1451                 tmp = list_entry(object->gray_list.next, typeof(*object),
1452                                  gray_list);
1453
1454                 /* remove the object from the list and release it */
1455                 list_del(&object->gray_list);
1456                 put_object(object);
1457
1458                 object = tmp;
1459         }
1460         WARN_ON(!list_empty(&gray_list));
1461 }
1462
1463 /*
1464  * Conditionally call resched() in a object iteration loop while making sure
1465  * that the given object won't go away without RCU read lock by performing a
1466  * get_object() if !pinned.
1467  *
1468  * Return: false if can't do a cond_resched() due to get_object() failure
1469  *         true otherwise
1470  */
1471 static bool kmemleak_cond_resched(struct kmemleak_object *object, bool pinned)
1472 {
1473         if (!pinned && !get_object(object))
1474                 return false;
1475
1476         rcu_read_unlock();
1477         cond_resched();
1478         rcu_read_lock();
1479         if (!pinned)
1480                 put_object(object);
1481         return true;
1482 }
1483
1484 /*
1485  * Scan data sections and all the referenced memory blocks allocated via the
1486  * kernel's standard allocators. This function must be called with the
1487  * scan_mutex held.
1488  */
1489 static void kmemleak_scan(void)
1490 {
1491         struct kmemleak_object *object;
1492         struct zone *zone;
1493         int __maybe_unused i;
1494         int new_leaks = 0;
1495         int loop_cnt = 0;
1496
1497         jiffies_last_scan = jiffies;
1498
1499         /* prepare the kmemleak_object's */
1500         rcu_read_lock();
1501         list_for_each_entry_rcu(object, &object_list, object_list) {
1502                 bool obj_pinned = false;
1503
1504                 raw_spin_lock_irq(&object->lock);
1505 #ifdef DEBUG
1506                 /*
1507                  * With a few exceptions there should be a maximum of
1508                  * 1 reference to any object at this point.
1509                  */
1510                 if (atomic_read(&object->use_count) > 1) {
1511                         pr_debug("object->use_count = %d\n",
1512                                  atomic_read(&object->use_count));
1513                         dump_object_info(object);
1514                 }
1515 #endif
1516
1517                 /* ignore objects outside lowmem (paint them black) */
1518                 if ((object->flags & OBJECT_PHYS) &&
1519                    !(object->flags & OBJECT_NO_SCAN)) {
1520                         unsigned long phys = object->pointer;
1521
1522                         if (PHYS_PFN(phys) < min_low_pfn ||
1523                             PHYS_PFN(phys + object->size) >= max_low_pfn)
1524                                 __paint_it(object, KMEMLEAK_BLACK);
1525                 }
1526
1527                 /* reset the reference count (whiten the object) */
1528                 object->count = 0;
1529                 if (color_gray(object) && get_object(object)) {
1530                         list_add_tail(&object->gray_list, &gray_list);
1531                         obj_pinned = true;
1532                 }
1533
1534                 raw_spin_unlock_irq(&object->lock);
1535
1536                 /*
1537                  * Do a cond_resched() every 64k objects to avoid soft lockup.
1538                  */
1539                 if (!(++loop_cnt & 0xffff) &&
1540                     !kmemleak_cond_resched(object, obj_pinned))
1541                         loop_cnt--; /* Try again on next object */
1542         }
1543         rcu_read_unlock();
1544
1545 #ifdef CONFIG_SMP
1546         /* per-cpu sections scanning */
1547         for_each_possible_cpu(i)
1548                 scan_large_block(__per_cpu_start + per_cpu_offset(i),
1549                                  __per_cpu_end + per_cpu_offset(i));
1550 #endif
1551
1552         /*
1553          * Struct page scanning for each node.
1554          */
1555         get_online_mems();
1556         for_each_populated_zone(zone) {
1557                 unsigned long start_pfn = zone->zone_start_pfn;
1558                 unsigned long end_pfn = zone_end_pfn(zone);
1559                 unsigned long pfn;
1560
1561                 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1562                         struct page *page = pfn_to_online_page(pfn);
1563
1564                         if (!page)
1565                                 continue;
1566
1567                         /* only scan pages belonging to this zone */
1568                         if (page_zone(page) != zone)
1569                                 continue;
1570                         /* only scan if page is in use */
1571                         if (page_count(page) == 0)
1572                                 continue;
1573                         scan_block(page, page + 1, NULL);
1574                         if (!(pfn & 63))
1575                                 cond_resched();
1576                 }
1577         }
1578         put_online_mems();
1579
1580         /*
1581          * Scanning the task stacks (may introduce false negatives).
1582          */
1583         if (kmemleak_stack_scan) {
1584                 struct task_struct *p, *g;
1585
1586                 rcu_read_lock();
1587                 for_each_process_thread(g, p) {
1588                         void *stack = try_get_task_stack(p);
1589                         if (stack) {
1590                                 scan_block(stack, stack + THREAD_SIZE, NULL);
1591                                 put_task_stack(p);
1592                         }
1593                 }
1594                 rcu_read_unlock();
1595         }
1596
1597         /*
1598          * Scan the objects already referenced from the sections scanned
1599          * above.
1600          */
1601         scan_gray_list();
1602
1603         /*
1604          * Check for new or unreferenced objects modified since the previous
1605          * scan and color them gray until the next scan.
1606          */
1607         rcu_read_lock();
1608         loop_cnt = 0;
1609         list_for_each_entry_rcu(object, &object_list, object_list) {
1610                 /*
1611                  * Do a cond_resched() every 64k objects to avoid soft lockup.
1612                  */
1613                 if (!(++loop_cnt & 0xffff) &&
1614                     !kmemleak_cond_resched(object, false))
1615                         loop_cnt--;     /* Try again on next object */
1616
1617                 /*
1618                  * This is racy but we can save the overhead of lock/unlock
1619                  * calls. The missed objects, if any, should be caught in
1620                  * the next scan.
1621                  */
1622                 if (!color_white(object))
1623                         continue;
1624                 raw_spin_lock_irq(&object->lock);
1625                 if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1626                     && update_checksum(object) && get_object(object)) {
1627                         /* color it gray temporarily */
1628                         object->count = object->min_count;
1629                         list_add_tail(&object->gray_list, &gray_list);
1630                 }
1631                 raw_spin_unlock_irq(&object->lock);
1632         }
1633         rcu_read_unlock();
1634
1635         /*
1636          * Re-scan the gray list for modified unreferenced objects.
1637          */
1638         scan_gray_list();
1639
1640         /*
1641          * If scanning was stopped do not report any new unreferenced objects.
1642          */
1643         if (scan_should_stop())
1644                 return;
1645
1646         /*
1647          * Scanning result reporting.
1648          */
1649         rcu_read_lock();
1650         loop_cnt = 0;
1651         list_for_each_entry_rcu(object, &object_list, object_list) {
1652                 /*
1653                  * Do a cond_resched() every 64k objects to avoid soft lockup.
1654                  */
1655                 if (!(++loop_cnt & 0xffff) &&
1656                     !kmemleak_cond_resched(object, false))
1657                         loop_cnt--;     /* Try again on next object */
1658
1659                 /*
1660                  * This is racy but we can save the overhead of lock/unlock
1661                  * calls. The missed objects, if any, should be caught in
1662                  * the next scan.
1663                  */
1664                 if (!color_white(object))
1665                         continue;
1666                 raw_spin_lock_irq(&object->lock);
1667                 if (unreferenced_object(object) &&
1668                     !(object->flags & OBJECT_REPORTED)) {
1669                         object->flags |= OBJECT_REPORTED;
1670
1671                         if (kmemleak_verbose)
1672                                 print_unreferenced(NULL, object);
1673
1674                         new_leaks++;
1675                 }
1676                 raw_spin_unlock_irq(&object->lock);
1677         }
1678         rcu_read_unlock();
1679
1680         if (new_leaks) {
1681                 kmemleak_found_leaks = true;
1682
1683                 pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1684                         new_leaks);
1685         }
1686
1687 }
1688
1689 /*
1690  * Thread function performing automatic memory scanning. Unreferenced objects
1691  * at the end of a memory scan are reported but only the first time.
1692  */
1693 static int kmemleak_scan_thread(void *arg)
1694 {
1695         static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
1696
1697         pr_info("Automatic memory scanning thread started\n");
1698         set_user_nice(current, 10);
1699
1700         /*
1701          * Wait before the first scan to allow the system to fully initialize.
1702          */
1703         if (first_run) {
1704                 signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1705                 first_run = 0;
1706                 while (timeout && !kthread_should_stop())
1707                         timeout = schedule_timeout_interruptible(timeout);
1708         }
1709
1710         while (!kthread_should_stop()) {
1711                 signed long timeout = READ_ONCE(jiffies_scan_wait);
1712
1713                 mutex_lock(&scan_mutex);
1714                 kmemleak_scan();
1715                 mutex_unlock(&scan_mutex);
1716
1717                 /* wait before the next scan */
1718                 while (timeout && !kthread_should_stop())
1719                         timeout = schedule_timeout_interruptible(timeout);
1720         }
1721
1722         pr_info("Automatic memory scanning thread ended\n");
1723
1724         return 0;
1725 }
1726
1727 /*
1728  * Start the automatic memory scanning thread. This function must be called
1729  * with the scan_mutex held.
1730  */
1731 static void start_scan_thread(void)
1732 {
1733         if (scan_thread)
1734                 return;
1735         scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1736         if (IS_ERR(scan_thread)) {
1737                 pr_warn("Failed to create the scan thread\n");
1738                 scan_thread = NULL;
1739         }
1740 }
1741
1742 /*
1743  * Stop the automatic memory scanning thread.
1744  */
1745 static void stop_scan_thread(void)
1746 {
1747         if (scan_thread) {
1748                 kthread_stop(scan_thread);
1749                 scan_thread = NULL;
1750         }
1751 }
1752
1753 /*
1754  * Iterate over the object_list and return the first valid object at or after
1755  * the required position with its use_count incremented. The function triggers
1756  * a memory scanning when the pos argument points to the first position.
1757  */
1758 static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1759 {
1760         struct kmemleak_object *object;
1761         loff_t n = *pos;
1762         int err;
1763
1764         err = mutex_lock_interruptible(&scan_mutex);
1765         if (err < 0)
1766                 return ERR_PTR(err);
1767
1768         rcu_read_lock();
1769         list_for_each_entry_rcu(object, &object_list, object_list) {
1770                 if (n-- > 0)
1771                         continue;
1772                 if (get_object(object))
1773                         goto out;
1774         }
1775         object = NULL;
1776 out:
1777         return object;
1778 }
1779
1780 /*
1781  * Return the next object in the object_list. The function decrements the
1782  * use_count of the previous object and increases that of the next one.
1783  */
1784 static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1785 {
1786         struct kmemleak_object *prev_obj = v;
1787         struct kmemleak_object *next_obj = NULL;
1788         struct kmemleak_object *obj = prev_obj;
1789
1790         ++(*pos);
1791
1792         list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1793                 if (get_object(obj)) {
1794                         next_obj = obj;
1795                         break;
1796                 }
1797         }
1798
1799         put_object(prev_obj);
1800         return next_obj;
1801 }
1802
1803 /*
1804  * Decrement the use_count of the last object required, if any.
1805  */
1806 static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1807 {
1808         if (!IS_ERR(v)) {
1809                 /*
1810                  * kmemleak_seq_start may return ERR_PTR if the scan_mutex
1811                  * waiting was interrupted, so only release it if !IS_ERR.
1812                  */
1813                 rcu_read_unlock();
1814                 mutex_unlock(&scan_mutex);
1815                 if (v)
1816                         put_object(v);
1817         }
1818 }
1819
1820 /*
1821  * Print the information for an unreferenced object to the seq file.
1822  */
1823 static int kmemleak_seq_show(struct seq_file *seq, void *v)
1824 {
1825         struct kmemleak_object *object = v;
1826         unsigned long flags;
1827
1828         raw_spin_lock_irqsave(&object->lock, flags);
1829         if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1830                 print_unreferenced(seq, object);
1831         raw_spin_unlock_irqrestore(&object->lock, flags);
1832         return 0;
1833 }
1834
1835 static const struct seq_operations kmemleak_seq_ops = {
1836         .start = kmemleak_seq_start,
1837         .next  = kmemleak_seq_next,
1838         .stop  = kmemleak_seq_stop,
1839         .show  = kmemleak_seq_show,
1840 };
1841
1842 static int kmemleak_open(struct inode *inode, struct file *file)
1843 {
1844         return seq_open(file, &kmemleak_seq_ops);
1845 }
1846
1847 static int dump_str_object_info(const char *str)
1848 {
1849         unsigned long flags;
1850         struct kmemleak_object *object;
1851         unsigned long addr;
1852
1853         if (kstrtoul(str, 0, &addr))
1854                 return -EINVAL;
1855         object = find_and_get_object(addr, 0);
1856         if (!object) {
1857                 pr_info("Unknown object at 0x%08lx\n", addr);
1858                 return -EINVAL;
1859         }
1860
1861         raw_spin_lock_irqsave(&object->lock, flags);
1862         dump_object_info(object);
1863         raw_spin_unlock_irqrestore(&object->lock, flags);
1864
1865         put_object(object);
1866         return 0;
1867 }
1868
1869 /*
1870  * We use grey instead of black to ensure we can do future scans on the same
1871  * objects. If we did not do future scans these black objects could
1872  * potentially contain references to newly allocated objects in the future and
1873  * we'd end up with false positives.
1874  */
1875 static void kmemleak_clear(void)
1876 {
1877         struct kmemleak_object *object;
1878
1879         rcu_read_lock();
1880         list_for_each_entry_rcu(object, &object_list, object_list) {
1881                 raw_spin_lock_irq(&object->lock);
1882                 if ((object->flags & OBJECT_REPORTED) &&
1883                     unreferenced_object(object))
1884                         __paint_it(object, KMEMLEAK_GREY);
1885                 raw_spin_unlock_irq(&object->lock);
1886         }
1887         rcu_read_unlock();
1888
1889         kmemleak_found_leaks = false;
1890 }
1891
1892 static void __kmemleak_do_cleanup(void);
1893
1894 /*
1895  * File write operation to configure kmemleak at run-time. The following
1896  * commands can be written to the /sys/kernel/debug/kmemleak file:
1897  *   off        - disable kmemleak (irreversible)
1898  *   stack=on   - enable the task stacks scanning
1899  *   stack=off  - disable the tasks stacks scanning
1900  *   scan=on    - start the automatic memory scanning thread
1901  *   scan=off   - stop the automatic memory scanning thread
1902  *   scan=...   - set the automatic memory scanning period in seconds (0 to
1903  *                disable it)
1904  *   scan       - trigger a memory scan
1905  *   clear      - mark all current reported unreferenced kmemleak objects as
1906  *                grey to ignore printing them, or free all kmemleak objects
1907  *                if kmemleak has been disabled.
1908  *   dump=...   - dump information about the object found at the given address
1909  */
1910 static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1911                               size_t size, loff_t *ppos)
1912 {
1913         char buf[64];
1914         int buf_size;
1915         int ret;
1916
1917         buf_size = min(size, (sizeof(buf) - 1));
1918         if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1919                 return -EFAULT;
1920         buf[buf_size] = 0;
1921
1922         ret = mutex_lock_interruptible(&scan_mutex);
1923         if (ret < 0)
1924                 return ret;
1925
1926         if (strncmp(buf, "clear", 5) == 0) {
1927                 if (kmemleak_enabled)
1928                         kmemleak_clear();
1929                 else
1930                         __kmemleak_do_cleanup();
1931                 goto out;
1932         }
1933
1934         if (!kmemleak_enabled) {
1935                 ret = -EPERM;
1936                 goto out;
1937         }
1938
1939         if (strncmp(buf, "off", 3) == 0)
1940                 kmemleak_disable();
1941         else if (strncmp(buf, "stack=on", 8) == 0)
1942                 kmemleak_stack_scan = 1;
1943         else if (strncmp(buf, "stack=off", 9) == 0)
1944                 kmemleak_stack_scan = 0;
1945         else if (strncmp(buf, "scan=on", 7) == 0)
1946                 start_scan_thread();
1947         else if (strncmp(buf, "scan=off", 8) == 0)
1948                 stop_scan_thread();
1949         else if (strncmp(buf, "scan=", 5) == 0) {
1950                 unsigned secs;
1951                 unsigned long msecs;
1952
1953                 ret = kstrtouint(buf + 5, 0, &secs);
1954                 if (ret < 0)
1955                         goto out;
1956
1957                 msecs = secs * MSEC_PER_SEC;
1958                 if (msecs > UINT_MAX)
1959                         msecs = UINT_MAX;
1960
1961                 stop_scan_thread();
1962                 if (msecs) {
1963                         WRITE_ONCE(jiffies_scan_wait, msecs_to_jiffies(msecs));
1964                         start_scan_thread();
1965                 }
1966         } else if (strncmp(buf, "scan", 4) == 0)
1967                 kmemleak_scan();
1968         else if (strncmp(buf, "dump=", 5) == 0)
1969                 ret = dump_str_object_info(buf + 5);
1970         else
1971                 ret = -EINVAL;
1972
1973 out:
1974         mutex_unlock(&scan_mutex);
1975         if (ret < 0)
1976                 return ret;
1977
1978         /* ignore the rest of the buffer, only one command at a time */
1979         *ppos += size;
1980         return size;
1981 }
1982
1983 static const struct file_operations kmemleak_fops = {
1984         .owner          = THIS_MODULE,
1985         .open           = kmemleak_open,
1986         .read           = seq_read,
1987         .write          = kmemleak_write,
1988         .llseek         = seq_lseek,
1989         .release        = seq_release,
1990 };
1991
1992 static void __kmemleak_do_cleanup(void)
1993 {
1994         struct kmemleak_object *object, *tmp;
1995
1996         /*
1997          * Kmemleak has already been disabled, no need for RCU list traversal
1998          * or kmemleak_lock held.
1999          */
2000         list_for_each_entry_safe(object, tmp, &object_list, object_list) {
2001                 __remove_object(object);
2002                 __delete_object(object);
2003         }
2004 }
2005
2006 /*
2007  * Stop the memory scanning thread and free the kmemleak internal objects if
2008  * no previous scan thread (otherwise, kmemleak may still have some useful
2009  * information on memory leaks).
2010  */
2011 static void kmemleak_do_cleanup(struct work_struct *work)
2012 {
2013         stop_scan_thread();
2014
2015         mutex_lock(&scan_mutex);
2016         /*
2017          * Once it is made sure that kmemleak_scan has stopped, it is safe to no
2018          * longer track object freeing. Ordering of the scan thread stopping and
2019          * the memory accesses below is guaranteed by the kthread_stop()
2020          * function.
2021          */
2022         kmemleak_free_enabled = 0;
2023         mutex_unlock(&scan_mutex);
2024
2025         if (!kmemleak_found_leaks)
2026                 __kmemleak_do_cleanup();
2027         else
2028                 pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
2029 }
2030
2031 static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
2032
2033 /*
2034  * Disable kmemleak. No memory allocation/freeing will be traced once this
2035  * function is called. Disabling kmemleak is an irreversible operation.
2036  */
2037 static void kmemleak_disable(void)
2038 {
2039         /* atomically check whether it was already invoked */
2040         if (cmpxchg(&kmemleak_error, 0, 1))
2041                 return;
2042
2043         /* stop any memory operation tracing */
2044         kmemleak_enabled = 0;
2045
2046         /* check whether it is too early for a kernel thread */
2047         if (kmemleak_initialized)
2048                 schedule_work(&cleanup_work);
2049         else
2050                 kmemleak_free_enabled = 0;
2051
2052         pr_info("Kernel memory leak detector disabled\n");
2053 }
2054
2055 /*
2056  * Allow boot-time kmemleak disabling (enabled by default).
2057  */
2058 static int __init kmemleak_boot_config(char *str)
2059 {
2060         if (!str)
2061                 return -EINVAL;
2062         if (strcmp(str, "off") == 0)
2063                 kmemleak_disable();
2064         else if (strcmp(str, "on") == 0)
2065                 kmemleak_skip_disable = 1;
2066         else
2067                 return -EINVAL;
2068         return 0;
2069 }
2070 early_param("kmemleak", kmemleak_boot_config);
2071
2072 /*
2073  * Kmemleak initialization.
2074  */
2075 void __init kmemleak_init(void)
2076 {
2077 #ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
2078         if (!kmemleak_skip_disable) {
2079                 kmemleak_disable();
2080                 return;
2081         }
2082 #endif
2083
2084         if (kmemleak_error)
2085                 return;
2086
2087         jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
2088         jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
2089
2090         object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
2091         scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
2092
2093         /* register the data/bss sections */
2094         create_object((unsigned long)_sdata, _edata - _sdata,
2095                       KMEMLEAK_GREY, GFP_ATOMIC);
2096         create_object((unsigned long)__bss_start, __bss_stop - __bss_start,
2097                       KMEMLEAK_GREY, GFP_ATOMIC);
2098         /* only register .data..ro_after_init if not within .data */
2099         if (&__start_ro_after_init < &_sdata || &__end_ro_after_init > &_edata)
2100                 create_object((unsigned long)__start_ro_after_init,
2101                               __end_ro_after_init - __start_ro_after_init,
2102                               KMEMLEAK_GREY, GFP_ATOMIC);
2103 }
2104
2105 /*
2106  * Late initialization function.
2107  */
2108 static int __init kmemleak_late_init(void)
2109 {
2110         kmemleak_initialized = 1;
2111
2112         debugfs_create_file("kmemleak", 0644, NULL, NULL, &kmemleak_fops);
2113
2114         if (kmemleak_error) {
2115                 /*
2116                  * Some error occurred and kmemleak was disabled. There is a
2117                  * small chance that kmemleak_disable() was called immediately
2118                  * after setting kmemleak_initialized and we may end up with
2119                  * two clean-up threads but serialized by scan_mutex.
2120                  */
2121                 schedule_work(&cleanup_work);
2122                 return -ENOMEM;
2123         }
2124
2125         if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
2126                 mutex_lock(&scan_mutex);
2127                 start_scan_thread();
2128                 mutex_unlock(&scan_mutex);
2129         }
2130
2131         pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
2132                 mem_pool_free_count);
2133
2134         return 0;
2135 }
2136 late_initcall(kmemleak_late_init);