x86: add tizen_qemu_x86_defconfig & tizen_qemu_x86_64_defconfig

[platform/kernel/linux-rpi.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 3d2025f..f95ae13 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -308,6 +308,11 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si)
  */
 static nodemask_t slab_nodes;
 
+/*
+ * Workqueue used for flush_cpu_slab().
+ */
+static struct workqueue_struct *flushwq;
+
 /********************************************************************
  *                     Core slab cache functions
  *******************************************************************/
@@ -1701,7 +1706,8 @@ static __always_inline bool slab_free_hook(struct kmem_cache *s,
 }
 
 static inline bool slab_free_freelist_hook(struct kmem_cache *s,
-                                          void **head, void **tail)
+                                          void **head, void **tail,
+                                          int *cnt)
 {
 
        void *object;
@@ -1728,6 +1734,12 @@ static inline bool slab_free_freelist_hook(struct kmem_cache *s,
                        *head = object;
                        if (!*tail)
                                *tail = object;
+               } else {
+                       /*
+                        * Adjust the reconstructed freelist depth
+                        * accordingly if object's reuse is delayed.
+                        */
+                       --(*cnt);
                }
        } while (object != old_tail);
 
@@ -2681,7 +2693,7 @@ static void flush_all_cpus_locked(struct kmem_cache *s)
                INIT_WORK(&sfw->work, flush_cpu_slab);
                sfw->skip = false;
                sfw->s = s;
-               schedule_work_on(cpu, &sfw->work);
+               queue_work_on(cpu, flushwq, &sfw->work);
        }
 
        for_each_online_cpu(cpu) {
@@ -2928,6 +2940,7 @@ redo:
 
        if (!freelist) {
                c->page = NULL;
+               c->tid = next_tid(c->tid);
                local_unlock_irqrestore(&s->cpu_slab->lock, flags);
                stat(s, DEACTIVATE_BYPASS);
                goto new_slab;
@@ -2960,6 +2973,7 @@ deactivate_slab:
        freelist = c->freelist;
        c->page = NULL;
        c->freelist = NULL;
+       c->tid = next_tid(c->tid);
        local_unlock_irqrestore(&s->cpu_slab->lock, flags);
        deactivate_slab(s, page, freelist);
 
@@ -3413,7 +3427,9 @@ static __always_inline void do_slab_free(struct kmem_cache *s,
        struct kmem_cache_cpu *c;
        unsigned long tid;
 
-       memcg_slab_free_hook(s, &head, 1);
+       /* memcg_slab_free_hook() is already called for bulk free. */
+       if (!tail)
+               memcg_slab_free_hook(s, &head, 1);
 redo:
        /*
         * Determine the currently cpus per cpu slab.
@@ -3480,7 +3496,7 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
         * With KASAN enabled slab_free_freelist_hook modifies the freelist
         * to remove objects, whose reuse must be delayed.
         */
-       if (slab_free_freelist_hook(s, &head, &tail))
+       if (slab_free_freelist_hook(s, &head, &tail, &cnt))
                do_slab_free(s, page, head, tail, cnt, addr);
 }
 
@@ -4203,8 +4219,8 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
        if (alloc_kmem_cache_cpus(s))
                return 0;
 
-       free_kmem_cache_nodes(s);
 error:
+       __kmem_cache_release(s);
        return -EINVAL;
 }
 
@@ -4290,7 +4306,7 @@ int __kmem_cache_shutdown(struct kmem_cache *s)
 }
 
 #ifdef CONFIG_PRINTK
-void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
+void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page)
 {
        void *base;
        int __maybe_unused i;
@@ -4839,6 +4855,8 @@ void __init kmem_cache_init(void)
 
 void __init kmem_cache_init_late(void)
 {
+       flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0);
+       WARN_ON(!flushwq);
 }
 
 struct kmem_cache *
@@ -4880,13 +4898,15 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
                return 0;
 
        err = sysfs_slab_add(s);
-       if (err)
+       if (err) {
                __kmem_cache_release(s);
+               return err;
+       }
 
        if (s->flags & SLAB_STORE_USER)
                debugfs_slab_add(s);
 
-       return err;
+       return 0;
 }
 
 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
@@ -4907,6 +4927,8 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
        /* Honor the call site pointer we received. */
        trace_kmalloc(caller, ret, size, s->size, gfpflags);
 
+       ret = kasan_kmalloc(s, ret, size, gfpflags);
+
        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_track_caller);
@@ -4938,6 +4960,8 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
        /* Honor the call site pointer we received. */
        trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
 
+       ret = kasan_kmalloc(s, ret, size, gfpflags);
+
        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node_track_caller);
@@ -5061,6 +5085,7 @@ struct loc_track {
        unsigned long max;
        unsigned long count;
        struct location *loc;
+       loff_t idx;
 };
 
 static struct dentry *slab_debugfs_root;
@@ -5851,7 +5876,8 @@ static char *create_unique_id(struct kmem_cache *s)
        char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
        char *p = name;
 
-       BUG_ON(!name);
+       if (!name)
+               return ERR_PTR(-ENOMEM);
 
        *p++ = ':';
        /*
@@ -5909,6 +5935,8 @@ static int sysfs_slab_add(struct kmem_cache *s)
                 * for the symlinks.
                 */
                name = create_unique_id(s);
+               if (IS_ERR(name))
+                       return PTR_ERR(name);
        }
 
        s->kobj.kset = kset;
@@ -6024,11 +6052,11 @@ __initcall(slab_sysfs_init);
 #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
 static int slab_debugfs_show(struct seq_file *seq, void *v)
 {
-
-       struct location *l;
-       unsigned int idx = *(unsigned int *)v;
        struct loc_track *t = seq->private;
+       struct location *l;
+       unsigned long idx;
 
+       idx = (unsigned long) t->idx;
        if (idx < t->count) {
                l = &t->loc[idx];
 
@@ -6077,16 +6105,18 @@ static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
 {
        struct loc_track *t = seq->private;
 
-       v = ppos;
-       ++*ppos;
+       t->idx = ++(*ppos);
        if (*ppos <= t->count)
-               return v;
+               return ppos;
 
        return NULL;
 }
 
 static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
 {
+       struct loc_track *t = seq->private;
+
+       t->idx = *ppos;
        return ppos;
 }
 
@@ -6108,9 +6138,14 @@ static int slab_debug_trace_open(struct inode *inode, struct file *filep)
        struct kmem_cache *s = file_inode(filep)->i_private;
        unsigned long *obj_map;
 
+       if (!t)
+               return -ENOMEM;
+
        obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
-       if (!obj_map)
+       if (!obj_map) {
+               seq_release_private(inode, filep);
                return -ENOMEM;
+       }
 
        if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
                alloc = TRACK_ALLOC;
@@ -6119,6 +6154,7 @@ static int slab_debug_trace_open(struct inode *inode, struct file *filep)
 
        if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
                bitmap_free(obj_map);
+               seq_release_private(inode, filep);
                return -ENOMEM;
        }