mm, slub: ensure irqs are enabled for kmemcheck

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 4c38516..e517d43 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -36,13 +36,13 @@
 
 /*
  * Lock order:
- *   1. slub_lock (Global Semaphore)
+ *   1. slab_mutex (Global Mutex)
  *   2. node->list_lock
  *   3. slab_lock(page) (Only on some arches and for debugging)
  *
- *   slub_lock
+ *   slab_mutex
  *
- *   The role of the slub_lock is to protect the list of all the slabs
+ *   The role of the slab_mutex is to protect the list of all the slabs
  *   and to synchronize major metadata changes to slab cache structures.
  *
  *   The slab_lock is only used for debugging and on arches that do not
@@ -183,10 +183,6 @@ static int kmem_size = sizeof(struct kmem_cache);
 static struct notifier_block slab_notifier;
 #endif
 
-/* A list of all slab caches on the system */
-static DECLARE_RWSEM(slub_lock);
-static LIST_HEAD(slab_caches);
-
 /*
  * Tracking user of a slab.
  */
@@ -1303,13 +1299,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                        stat(s, ORDER_FALLBACK);
        }
 
-       if (flags & __GFP_WAIT)
-               local_irq_disable();
-
-       if (!page)
-               return NULL;
-
-       if (kmemcheck_enabled
+       if (kmemcheck_enabled && page
                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
                int pages = 1 << oo_order(oo);
 
@@ -1325,6 +1315,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                        kmemcheck_mark_unallocated_pages(page, pages);
        }
 
+       if (flags & __GFP_WAIT)
+               local_irq_disable();
+       if (!page)
+               return NULL;
+
        page->objects = oo_objects(oo);
        mod_zone_page_state(page_zone(page),
                (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -3177,11 +3172,11 @@ static inline int kmem_cache_close(struct kmem_cache *s)
  */
 void kmem_cache_destroy(struct kmem_cache *s)
 {
-       down_write(&slub_lock);
+       mutex_lock(&slab_mutex);
        s->refcount--;
        if (!s->refcount) {
                list_del(&s->list);
-               up_write(&slub_lock);
+               mutex_unlock(&slab_mutex);
                if (kmem_cache_close(s)) {
                        printk(KERN_ERR "SLUB %s: %s called for cache that "
                                "still has objects.\n", s->name, __func__);
@@ -3191,7 +3186,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
                        rcu_barrier();
                sysfs_slab_remove(s);
        } else
-               up_write(&slub_lock);
+               mutex_unlock(&slab_mutex);
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
 
@@ -3253,7 +3248,7 @@ static struct kmem_cache *__init create_kmalloc_cache(const char *name,
 
        /*
         * This function is called with IRQs disabled during early-boot on
-        * single CPU so there's no need to take slub_lock here.
+        * single CPU so there's no need to take slab_mutex here.
         */
        if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
                                                                flags, NULL))
@@ -3538,10 +3533,10 @@ static int slab_mem_going_offline_callback(void *arg)
 {
        struct kmem_cache *s;
 
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list)
                kmem_cache_shrink(s);
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
 
        return 0;
 }
@@ -3562,7 +3557,7 @@ static void slab_mem_offline_callback(void *arg)
        if (offline_node < 0)
                return;
 
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list) {
                n = get_node(s, offline_node);
                if (n) {
@@ -3578,7 +3573,7 @@ static void slab_mem_offline_callback(void *arg)
                        kmem_cache_free(kmem_cache_node, n);
                }
        }
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
 }
 
 static int slab_mem_going_online_callback(void *arg)
@@ -3601,7 +3596,7 @@ static int slab_mem_going_online_callback(void *arg)
         * allocate a kmem_cache_node structure in order to bring the node
         * online.
         */
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list) {
                /*
                 * XXX: kmem_cache_alloc_node will fallback to other nodes
@@ -3617,7 +3612,7 @@ static int slab_mem_going_online_callback(void *arg)
                s->node[nid] = n;
        }
 out:
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
        return ret;
 }
 
@@ -3915,7 +3910,6 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
        struct kmem_cache *s;
        char *n;
 
-       down_write(&slub_lock);
        s = find_mergeable(size, align, flags, name, ctor);
        if (s) {
                s->refcount++;
@@ -3928,37 +3922,36 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
 
                if (sysfs_slab_alias(s, name)) {
                        s->refcount--;
-                       goto err;
+                       return NULL;
                }
-               up_write(&slub_lock);
                return s;
        }
 
        n = kstrdup(name, GFP_KERNEL);
        if (!n)
-               goto err;
+               return NULL;
 
        s = kmalloc(kmem_size, GFP_KERNEL);
        if (s) {
                if (kmem_cache_open(s, n,
                                size, align, flags, ctor)) {
+                       int r;
+
                        list_add(&s->list, &slab_caches);
-                       up_write(&slub_lock);
-                       if (sysfs_slab_add(s)) {
-                               down_write(&slub_lock);
-                               list_del(&s->list);
-                               kfree(n);
-                               kfree(s);
-                               goto err;
-                       }
-                       return s;
+                       mutex_unlock(&slab_mutex);
+                       r = sysfs_slab_add(s);
+                       mutex_lock(&slab_mutex);
+
+                       if (!r)
+                               return s;
+
+                       list_del(&s->list);
+                       kmem_cache_close(s);
                }
                kfree(s);
        }
        kfree(n);
-err:
-       up_write(&slub_lock);
-       return s;
+       return NULL;
 }
 
 #ifdef CONFIG_SMP
@@ -3978,13 +3971,13 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
        case CPU_UP_CANCELED_FROZEN:
        case CPU_DEAD:
        case CPU_DEAD_FROZEN:
-               down_read(&slub_lock);
+               mutex_lock(&slab_mutex);
                list_for_each_entry(s, &slab_caches, list) {
                        local_irq_save(flags);
                        __flush_cpu_slab(s, cpu);
                        local_irq_restore(flags);
                }
-               up_read(&slub_lock);
+               mutex_unlock(&slab_mutex);
                break;
        default:
                break;
@@ -5360,11 +5353,11 @@ static int __init slab_sysfs_init(void)
        struct kmem_cache *s;
        int err;
 
-       down_write(&slub_lock);
+       mutex_lock(&slab_mutex);
 
        slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
        if (!slab_kset) {
-               up_write(&slub_lock);
+               mutex_unlock(&slab_mutex);
                printk(KERN_ERR "Cannot register slab subsystem.\n");
                return -ENOSYS;
        }
@@ -5389,7 +5382,7 @@ static int __init slab_sysfs_init(void)
                kfree(al);
        }
 
-       up_write(&slub_lock);
+       mutex_unlock(&slab_mutex);
        resiliency_test();
        return 0;
 }
@@ -5415,7 +5408,7 @@ static void *s_start(struct seq_file *m, loff_t *pos)
 {
        loff_t n = *pos;
 
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        if (!n)
                print_slabinfo_header(m);
 
@@ -5429,7 +5422,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos)
 
 static void s_stop(struct seq_file *m, void *p)
 {
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
 }
 
 static int s_show(struct seq_file *m, void *p)