mm, slub: ensure irqs are enabled for kmemcheck
[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / slub.c
index 2389a01..e517d43 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -16,6 +16,7 @@
 #include <linux/interrupt.h>
 #include <linux/bitops.h>
 #include <linux/slab.h>
+#include "slab.h"
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/kmemcheck.h>
 
 /*
  * Lock order:
- *   1. slub_lock (Global Semaphore)
+ *   1. slab_mutex (Global Mutex)
  *   2. node->list_lock
  *   3. slab_lock(page) (Only on some arches and for debugging)
  *
- *   slub_lock
+ *   slab_mutex
  *
- *   The role of the slub_lock is to protect the list of all the slabs
+ *   The role of the slab_mutex is to protect the list of all the slabs
  *   and to synchronize major metadata changes to slab cache structures.
  *
  *   The slab_lock is only used for debugging and on arches that do not
@@ -182,17 +183,6 @@ static int kmem_size = sizeof(struct kmem_cache);
 static struct notifier_block slab_notifier;
 #endif
 
-static enum {
-       DOWN,           /* No slab functionality available */
-       PARTIAL,        /* Kmem_cache_node works */
-       UP,             /* Everything works but does not show up in sysfs */
-       SYSFS           /* Sysfs up */
-} slab_state = DOWN;
-
-/* A list of all slab caches on the system */
-static DECLARE_RWSEM(slub_lock);
-static LIST_HEAD(slab_caches);
-
 /*
  * Tracking user of a slab.
  */
@@ -237,11 +227,6 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si)
  *                     Core slab cache functions
  *******************************************************************/
 
-int slab_is_available(void)
-{
-       return slab_state >= UP;
-}
-
 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
 {
        return s->node[node];
@@ -311,7 +296,7 @@ static inline size_t slab_ksize(const struct kmem_cache *s)
         * and whatever may come after it.
         */
        if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
-               return s->objsize;
+               return s->object_size;
 
 #endif
        /*
@@ -609,11 +594,11 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
        if (p > addr + 16)
                print_section("Bytes b4 ", p - 16, 16);
 
-       print_section("Object ", p, min_t(unsigned long, s->objsize,
+       print_section("Object ", p, min_t(unsigned long, s->object_size,
                                PAGE_SIZE));
        if (s->flags & SLAB_RED_ZONE)
-               print_section("Redzone ", p + s->objsize,
-                       s->inuse - s->objsize);
+               print_section("Redzone ", p + s->object_size,
+                       s->inuse - s->object_size);
 
        if (s->offset)
                off = s->offset + sizeof(void *);
@@ -655,12 +640,12 @@ static void init_object(struct kmem_cache *s, void *object, u8 val)
        u8 *p = object;
 
        if (s->flags & __OBJECT_POISON) {
-               memset(p, POISON_FREE, s->objsize - 1);
-               p[s->objsize - 1] = POISON_END;
+               memset(p, POISON_FREE, s->object_size - 1);
+               p[s->object_size - 1] = POISON_END;
        }
 
        if (s->flags & SLAB_RED_ZONE)
-               memset(p + s->objsize, val, s->inuse - s->objsize);
+               memset(p + s->object_size, val, s->inuse - s->object_size);
 }
 
 static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
@@ -705,10 +690,10 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
  *     Poisoning uses 0x6b (POISON_FREE) and the last byte is
  *     0xa5 (POISON_END)
  *
- * object + s->objsize
+ * object + s->object_size
  *     Padding to reach word boundary. This is also used for Redzoning.
  *     Padding is extended by another word if Redzoning is enabled and
- *     objsize == inuse.
+ *     object_size == inuse.
  *
  *     We fill with 0xbb (RED_INACTIVE) for inactive objects and with
  *     0xcc (RED_ACTIVE) for objects in use.
@@ -727,7 +712,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
  * object + s->size
  *     Nothing is used beyond s->size.
  *
- * If slabcaches are merged then the objsize and inuse boundaries are mostly
+ * If slabcaches are merged then the object_size and inuse boundaries are mostly
  * ignored. And therefore no slab options that rely on these boundaries
  * may be used with merged slabcaches.
  */
@@ -787,25 +772,25 @@ static int check_object(struct kmem_cache *s, struct page *page,
                                        void *object, u8 val)
 {
        u8 *p = object;
-       u8 *endobject = object + s->objsize;
+       u8 *endobject = object + s->object_size;
 
        if (s->flags & SLAB_RED_ZONE) {
                if (!check_bytes_and_report(s, page, object, "Redzone",
-                       endobject, val, s->inuse - s->objsize))
+                       endobject, val, s->inuse - s->object_size))
                        return 0;
        } else {
-               if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
+               if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
                        check_bytes_and_report(s, page, p, "Alignment padding",
-                               endobject, POISON_INUSE, s->inuse - s->objsize);
+                               endobject, POISON_INUSE, s->inuse - s->object_size);
                }
        }
 
        if (s->flags & SLAB_POISON) {
                if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
                        (!check_bytes_and_report(s, page, p, "Poison", p,
-                                       POISON_FREE, s->objsize - 1) ||
+                                       POISON_FREE, s->object_size - 1) ||
                         !check_bytes_and_report(s, page, p, "Poison",
-                               p + s->objsize - 1, POISON_END, 1)))
+                               p + s->object_size - 1, POISON_END, 1)))
                        return 0;
                /*
                 * check_pad_bytes cleans up on its own.
@@ -926,7 +911,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object,
                        page->freelist);
 
                if (!alloc)
-                       print_section("Object ", (void *)object, s->objsize);
+                       print_section("Object ", (void *)object, s->object_size);
 
                dump_stack();
        }
@@ -942,14 +927,14 @@ static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
        lockdep_trace_alloc(flags);
        might_sleep_if(flags & __GFP_WAIT);
 
-       return should_failslab(s->objsize, flags, s->flags);
+       return should_failslab(s->object_size, flags, s->flags);
 }
 
 static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
 {
        flags &= gfp_allowed_mask;
        kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
-       kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
+       kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
 }
 
 static inline void slab_free_hook(struct kmem_cache *s, void *x)
@@ -966,13 +951,13 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
                unsigned long flags;
 
                local_irq_save(flags);
-               kmemcheck_slab_free(s, x, s->objsize);
-               debug_check_no_locks_freed(x, s->objsize);
+               kmemcheck_slab_free(s, x, s->object_size);
+               debug_check_no_locks_freed(x, s->object_size);
                local_irq_restore(flags);
        }
 #endif
        if (!(s->flags & SLAB_DEBUG_OBJECTS))
-               debug_check_no_obj_freed(x, s->objsize);
+               debug_check_no_obj_freed(x, s->object_size);
 }
 
 /*
@@ -1207,7 +1192,7 @@ out:
 
 __setup("slub_debug", setup_slub_debug);
 
-static unsigned long kmem_cache_flags(unsigned long objsize,
+static unsigned long kmem_cache_flags(unsigned long object_size,
        unsigned long flags, const char *name,
        void (*ctor)(void *))
 {
@@ -1237,7 +1222,7 @@ static inline int check_object(struct kmem_cache *s, struct page *page,
 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
                                        struct page *page) {}
 static inline void remove_full(struct kmem_cache *s, struct page *page) {}
-static inline unsigned long kmem_cache_flags(unsigned long objsize,
+static inline unsigned long kmem_cache_flags(unsigned long object_size,
        unsigned long flags, const char *name,
        void (*ctor)(void *))
 {
@@ -1314,13 +1299,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                        stat(s, ORDER_FALLBACK);
        }
 
-       if (flags & __GFP_WAIT)
-               local_irq_disable();
-
-       if (!page)
-               return NULL;
-
-       if (kmemcheck_enabled
+       if (kmemcheck_enabled && page
                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
                int pages = 1 << oo_order(oo);
 
@@ -1336,6 +1315,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                        kmemcheck_mark_unallocated_pages(page, pages);
        }
 
+       if (flags & __GFP_WAIT)
+               local_irq_disable();
+       if (!page)
+               return NULL;
+
        page->objects = oo_objects(oo);
        mod_zone_page_state(page_zone(page),
                (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -1369,7 +1353,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
 
        inc_slabs_node(s, page_to_nid(page), page->objects);
        page->slab = s;
-       page->flags |= 1 << PG_slab;
+       __SetPageSlab(page);
 
        start = page_address(page);
 
@@ -1513,17 +1497,20 @@ static inline void *acquire_slab(struct kmem_cache *s,
        freelist = page->freelist;
        counters = page->counters;
        new.counters = counters;
-       if (mode)
+       if (mode) {
                new.inuse = page->objects;
+               new.freelist = NULL;
+       } else {
+               new.freelist = freelist;
+       }
 
        VM_BUG_ON(new.frozen);
        new.frozen = 1;
 
        if (!__cmpxchg_double_slab(s, page,
                        freelist, counters,
-                       NULL, new.counters,
+                       new.freelist, new.counters,
                        "acquire_slab"))
-
                return NULL;
 
        remove_partial(n, page);
@@ -1565,7 +1552,6 @@ static void *get_partial_node(struct kmem_cache *s,
                        object = t;
                        available =  page->objects - page->inuse;
                } else {
-                       page->freelist = t;
                        available = put_cpu_partial(s, page, 0);
                        stat(s, CPU_PARTIAL_NODE);
                }
@@ -1580,7 +1566,7 @@ static void *get_partial_node(struct kmem_cache *s,
 /*
  * Get a page from somewhere. Search in increasing NUMA distances.
  */
-static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags,
+static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
                struct kmem_cache_cpu *c)
 {
 #ifdef CONFIG_NUMA
@@ -1615,7 +1601,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags,
 
        do {
                cpuset_mems_cookie = get_mems_allowed();
-               zonelist = node_zonelist(slab_node(current->mempolicy), flags);
+               zonelist = node_zonelist(slab_node(), flags);
                for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
                        struct kmem_cache_node *n;
 
@@ -1870,21 +1856,31 @@ redo:
        }
 }
 
-/* Unfreeze all the cpu partial slabs */
+/*
+ * Unfreeze all the cpu partial slabs.
+ *
+ * This function must be called with interrupt disabled.
+ */
 static void unfreeze_partials(struct kmem_cache *s)
 {
-       struct kmem_cache_node *n = NULL;
+       struct kmem_cache_node *n = NULL, *n2 = NULL;
        struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
        struct page *page, *discard_page = NULL;
 
        while ((page = c->partial)) {
-               enum slab_modes { M_PARTIAL, M_FREE };
-               enum slab_modes l, m;
                struct page new;
                struct page old;
 
                c->partial = page->next;
-               l = M_FREE;
+
+               n2 = get_node(s, page_to_nid(page));
+               if (n != n2) {
+                       if (n)
+                               spin_unlock(&n->list_lock);
+
+                       n = n2;
+                       spin_lock(&n->list_lock);
+               }
 
                do {
 
@@ -1897,43 +1893,17 @@ static void unfreeze_partials(struct kmem_cache *s)
 
                        new.frozen = 0;
 
-                       if (!new.inuse && (!n || n->nr_partial > s->min_partial))
-                               m = M_FREE;
-                       else {
-                               struct kmem_cache_node *n2 = get_node(s,
-                                                       page_to_nid(page));
-
-                               m = M_PARTIAL;
-                               if (n != n2) {
-                                       if (n)
-                                               spin_unlock(&n->list_lock);
-
-                                       n = n2;
-                                       spin_lock(&n->list_lock);
-                               }
-                       }
-
-                       if (l != m) {
-                               if (l == M_PARTIAL) {
-                                       remove_partial(n, page);
-                                       stat(s, FREE_REMOVE_PARTIAL);
-                               } else {
-                                       add_partial(n, page,
-                                               DEACTIVATE_TO_TAIL);
-                                       stat(s, FREE_ADD_PARTIAL);
-                               }
-
-                               l = m;
-                       }
-
-               } while (!cmpxchg_double_slab(s, page,
+               } while (!__cmpxchg_double_slab(s, page,
                                old.freelist, old.counters,
                                new.freelist, new.counters,
                                "unfreezing slab"));
 
-               if (m == M_FREE) {
+               if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) {
                        page->next = discard_page;
                        discard_page = page;
+               } else {
+                       add_partial(n, page, DEACTIVATE_TO_TAIL);
+                       stat(s, FREE_ADD_PARTIAL);
                }
        }
 
@@ -2050,10 +2020,10 @@ static void flush_all(struct kmem_cache *s)
  * Check if the objects in a per cpu structure fit numa
  * locality expectations.
  */
-static inline int node_match(struct kmem_cache_cpu *c, int node)
+static inline int node_match(struct page *page, int node)
 {
 #ifdef CONFIG_NUMA
-       if (node != NUMA_NO_NODE && page_to_nid(c->page) != node)
+       if (node != NUMA_NO_NODE && page_to_nid(page) != node)
                return 0;
 #endif
        return 1;
@@ -2096,10 +2066,10 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
                "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
                nid, gfpflags);
        printk(KERN_WARNING "  cache: %s, object size: %d, buffer size: %d, "
-               "default order: %d, min order: %d\n", s->name, s->objsize,
+               "default order: %d, min order: %d\n", s->name, s->object_size,
                s->size, oo_order(s->oo), oo_order(s->min));
 
-       if (oo_order(s->min) > get_order(s->objsize))
+       if (oo_order(s->min) > get_order(s->object_size))
                printk(KERN_WARNING "  %s debugging increased min order, use "
                       "slub_debug=O to disable.\n", s->name);
 
@@ -2163,6 +2133,8 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
  * The page is still frozen if the return value is not NULL.
  *
  * If this function returns NULL then the page has been unfrozen.
+ *
+ * This function must be called with interrupt disabled.
  */
 static inline void *get_freelist(struct kmem_cache *s, struct page *page)
 {
@@ -2180,7 +2152,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
                new.inuse = page->objects;
                new.frozen = freelist != NULL;
 
-       } while (!cmpxchg_double_slab(s, page,
+       } while (!__cmpxchg_double_slab(s, page,
                freelist, counters,
                NULL, new.counters,
                "get_freelist"));
@@ -2208,6 +2180,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
                          unsigned long addr, struct kmem_cache_cpu *c)
 {
        void *freelist;
+       struct page *page;
        unsigned long flags;
 
        local_irq_save(flags);
@@ -2220,13 +2193,14 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
        c = this_cpu_ptr(s->cpu_slab);
 #endif
 
-       if (!c->page)
+       page = c->page;
+       if (!page)
                goto new_slab;
 redo:
 
-       if (unlikely(!node_match(c, node))) {
+       if (unlikely(!node_match(page, node))) {
                stat(s, ALLOC_NODE_MISMATCH);
-               deactivate_slab(s, c->page, c->freelist);
+               deactivate_slab(s, page, c->freelist);
                c->page = NULL;
                c->freelist = NULL;
                goto new_slab;
@@ -2239,7 +2213,7 @@ redo:
 
        stat(s, ALLOC_SLOWPATH);
 
-       freelist = get_freelist(s, c->page);
+       freelist = get_freelist(s, page);
 
        if (!freelist) {
                c->page = NULL;
@@ -2264,8 +2238,8 @@ load_freelist:
 new_slab:
 
        if (c->partial) {
-               c->page = c->partial;
-               c->partial = c->page->next;
+               page = c->page = c->partial;
+               c->partial = page->next;
                stat(s, CPU_PARTIAL_ALLOC);
                c->freelist = NULL;
                goto redo;
@@ -2281,14 +2255,15 @@ new_slab:
                return NULL;
        }
 
+       page = c->page;
        if (likely(!kmem_cache_debug(s)))
                goto load_freelist;
 
        /* Only entered in the debug case */
-       if (!alloc_debug_processing(s, c->page, freelist, addr))
+       if (!alloc_debug_processing(s, page, freelist, addr))
                goto new_slab;  /* Slab failed checks. Next slab needed */
 
-       deactivate_slab(s, c->page, get_freepointer(s, freelist));
+       deactivate_slab(s, page, get_freepointer(s, freelist));
        c->page = NULL;
        c->freelist = NULL;
        local_irq_restore(flags);
@@ -2310,6 +2285,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
 {
        void **object;
        struct kmem_cache_cpu *c;
+       struct page *page;
        unsigned long tid;
 
        if (slab_pre_alloc_hook(s, gfpflags))
@@ -2335,7 +2311,8 @@ redo:
        barrier();
 
        object = c->freelist;
-       if (unlikely(!object || !node_match(c, node)))
+       page = c->page;
+       if (unlikely(!object || !node_match(page, node)))
 
                object = __slab_alloc(s, gfpflags, node, addr, c);
 
@@ -2367,7 +2344,7 @@ redo:
        }
 
        if (unlikely(gfpflags & __GFP_ZERO) && object)
-               memset(object, 0, s->objsize);
+               memset(object, 0, s->object_size);
 
        slab_post_alloc_hook(s, gfpflags, object);
 
@@ -2378,7 +2355,7 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
 {
        void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
 
-       trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
+       trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags);
 
        return ret;
 }
@@ -2408,7 +2385,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
        void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
 
        trace_kmem_cache_alloc_node(_RET_IP_, ret,
-                                   s->objsize, s->size, gfpflags, node);
+                                   s->object_size, s->size, gfpflags, node);
 
        return ret;
 }
@@ -2772,7 +2749,7 @@ static unsigned long calculate_alignment(unsigned long flags,
 }
 
 static void
-init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
+init_kmem_cache_node(struct kmem_cache_node *n)
 {
        n->nr_partial = 0;
        spin_lock_init(&n->list_lock);
@@ -2842,7 +2819,7 @@ static void early_kmem_cache_node_alloc(int node)
        init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
        init_tracking(kmem_cache_node, n);
 #endif
-       init_kmem_cache_node(n, kmem_cache_node);
+       init_kmem_cache_node(n);
        inc_slabs_node(kmem_cache_node, node, page->objects);
 
        add_partial(n, page, DEACTIVATE_TO_HEAD);
@@ -2882,7 +2859,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)
                }
 
                s->node[node] = n;
-               init_kmem_cache_node(n, s);
+               init_kmem_cache_node(n);
        }
        return 1;
 }
@@ -2903,7 +2880,7 @@ static void set_min_partial(struct kmem_cache *s, unsigned long min)
 static int calculate_sizes(struct kmem_cache *s, int forced_order)
 {
        unsigned long flags = s->flags;
-       unsigned long size = s->objsize;
+       unsigned long size = s->object_size;
        unsigned long align = s->align;
        int order;
 
@@ -2932,7 +2909,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
         * end of the object and the free pointer. If not then add an
         * additional word to have some bytes to store Redzone information.
         */
-       if ((flags & SLAB_RED_ZONE) && size == s->objsize)
+       if ((flags & SLAB_RED_ZONE) && size == s->object_size)
                size += sizeof(void *);
 #endif
 
@@ -2980,7 +2957,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
         * user specified and the dynamic determination of cache line size
         * on bootup.
         */
-       align = calculate_alignment(flags, align, s->objsize);
+       align = calculate_alignment(flags, align, s->object_size);
        s->align = align;
 
        /*
@@ -3028,7 +3005,7 @@ static int kmem_cache_open(struct kmem_cache *s,
        memset(s, 0, kmem_size);
        s->name = name;
        s->ctor = ctor;
-       s->objsize = size;
+       s->object_size = size;
        s->align = align;
        s->flags = kmem_cache_flags(size, flags, name, ctor);
        s->reserved = 0;
@@ -3043,7 +3020,7 @@ static int kmem_cache_open(struct kmem_cache *s,
                 * Disable debugging flags that store metadata if the min slab
                 * order increased.
                 */
-               if (get_order(s->size) > get_order(s->objsize)) {
+               if (get_order(s->size) > get_order(s->object_size)) {
                        s->flags &= ~DEBUG_METADATA_FLAGS;
                        s->offset = 0;
                        if (!calculate_sizes(s, -1))
@@ -3117,7 +3094,7 @@ error:
  */
 unsigned int kmem_cache_size(struct kmem_cache *s)
 {
-       return s->objsize;
+       return s->object_size;
 }
 EXPORT_SYMBOL(kmem_cache_size);
 
@@ -3195,11 +3172,11 @@ static inline int kmem_cache_close(struct kmem_cache *s)
  */
 void kmem_cache_destroy(struct kmem_cache *s)
 {
-       down_write(&slub_lock);
+       mutex_lock(&slab_mutex);
        s->refcount--;
        if (!s->refcount) {
                list_del(&s->list);
-               up_write(&slub_lock);
+               mutex_unlock(&slab_mutex);
                if (kmem_cache_close(s)) {
                        printk(KERN_ERR "SLUB %s: %s called for cache that "
                                "still has objects.\n", s->name, __func__);
@@ -3209,7 +3186,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
                        rcu_barrier();
                sysfs_slab_remove(s);
        } else
-               up_write(&slub_lock);
+               mutex_unlock(&slab_mutex);
 }
 EXPORT_SYMBOL(kmem_cache_destroy);
 
@@ -3271,7 +3248,7 @@ static struct kmem_cache *__init create_kmalloc_cache(const char *name,
 
        /*
         * This function is called with IRQs disabled during early-boot on
-        * single CPU so there's no need to take slub_lock here.
+        * single CPU so there's no need to take slab_mutex here.
         */
        if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
                                                                flags, NULL))
@@ -3556,10 +3533,10 @@ static int slab_mem_going_offline_callback(void *arg)
 {
        struct kmem_cache *s;
 
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list)
                kmem_cache_shrink(s);
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
 
        return 0;
 }
@@ -3580,7 +3557,7 @@ static void slab_mem_offline_callback(void *arg)
        if (offline_node < 0)
                return;
 
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list) {
                n = get_node(s, offline_node);
                if (n) {
@@ -3596,7 +3573,7 @@ static void slab_mem_offline_callback(void *arg)
                        kmem_cache_free(kmem_cache_node, n);
                }
        }
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
 }
 
 static int slab_mem_going_online_callback(void *arg)
@@ -3619,7 +3596,7 @@ static int slab_mem_going_online_callback(void *arg)
         * allocate a kmem_cache_node structure in order to bring the node
         * online.
         */
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list) {
                /*
                 * XXX: kmem_cache_alloc_node will fallback to other nodes
@@ -3631,11 +3608,11 @@ static int slab_mem_going_online_callback(void *arg)
                        ret = -ENOMEM;
                        goto out;
                }
-               init_kmem_cache_node(n, s);
+               init_kmem_cache_node(n);
                s->node[nid] = n;
        }
 out:
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
        return ret;
 }
 
@@ -3846,11 +3823,11 @@ void __init kmem_cache_init(void)
 
                if (s && s->size) {
                        char *name = kasprintf(GFP_NOWAIT,
-                                "dma-kmalloc-%d", s->objsize);
+                                "dma-kmalloc-%d", s->object_size);
 
                        BUG_ON(!name);
                        kmalloc_dma_caches[i] = create_kmalloc_cache(name,
-                               s->objsize, SLAB_CACHE_DMA);
+                               s->object_size, SLAB_CACHE_DMA);
                }
        }
 #endif
@@ -3927,16 +3904,12 @@ static struct kmem_cache *find_mergeable(size_t size,
        return NULL;
 }
 
-struct kmem_cache *kmem_cache_create(const char *name, size_t size,
+struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
                size_t align, unsigned long flags, void (*ctor)(void *))
 {
        struct kmem_cache *s;
        char *n;
 
-       if (WARN_ON(!name))
-               return NULL;
-
-       down_write(&slub_lock);
        s = find_mergeable(size, align, flags, name, ctor);
        if (s) {
                s->refcount++;
@@ -3944,49 +3917,42 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
                 * Adjust the object sizes so that we clear
                 * the complete object on kzalloc.
                 */
-               s->objsize = max(s->objsize, (int)size);
+               s->object_size = max(s->object_size, (int)size);
                s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
 
                if (sysfs_slab_alias(s, name)) {
                        s->refcount--;
-                       goto err;
+                       return NULL;
                }
-               up_write(&slub_lock);
                return s;
        }
 
        n = kstrdup(name, GFP_KERNEL);
        if (!n)
-               goto err;
+               return NULL;
 
        s = kmalloc(kmem_size, GFP_KERNEL);
        if (s) {
                if (kmem_cache_open(s, n,
                                size, align, flags, ctor)) {
+                       int r;
+
                        list_add(&s->list, &slab_caches);
-                       up_write(&slub_lock);
-                       if (sysfs_slab_add(s)) {
-                               down_write(&slub_lock);
-                               list_del(&s->list);
-                               kfree(n);
-                               kfree(s);
-                               goto err;
-                       }
-                       return s;
+                       mutex_unlock(&slab_mutex);
+                       r = sysfs_slab_add(s);
+                       mutex_lock(&slab_mutex);
+
+                       if (!r)
+                               return s;
+
+                       list_del(&s->list);
+                       kmem_cache_close(s);
                }
-               kfree(n);
                kfree(s);
        }
-err:
-       up_write(&slub_lock);
-
-       if (flags & SLAB_PANIC)
-               panic("Cannot create slabcache %s\n", name);
-       else
-               s = NULL;
-       return s;
+       kfree(n);
+       return NULL;
 }
-EXPORT_SYMBOL(kmem_cache_create);
 
 #ifdef CONFIG_SMP
 /*
@@ -4005,13 +3971,13 @@ static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
        case CPU_UP_CANCELED_FROZEN:
        case CPU_DEAD:
        case CPU_DEAD_FROZEN:
-               down_read(&slub_lock);
+               mutex_lock(&slab_mutex);
                list_for_each_entry(s, &slab_caches, list) {
                        local_irq_save(flags);
                        __flush_cpu_slab(s, cpu);
                        local_irq_restore(flags);
                }
-               up_read(&slub_lock);
+               mutex_unlock(&slab_mutex);
                break;
        default:
                break;
@@ -4627,7 +4593,7 @@ SLAB_ATTR_RO(align);
 
 static ssize_t object_size_show(struct kmem_cache *s, char *buf)
 {
-       return sprintf(buf, "%d\n", s->objsize);
+       return sprintf(buf, "%d\n", s->object_size);
 }
 SLAB_ATTR_RO(object_size);
 
@@ -5290,7 +5256,7 @@ static int sysfs_slab_add(struct kmem_cache *s)
        const char *name;
        int unmergeable;
 
-       if (slab_state < SYSFS)
+       if (slab_state < FULL)
                /* Defer until later */
                return 0;
 
@@ -5335,7 +5301,7 @@ static int sysfs_slab_add(struct kmem_cache *s)
 
 static void sysfs_slab_remove(struct kmem_cache *s)
 {
-       if (slab_state < SYSFS)
+       if (slab_state < FULL)
                /*
                 * Sysfs has not been setup yet so no need to remove the
                 * cache from sysfs.
@@ -5363,7 +5329,7 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
 {
        struct saved_alias *al;
 
-       if (slab_state == SYSFS) {
+       if (slab_state == FULL) {
                /*
                 * If we have a leftover link then remove it.
                 */
@@ -5387,16 +5353,16 @@ static int __init slab_sysfs_init(void)
        struct kmem_cache *s;
        int err;
 
-       down_write(&slub_lock);
+       mutex_lock(&slab_mutex);
 
        slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
        if (!slab_kset) {
-               up_write(&slub_lock);
+               mutex_unlock(&slab_mutex);
                printk(KERN_ERR "Cannot register slab subsystem.\n");
                return -ENOSYS;
        }
 
-       slab_state = SYSFS;
+       slab_state = FULL;
 
        list_for_each_entry(s, &slab_caches, list) {
                err = sysfs_slab_add(s);
@@ -5412,11 +5378,11 @@ static int __init slab_sysfs_init(void)
                err = sysfs_slab_alias(al->s, al->name);
                if (err)
                        printk(KERN_ERR "SLUB: Unable to add boot slab alias"
-                                       " %s to sysfs\n", s->name);
+                                       " %s to sysfs\n", al->name);
                kfree(al);
        }
 
-       up_write(&slub_lock);
+       mutex_unlock(&slab_mutex);
        resiliency_test();
        return 0;
 }
@@ -5431,7 +5397,7 @@ __initcall(slab_sysfs_init);
 static void print_slabinfo_header(struct seq_file *m)
 {
        seq_puts(m, "slabinfo - version: 2.1\n");
-       seq_puts(m, "# name            <active_objs> <num_objs> <objsize> "
+       seq_puts(m, "# name            <active_objs> <num_objs> <object_size> "
                 "<objperslab> <pagesperslab>");
        seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
        seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
@@ -5442,7 +5408,7 @@ static void *s_start(struct seq_file *m, loff_t *pos)
 {
        loff_t n = *pos;
 
-       down_read(&slub_lock);
+       mutex_lock(&slab_mutex);
        if (!n)
                print_slabinfo_header(m);
 
@@ -5456,7 +5422,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos)
 
 static void s_stop(struct seq_file *m, void *p)
 {
-       up_read(&slub_lock);
+       mutex_unlock(&slab_mutex);
 }
 
 static int s_show(struct seq_file *m, void *p)