slub: Take node lock during object free checks

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index c3f05e1..e131084 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1069,13 +1069,13 @@ bad:
        return 0;
 }
 
-static noinline int free_debug_processing(struct kmem_cache *s,
-                struct page *page, void *object, unsigned long addr)
+static noinline struct kmem_cache_node *free_debug_processing(
+       struct kmem_cache *s, struct page *page, void *object,
+       unsigned long addr, unsigned long *flags)
 {
-       unsigned long flags;
-       int rc = 0;
+       struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
-       local_irq_save(flags);
+       spin_lock_irqsave(&n->list_lock, *flags);
        slab_lock(page);
 
        if (!check_slab(s, page))
@@ -1113,15 +1113,19 @@ static noinline int free_debug_processing(struct kmem_cache *s,
                set_track(s, object, TRACK_FREE, addr);
        trace(s, page, object, 0);
        init_object(s, object, SLUB_RED_INACTIVE);
-       rc = 1;
 out:
        slab_unlock(page);
-       local_irq_restore(flags);
-       return rc;
+       /*
+        * Keep node_lock to preserve integrity
+        * until the object is actually freed
+        */
+       return n;
 
 fail:
+       slab_unlock(page);
+       spin_unlock_irqrestore(&n->list_lock, *flags);
        slab_fix(s, "Object at 0x%p not freed", object);
-       goto out;
+       return NULL;
 }
 
 static int __init setup_slub_debug(char *str)
@@ -1214,8 +1218,9 @@ static inline void setup_object_debug(struct kmem_cache *s,
 static inline int alloc_debug_processing(struct kmem_cache *s,
        struct page *page, void *object, unsigned long addr) { return 0; }
 
-static inline int free_debug_processing(struct kmem_cache *s,
-       struct page *page, void *object, unsigned long addr) { return 0; }
+static inline struct kmem_cache_node *free_debug_processing(
+       struct kmem_cache *s, struct page *page, void *object,
+       unsigned long addr, unsigned long *flags) { return NULL; }
 
 static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
                        { return 1; }
@@ -2281,11 +2286,11 @@ new_slab:
        }
 
        page = c->page;
-       if (likely(!kmem_cache_debug(s)))
+       if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
                goto load_freelist;
 
        /* Only entered in the debug case */
-       if (!alloc_debug_processing(s, page, freelist, addr))
+       if (kmem_cache_debug(s) && !alloc_debug_processing(s, page, freelist, addr))
                goto new_slab;  /* Slab failed checks. Next slab needed */
 
        deactivate_slab(s, page, get_freepointer(s, freelist));
@@ -2337,8 +2342,7 @@ redo:
 
        object = c->freelist;
        page = c->page;
-       if (unlikely(!object || !node_match(page, node) ||
-                                       !pfmemalloc_match(page, gfpflags)))
+       if (unlikely(!object || !node_match(page, node)))
                object = __slab_alloc(s, gfpflags, node, addr, c);
 
        else {
@@ -2453,7 +2457,8 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 
        stat(s, FREE_SLOWPATH);
 
-       if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
+       if (kmem_cache_debug(s) &&
+               !(n = free_debug_processing(s, page, x, addr, &flags)))
                return;
 
        do {
@@ -3478,7 +3483,7 @@ void kfree(const void *x)
        if (unlikely(!PageSlab(page))) {
                BUG_ON(!PageCompound(page));
                kmemleak_free(x);
-               put_page(page);
+               __free_pages(page, compound_order(page));
                return;
        }
        slab_free(page->slab, page, object, _RET_IP_);