slab: propagate tunable values

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 9640edd..692177b 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -31,6 +31,7 @@
 #include <linux/fault-inject.h>
 #include <linux/stacktrace.h>
 #include <linux/prefetch.h>
+#include <linux/memcontrol.h>
 
 #include <trace/events/kmem.h>
 
@@ -1343,6 +1344,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        void *start;
        void *last;
        void *p;
+       int order;
 
        BUG_ON(flags & GFP_SLAB_BUG_MASK);
 
@@ -1351,7 +1353,9 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        if (!page)
                goto out;
 
+       order = compound_order(page);
        inc_slabs_node(s, page_to_nid(page), page->objects);
+       memcg_bind_pages(s, order);
        page->slab_cache = s;
        __SetPageSlab(page);
        if (page->pfmemalloc)
@@ -1360,7 +1364,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        start = page_address(page);
 
        if (unlikely(s->flags & SLAB_POISON))
-               memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
+               memset(start, POISON_INUSE, PAGE_SIZE << order);
 
        last = start;
        for_each_object(p, s, start, page->objects) {
@@ -1401,10 +1405,12 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
 
        __ClearPageSlabPfmemalloc(page);
        __ClearPageSlab(page);
+
+       memcg_release_pages(s, order);
        reset_page_mapcount(page);
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += pages;
-       __free_pages(page, order);
+       __free_memcg_kmem_pages(page, order);
 }
 
 #define need_reserve_slab_rcu                                          \
@@ -2322,6 +2328,7 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s,
        if (slab_pre_alloc_hook(s, gfpflags))
                return NULL;
 
+       s = memcg_kmem_get_cache(s, gfpflags);
 redo:
 
        /*
@@ -2610,19 +2617,10 @@ redo:
 
 void kmem_cache_free(struct kmem_cache *s, void *x)
 {
-       struct page *page;
-
-       page = virt_to_head_page(x);
-
-       if (kmem_cache_debug(s) && page->slab_cache != s) {
-               pr_err("kmem_cache_free: Wrong slab cache. %s but object"
-                       " is from  %s\n", page->slab_cache->name, s->name);
-               WARN_ON_ONCE(1);
+       s = cache_from_obj(s, x);
+       if (!s)
                return;
-       }
-
-       slab_free(s, page, x, _RET_IP_);
-
+       slab_free(s, virt_to_head_page(x), x, _RET_IP_);
        trace_kmem_cache_free(_RET_IP_, x);
 }
 EXPORT_SYMBOL(kmem_cache_free);
@@ -3292,7 +3290,7 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
        struct page *page;
        void *ptr = NULL;
 
-       flags |= __GFP_COMP | __GFP_NOTRACK;
+       flags |= __GFP_COMP | __GFP_NOTRACK | __GFP_KMEMCG;
        page = alloc_pages_node(node, flags, get_order(size));
        if (page)
                ptr = page_address(page);
@@ -3398,7 +3396,7 @@ void kfree(const void *x)
        if (unlikely(!PageSlab(page))) {
                BUG_ON(!PageCompound(page));
                kmemleak_free(x);
-               __free_pages(page, compound_order(page));
+               __free_memcg_kmem_pages(page, compound_order(page));
                return;
        }
        slab_free(page->slab_cache, page, object, _RET_IP_);
@@ -3493,7 +3491,7 @@ static void slab_mem_offline_callback(void *arg)
        struct memory_notify *marg = arg;
        int offline_node;
 
-       offline_node = marg->status_change_nid;
+       offline_node = marg->status_change_nid_normal;
 
        /*
         * If the node still has available memory. we need kmem_cache_node
@@ -3526,7 +3524,7 @@ static int slab_mem_going_online_callback(void *arg)
        struct kmem_cache_node *n;
        struct kmem_cache *s;
        struct memory_notify *marg = arg;
-       int nid = marg->status_change_nid;
+       int nid = marg->status_change_nid_normal;
        int ret = 0;
 
        /*
@@ -3786,7 +3784,7 @@ static int slab_unmergeable(struct kmem_cache *s)
        return 0;
 }
 
-static struct kmem_cache *find_mergeable(size_t size,
+static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
                size_t align, unsigned long flags, const char *name,
                void (*ctor)(void *))
 {
@@ -3822,17 +3820,21 @@ static struct kmem_cache *find_mergeable(size_t size,
                if (s->size - size >= sizeof(void *))
                        continue;
 
+               if (!cache_match_memcg(s, memcg))
+                       continue;
+
                return s;
        }
        return NULL;
 }
 
-struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
-               size_t align, unsigned long flags, void (*ctor)(void *))
+struct kmem_cache *
+__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
+                  size_t align, unsigned long flags, void (*ctor)(void *))
 {
        struct kmem_cache *s;
 
-       s = find_mergeable(size, align, flags, name, ctor);
+       s = find_mergeable(memcg, size, align, flags, name, ctor);
        if (s) {
                s->refcount++;
                /*
@@ -5156,6 +5158,12 @@ static char *create_unique_id(struct kmem_cache *s)
        if (p != name + 1)
                *p++ = '-';
        p += sprintf(p, "%07d", s->size);
+
+#ifdef CONFIG_MEMCG_KMEM
+       if (!is_root_cache(s))
+               p += sprintf(p, "-%08d", memcg_cache_id(s->memcg_params->memcg));
+#endif
+
        BUG_ON(p > name + ID_STR_LENGTH - 1);
        return name;
 }