slab: introduce __kmemcg_cache_deactivate()

__kmem_cache_shrink() is called with %true @deactivate only for memcg
caches.  Remove @deactivate from __kmem_cache_shrink() and introduce
__kmemcg_cache_deactivate() instead.  Each memcg-supporting allocator
should implement it and it should deactivate and drain the cache.

This is to allow memcg cache deactivation behavior to further deviate
from simple shrinking without messing up __kmem_cache_shrink().

This is pure reorganization and doesn't introduce any observable
behavior changes.

v2: Dropped unnecessary ifdef in mm/slab.h as suggested by Vladimir.

Link: http://lkml.kernel.org/r/20170117235411.9408-8-tj@kernel.org
Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/slab.c b/mm/slab.c
index 8a0e339..bd63450 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -2315,7 +2315,7 @@ out:
        return nr_freed;
 }
 
-int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
+int __kmem_cache_shrink(struct kmem_cache *cachep)
 {
        int ret = 0;
        int node;
@@ -2333,9 +2333,16 @@ int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
        return (ret ? 1 : 0);
 }
 
+#ifdef CONFIG_MEMCG
+void __kmemcg_cache_deactivate(struct kmem_cache *cachep)
+{
+       __kmem_cache_shrink(cachep);
+}
+#endif
+
 int __kmem_cache_shutdown(struct kmem_cache *cachep)
 {
-       return __kmem_cache_shrink(cachep, false);
+       return __kmem_cache_shrink(cachep);
 }
 
 void __kmem_cache_release(struct kmem_cache *cachep)

diff --git a/mm/slab.h b/mm/slab.h
index 9631bb2..7bff1ee 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -167,7 +167,8 @@ static inline unsigned long kmem_cache_flags(unsigned long object_size,
 
 int __kmem_cache_shutdown(struct kmem_cache *);
 void __kmem_cache_release(struct kmem_cache *);
-int __kmem_cache_shrink(struct kmem_cache *, bool);
+int __kmem_cache_shrink(struct kmem_cache *);
+void __kmemcg_cache_deactivate(struct kmem_cache *s);
 void slab_kmem_cache_release(struct kmem_cache *);
 
 struct seq_file;

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 274697e..59e41bb 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -646,7 +646,7 @@ void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
                if (!c)
                        continue;
 
-               __kmem_cache_shrink(c, true);
+               __kmemcg_cache_deactivate(c);
                arr->entries[idx] = NULL;
        }
        mutex_unlock(&slab_mutex);
@@ -794,7 +794,7 @@ int kmem_cache_shrink(struct kmem_cache *cachep)
        get_online_cpus();
        get_online_mems();
        kasan_cache_shrink(cachep);
-       ret = __kmem_cache_shrink(cachep, false);
+       ret = __kmem_cache_shrink(cachep);
        put_online_mems();
        put_online_cpus();
        return ret;

diff --git a/mm/slob.c b/mm/slob.c
index 5ec1580..eac04d4 100644 (file)
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -634,7 +634,7 @@ void __kmem_cache_release(struct kmem_cache *c)
 {
 }
 
-int __kmem_cache_shrink(struct kmem_cache *d, bool deactivate)
+int __kmem_cache_shrink(struct kmem_cache *d)
 {
        return 0;
 }

diff --git a/mm/slub.c b/mm/slub.c
index 03b012b..8a45915 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3894,7 +3894,7 @@ EXPORT_SYMBOL(kfree);
  * being allocated from last increasing the chance that the last objects
  * are freed in them.
  */
-int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
+int __kmem_cache_shrink(struct kmem_cache *s)
 {
        int node;
        int i;
@@ -3906,21 +3906,6 @@ int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
        unsigned long flags;
        int ret = 0;
 
-       if (deactivate) {
-               /*
-                * Disable empty slabs caching. Used to avoid pinning offline
-                * memory cgroups by kmem pages that can be freed.
-                */
-               s->cpu_partial = 0;
-               s->min_partial = 0;
-
-               /*
-                * s->cpu_partial is checked locklessly (see put_cpu_partial),
-                * so we have to make sure the change is visible.
-                */
-               synchronize_sched();
-       }
-
        flush_all(s);
        for_each_kmem_cache_node(s, node, n) {
                INIT_LIST_HEAD(&discard);
@@ -3971,13 +3956,33 @@ int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
        return ret;
 }
 
+#ifdef CONFIG_MEMCG
+void __kmemcg_cache_deactivate(struct kmem_cache *s)
+{
+       /*
+        * Disable empty slabs caching. Used to avoid pinning offline
+        * memory cgroups by kmem pages that can be freed.
+        */
+       s->cpu_partial = 0;
+       s->min_partial = 0;
+
+       /*
+        * s->cpu_partial is checked locklessly (see put_cpu_partial), so
+        * we have to make sure the change is visible.
+        */
+       synchronize_sched();
+
+       __kmem_cache_shrink(s);
+}
+#endif
+
 static int slab_mem_going_offline_callback(void *arg)
 {
        struct kmem_cache *s;
 
        mutex_lock(&slab_mutex);
        list_for_each_entry(s, &slab_caches, list)
-               __kmem_cache_shrink(s, false);
+               __kmem_cache_shrink(s);
        mutex_unlock(&slab_mutex);
 
        return 0;