mm: slab: optimize memcg_slab_free_hook()

Most callers of memcg_slab_free_hook() already know the slab,  which could
be passed to memcg_slab_free_hook() directly to reduce the overhead of an
another call of virt_to_slab().  For bulk freeing of objects, the call of
slab_objcgs() in the loop in memcg_slab_free_hook() is redundant as well.
Rework memcg_slab_free_hook() and build_detached_freelist() to reduce
those unnecessary overhead and make memcg_slab_free_hook() can handle bulk
freeing in slab_free().

Move the calling site of memcg_slab_free_hook() from do_slab_free() to
slab_free() for slub to make the code clearer since the logic is weird
(e.g. the caller need to judge whether it needs to call
memcg_slab_free_hook()). It is easy to make mistakes like missing calling
of memcg_slab_free_hook() like fixes of:

  commit d1b2cf6cb84a ("mm: memcg/slab: uncharge during kmem_cache_free_bulk()")
  commit ae085d7f9365 ("mm: kfence: fix missing objcg housekeeping for SLAB")

This optimization is mainly for bulk objects freeing.  The following numbers
is shown for 16-object freeing.

                           before      after
  kmem_cache_free_bulk:   ~430 ns     ~400 ns

The overhead is reduced by about 7% for 16-object freeing.

Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Link: https://lore.kernel.org/r/20220429123044.37885-1-songmuchun@bytedance.com
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/slab.c b/mm/slab.c
index f7d6ca7..764cbad 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3406,9 +3406,10 @@ static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
 {
        bool init;
 
+       memcg_slab_free_hook(cachep, virt_to_slab(objp), &objp, 1);
+
        if (is_kfence_address(objp)) {
                kmemleak_free_recursive(objp, cachep->flags);
-               memcg_slab_free_hook(cachep, &objp, 1);
                __kfence_free(objp);
                return;
        }
@@ -3441,7 +3442,6 @@ void ___cache_free(struct kmem_cache *cachep, void *objp,
        check_irq_off();
        kmemleak_free_recursive(objp, cachep->flags);
        objp = cache_free_debugcheck(cachep, objp, caller);
-       memcg_slab_free_hook(cachep, &objp, 1);
 
        /*
         * Skip calling cache_free_alien() when the platform is not numa.

diff --git a/mm/slab.h b/mm/slab.h
index db9fb5c..a8d5eb1 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -547,36 +547,22 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
        obj_cgroup_put(objcg);
 }
 
-static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,
+static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
                                        void **p, int objects)
 {
-       struct kmem_cache *s;
        struct obj_cgroup **objcgs;
-       struct obj_cgroup *objcg;
-       struct slab *slab;
-       unsigned int off;
        int i;
 
        if (!memcg_kmem_enabled())
                return;
 
-       for (i = 0; i < objects; i++) {
-               if (unlikely(!p[i]))
-                       continue;
-
-               slab = virt_to_slab(p[i]);
-               /* we could be given a kmalloc_large() object, skip those */
-               if (!slab)
-                       continue;
-
-               objcgs = slab_objcgs(slab);
-               if (!objcgs)
-                       continue;
+       objcgs = slab_objcgs(slab);
+       if (!objcgs)
+               return;
 
-               if (!s_orig)
-                       s = slab->slab_cache;
-               else
-                       s = s_orig;
+       for (i = 0; i < objects; i++) {
+               struct obj_cgroup *objcg;
+               unsigned int off;
 
                off = obj_to_index(s, slab, p[i]);
                objcg = objcgs[off];
@@ -628,7 +614,7 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
 {
 }
 
-static inline void memcg_slab_free_hook(struct kmem_cache *s,
+static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
                                        void **p, int objects)
 {
 }

diff --git a/mm/slub.c b/mm/slub.c
index 514741c..26b0095 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3464,9 +3464,6 @@ static __always_inline void do_slab_free(struct kmem_cache *s,
        struct kmem_cache_cpu *c;
        unsigned long tid;
 
-       /* memcg_slab_free_hook() is already called for bulk free. */
-       if (!tail)
-               memcg_slab_free_hook(s, &head, 1);
 redo:
        /*
         * Determine the currently cpus per cpu slab.
@@ -3526,9 +3523,10 @@ redo:
 }
 
 static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
-                                     void *head, void *tail, int cnt,
+                                     void *head, void *tail, void **p, int cnt,
                                      unsigned long addr)
 {
+       memcg_slab_free_hook(s, slab, p, cnt);
        /*
         * With KASAN enabled slab_free_freelist_hook modifies the freelist
         * to remove objects, whose reuse must be delayed.
@@ -3550,7 +3548,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
        if (!s)
                return;
        trace_kmem_cache_free(_RET_IP_, x, s->name);
-       slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_);
+       slab_free(s, virt_to_slab(x), x, NULL, &x, 1, _RET_IP_);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -3591,79 +3589,59 @@ static inline
 int build_detached_freelist(struct kmem_cache *s, size_t size,
                            void **p, struct detached_freelist *df)
 {
-       size_t first_skipped_index = 0;
        int lookahead = 3;
        void *object;
        struct folio *folio;
-       struct slab *slab;
-
-       /* Always re-init detached_freelist */
-       df->slab = NULL;
-
-       do {
-               object = p[--size];
-               /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
-       } while (!object && size);
-
-       if (!object)
-               return 0;
+       size_t same;
 
+       object = p[--size];
        folio = virt_to_folio(object);
        if (!s) {
                /* Handle kalloc'ed objects */
                if (unlikely(!folio_test_slab(folio))) {
                        free_large_kmalloc(folio, object);
-                       p[size] = NULL; /* mark object processed */
+                       df->slab = NULL;
                        return size;
                }
                /* Derive kmem_cache from object */
-               slab = folio_slab(folio);
-               df->s = slab->slab_cache;
+               df->slab = folio_slab(folio);
+               df->s = df->slab->slab_cache;
        } else {
-               slab = folio_slab(folio);
+               df->slab = folio_slab(folio);
                df->s = cache_from_obj(s, object); /* Support for memcg */
        }
 
-       if (is_kfence_address(object)) {
-               slab_free_hook(df->s, object, false);
-               __kfence_free(object);
-               p[size] = NULL; /* mark object processed */
-               return size;
-       }
-
        /* Start new detached freelist */
-       df->slab = slab;
-       set_freepointer(df->s, object, NULL);
        df->tail = object;
        df->freelist = object;
-       p[size] = NULL; /* mark object processed */
        df->cnt = 1;
 
+       if (is_kfence_address(object))
+               return size;
+
+       set_freepointer(df->s, object, NULL);
+
+       same = size;
        while (size) {
                object = p[--size];
-               if (!object)
-                       continue; /* Skip processed objects */
-
                /* df->slab is always set at this point */
                if (df->slab == virt_to_slab(object)) {
                        /* Opportunity build freelist */
                        set_freepointer(df->s, object, df->freelist);
                        df->freelist = object;
                        df->cnt++;
-                       p[size] = NULL; /* mark object processed */
-
+                       same--;
+                       if (size != same)
+                               swap(p[size], p[same]);
                        continue;
                }
 
                /* Limit look ahead search */
                if (!--lookahead)
                        break;
-
-               if (!first_skipped_index)
-                       first_skipped_index = size + 1;
        }
 
-       return first_skipped_index;
+       return same;
 }
 
 /* Note that interrupts must be enabled when calling this function. */
@@ -3672,7 +3650,6 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
        if (WARN_ON(!size))
                return;
 
-       memcg_slab_free_hook(s, p, size);
        do {
                struct detached_freelist df;
 
@@ -3680,7 +3657,8 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
                if (!df.slab)
                        continue;
 
-               slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, _RET_IP_);
+               slab_free(df.s, df.slab, df.freelist, df.tail, &p[size], df.cnt,
+                         _RET_IP_);
        } while (likely(size));
 }
 EXPORT_SYMBOL(kmem_cache_free_bulk);
@@ -4581,7 +4559,7 @@ void kfree(const void *x)
                return;
        }
        slab = folio_slab(folio);
-       slab_free(slab->slab_cache, slab, object, NULL, 1, _RET_IP_);
+       slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_);
 }
 EXPORT_SYMBOL(kfree);