[PATCH] slab: distinguish between object and buffer size

An object cache has two different object lengths:

  - the amount of memory available for the user (object size)
  - the amount of memory allocated internally (buffer size)

This patch does some renames to make the code reflect that better.

Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/mm/slab.c b/mm/slab.c
index 88082ae..1a014aa 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -375,7 +375,7 @@ struct kmem_cache {
        unsigned int batchcount;
        unsigned int limit;
        unsigned int shared;
-       unsigned int objsize;
+       unsigned int buffer_size;
 /* 2) touched by every alloc & free from the backend */
        struct kmem_list3 *nodelists[MAX_NUMNODES];
        unsigned int flags;     /* constant flags */
@@ -423,8 +423,14 @@ struct kmem_cache {
        atomic_t freemiss;
 #endif
 #if DEBUG
-       int dbghead;
-       int reallen;
+       /*
+        * If debugging is enabled, then the allocator can add additional
+        * fields and/or padding to every object. buffer_size contains the total
+        * object size including these internal fields, the following two
+        * variables contain the offset to the user object and its size.
+        */
+       int obj_offset;
+       int obj_size;
 #endif
 };
 
@@ -495,50 +501,50 @@ struct kmem_cache {
 
 /* memory layout of objects:
  * 0           : objp
- * 0 .. cachep->dbghead - BYTES_PER_WORD - 1: padding. This ensures that
+ * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
  *             the end of an object is aligned with the end of the real
  *             allocation. Catches writes behind the end of the allocation.
- * cachep->dbghead - BYTES_PER_WORD .. cachep->dbghead - 1:
+ * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
  *             redzone word.
- * cachep->dbghead: The real object.
- * cachep->objsize - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
- * cachep->objsize - 1* BYTES_PER_WORD: last caller address [BYTES_PER_WORD long]
+ * cachep->obj_offset: The real object.
+ * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
+ * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address [BYTES_PER_WORD long]
  */
-static int obj_dbghead(kmem_cache_t *cachep)
+static int obj_offset(kmem_cache_t *cachep)
 {
-       return cachep->dbghead;
+       return cachep->obj_offset;
 }
 
-static int obj_reallen(kmem_cache_t *cachep)
+static int obj_size(kmem_cache_t *cachep)
 {
-       return cachep->reallen;
+       return cachep->obj_size;
 }
 
 static unsigned long *dbg_redzone1(kmem_cache_t *cachep, void *objp)
 {
        BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
-       return (unsigned long*) (objp+obj_dbghead(cachep)-BYTES_PER_WORD);
+       return (unsigned long*) (objp+obj_offset(cachep)-BYTES_PER_WORD);
 }
 
 static unsigned long *dbg_redzone2(kmem_cache_t *cachep, void *objp)
 {
        BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
        if (cachep->flags & SLAB_STORE_USER)
-               return (unsigned long *)(objp + cachep->objsize -
+               return (unsigned long *)(objp + cachep->buffer_size -
                                         2 * BYTES_PER_WORD);
-       return (unsigned long *)(objp + cachep->objsize - BYTES_PER_WORD);
+       return (unsigned long *)(objp + cachep->buffer_size - BYTES_PER_WORD);
 }
 
 static void **dbg_userword(kmem_cache_t *cachep, void *objp)
 {
        BUG_ON(!(cachep->flags & SLAB_STORE_USER));
-       return (void **)(objp + cachep->objsize - BYTES_PER_WORD);
+       return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD);
 }
 
 #else
 
-#define obj_dbghead(x)                 0
-#define obj_reallen(cachep)            (cachep->objsize)
+#define obj_offset(x)                  0
+#define obj_size(cachep)               (cachep->buffer_size)
 #define dbg_redzone1(cachep, objp)     ({BUG(); (unsigned long *)NULL;})
 #define dbg_redzone2(cachep, objp)     ({BUG(); (unsigned long *)NULL;})
 #define dbg_userword(cachep, objp)     ({BUG(); (void **)NULL;})
@@ -623,12 +629,12 @@ static kmem_cache_t cache_cache = {
        .batchcount = 1,
        .limit = BOOT_CPUCACHE_ENTRIES,
        .shared = 1,
-       .objsize = sizeof(kmem_cache_t),
+       .buffer_size = sizeof(kmem_cache_t),
        .flags = SLAB_NO_REAP,
        .spinlock = SPIN_LOCK_UNLOCKED,
        .name = "kmem_cache",
 #if DEBUG
-       .reallen = sizeof(kmem_cache_t),
+       .obj_size = sizeof(kmem_cache_t),
 #endif
 };
 
@@ -1057,9 +1063,9 @@ void __init kmem_cache_init(void)
        cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
        cache_cache.nodelists[numa_node_id()] = &initkmem_list3[CACHE_CACHE];
 
-       cache_cache.objsize = ALIGN(cache_cache.objsize, cache_line_size());
+       cache_cache.buffer_size = ALIGN(cache_cache.buffer_size, cache_line_size());
 
-       cache_estimate(0, cache_cache.objsize, cache_line_size(), 0,
+       cache_estimate(0, cache_cache.buffer_size, cache_line_size(), 0,
                       &left_over, &cache_cache.num);
        if (!cache_cache.num)
                BUG();
@@ -1274,9 +1280,9 @@ static void kmem_rcu_free(struct rcu_head *head)
 static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr,
                            unsigned long caller)
 {
-       int size = obj_reallen(cachep);
+       int size = obj_size(cachep);
 
-       addr = (unsigned long *)&((char *)addr)[obj_dbghead(cachep)];
+       addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
 
        if (size < 5 * sizeof(unsigned long))
                return;
@@ -1306,8 +1312,8 @@ static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr,
 
 static void poison_obj(kmem_cache_t *cachep, void *addr, unsigned char val)
 {
-       int size = obj_reallen(cachep);
-       addr = &((char *)addr)[obj_dbghead(cachep)];
+       int size = obj_size(cachep);
+       addr = &((char *)addr)[obj_offset(cachep)];
 
        memset(addr, val, size);
        *(unsigned char *)(addr + size - 1) = POISON_END;
@@ -1344,8 +1350,8 @@ static void print_objinfo(kmem_cache_t *cachep, void *objp, int lines)
                             (unsigned long)*dbg_userword(cachep, objp));
                printk("\n");
        }
-       realobj = (char *)objp + obj_dbghead(cachep);
-       size = obj_reallen(cachep);
+       realobj = (char *)objp + obj_offset(cachep);
+       size = obj_size(cachep);
        for (i = 0; i < size && lines; i += 16, lines--) {
                int limit;
                limit = 16;
@@ -1361,8 +1367,8 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp)
        int size, i;
        int lines = 0;
 
-       realobj = (char *)objp + obj_dbghead(cachep);
-       size = obj_reallen(cachep);
+       realobj = (char *)objp + obj_offset(cachep);
+       size = obj_size(cachep);
 
        for (i = 0; i < size; i++) {
                char exp = POISON_FREE;
@@ -1398,17 +1404,17 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp)
                struct slab *slabp = page_get_slab(virt_to_page(objp));
                int objnr;
 
-               objnr = (unsigned)(objp - slabp->s_mem) / cachep->objsize;
+               objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
                if (objnr) {
-                       objp = slabp->s_mem + (objnr - 1) * cachep->objsize;
-                       realobj = (char *)objp + obj_dbghead(cachep);
+                       objp = slabp->s_mem + (objnr - 1) * cachep->buffer_size;
+                       realobj = (char *)objp + obj_offset(cachep);
                        printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
                               realobj, size);
                        print_objinfo(cachep, objp, 2);
                }
                if (objnr + 1 < cachep->num) {
-                       objp = slabp->s_mem + (objnr + 1) * cachep->objsize;
-                       realobj = (char *)objp + obj_dbghead(cachep);
+                       objp = slabp->s_mem + (objnr + 1) * cachep->buffer_size;
+                       realobj = (char *)objp + obj_offset(cachep);
                        printk(KERN_ERR "Next obj: start=%p, len=%d\n",
                               realobj, size);
                        print_objinfo(cachep, objp, 2);
@@ -1428,14 +1434,14 @@ static void slab_destroy(kmem_cache_t *cachep, struct slab *slabp)
 #if DEBUG
        int i;
        for (i = 0; i < cachep->num; i++) {
-               void *objp = slabp->s_mem + cachep->objsize * i;
+               void *objp = slabp->s_mem + cachep->buffer_size * i;
 
                if (cachep->flags & SLAB_POISON) {
 #ifdef CONFIG_DEBUG_PAGEALLOC
-                       if ((cachep->objsize % PAGE_SIZE) == 0
+                       if ((cachep->buffer_size % PAGE_SIZE) == 0
                            && OFF_SLAB(cachep))
                                kernel_map_pages(virt_to_page(objp),
-                                                cachep->objsize / PAGE_SIZE,
+                                                cachep->buffer_size / PAGE_SIZE,
                                                 1);
                        else
                                check_poison_obj(cachep, objp);
@@ -1452,13 +1458,13 @@ static void slab_destroy(kmem_cache_t *cachep, struct slab *slabp)
                                           "was overwritten");
                }
                if (cachep->dtor && !(cachep->flags & SLAB_POISON))
-                       (cachep->dtor) (objp + obj_dbghead(cachep), cachep, 0);
+                       (cachep->dtor) (objp + obj_offset(cachep), cachep, 0);
        }
 #else
        if (cachep->dtor) {
                int i;
                for (i = 0; i < cachep->num; i++) {
-                       void *objp = slabp->s_mem + cachep->objsize * i;
+                       void *objp = slabp->s_mem + cachep->buffer_size * i;
                        (cachep->dtor) (objp, cachep, 0);
                }
        }
@@ -1478,7 +1484,7 @@ static void slab_destroy(kmem_cache_t *cachep, struct slab *slabp)
        }
 }
 
-/* For setting up all the kmem_list3s for cache whose objsize is same
+/* For setting up all the kmem_list3s for cache whose buffer_size is same
    as size of kmem_list3. */
 static inline void set_up_list3s(kmem_cache_t *cachep, int index)
 {
@@ -1611,7 +1617,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
                set_fs(old_fs);
                if (res) {
                        printk("SLAB: cache with size %d has lost its name\n",
-                              pc->objsize);
+                              pc->buffer_size);
                        continue;
                }
 
@@ -1702,14 +1708,14 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        memset(cachep, 0, sizeof(kmem_cache_t));
 
 #if DEBUG
-       cachep->reallen = size;
+       cachep->obj_size = size;
 
        if (flags & SLAB_RED_ZONE) {
                /* redzoning only works with word aligned caches */
                align = BYTES_PER_WORD;
 
                /* add space for red zone words */
-               cachep->dbghead += BYTES_PER_WORD;
+               cachep->obj_offset += BYTES_PER_WORD;
                size += 2 * BYTES_PER_WORD;
        }
        if (flags & SLAB_STORE_USER) {
@@ -1722,8 +1728,8 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        }
 #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
        if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
-           && cachep->reallen > cache_line_size() && size < PAGE_SIZE) {
-               cachep->dbghead += PAGE_SIZE - size;
+           && cachep->obj_size > cache_line_size() && size < PAGE_SIZE) {
+               cachep->obj_offset += PAGE_SIZE - size;
                size = PAGE_SIZE;
        }
 #endif
@@ -1786,7 +1792,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
        if (flags & SLAB_CACHE_DMA)
                cachep->gfpflags |= GFP_DMA;
        spin_lock_init(&cachep->spinlock);
-       cachep->objsize = size;
+       cachep->buffer_size = size;
 
        if (flags & CFLGS_OFF_SLAB)
                cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
@@ -2118,7 +2124,7 @@ static void cache_init_objs(kmem_cache_t *cachep,
        int i;
 
        for (i = 0; i < cachep->num; i++) {
-               void *objp = slabp->s_mem + cachep->objsize * i;
+               void *objp = slabp->s_mem + cachep->buffer_size * i;
 #if DEBUG
                /* need to poison the objs? */
                if (cachep->flags & SLAB_POISON)
@@ -2136,7 +2142,7 @@ static void cache_init_objs(kmem_cache_t *cachep,
                 * Otherwise, deadlock. They must also be threaded.
                 */
                if (cachep->ctor && !(cachep->flags & SLAB_POISON))
-                       cachep->ctor(objp + obj_dbghead(cachep), cachep,
+                       cachep->ctor(objp + obj_offset(cachep), cachep,
                                     ctor_flags);
 
                if (cachep->flags & SLAB_RED_ZONE) {
@@ -2147,10 +2153,10 @@ static void cache_init_objs(kmem_cache_t *cachep,
                                slab_error(cachep, "constructor overwrote the"
                                           " start of an object");
                }
-               if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)
+               if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)
                    && cachep->flags & SLAB_POISON)
                        kernel_map_pages(virt_to_page(objp),
-                                        cachep->objsize / PAGE_SIZE, 0);
+                                        cachep->buffer_size / PAGE_SIZE, 0);
 #else
                if (cachep->ctor)
                        cachep->ctor(objp, cachep, ctor_flags);
@@ -2309,7 +2315,7 @@ static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp,
        unsigned int objnr;
        struct slab *slabp;
 
-       objp -= obj_dbghead(cachep);
+       objp -= obj_offset(cachep);
        kfree_debugcheck(objp);
        page = virt_to_page(objp);
 
@@ -2341,31 +2347,31 @@ static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp,
        if (cachep->flags & SLAB_STORE_USER)
                *dbg_userword(cachep, objp) = caller;
 
-       objnr = (unsigned)(objp - slabp->s_mem) / cachep->objsize;
+       objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
 
        BUG_ON(objnr >= cachep->num);
-       BUG_ON(objp != slabp->s_mem + objnr * cachep->objsize);
+       BUG_ON(objp != slabp->s_mem + objnr * cachep->buffer_size);
 
        if (cachep->flags & SLAB_DEBUG_INITIAL) {
                /* Need to call the slab's constructor so the
                 * caller can perform a verify of its state (debugging).
                 * Called without the cache-lock held.
                 */
-               cachep->ctor(objp + obj_dbghead(cachep),
+               cachep->ctor(objp + obj_offset(cachep),
                             cachep, SLAB_CTOR_CONSTRUCTOR | SLAB_CTOR_VERIFY);
        }
        if (cachep->flags & SLAB_POISON && cachep->dtor) {
                /* we want to cache poison the object,
                 * call the destruction callback
                 */
-               cachep->dtor(objp + obj_dbghead(cachep), cachep, 0);
+               cachep->dtor(objp + obj_offset(cachep), cachep, 0);
        }
        if (cachep->flags & SLAB_POISON) {
 #ifdef CONFIG_DEBUG_PAGEALLOC
-               if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) {
+               if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) {
                        store_stackinfo(cachep, objp, (unsigned long)caller);
                        kernel_map_pages(virt_to_page(objp),
-                                        cachep->objsize / PAGE_SIZE, 0);
+                                        cachep->buffer_size / PAGE_SIZE, 0);
                } else {
                        poison_obj(cachep, objp, POISON_FREE);
                }
@@ -2468,7 +2474,7 @@ static void *cache_alloc_refill(kmem_cache_t *cachep, gfp_t flags)
 
                        /* get obj pointer */
                        ac->entry[ac->avail++] = slabp->s_mem +
-                           slabp->free * cachep->objsize;
+                           slabp->free * cachep->buffer_size;
 
                        slabp->inuse++;
                        next = slab_bufctl(slabp)[slabp->free];
@@ -2526,9 +2532,9 @@ static void *cache_alloc_debugcheck_after(kmem_cache_t *cachep, gfp_t flags,
                return objp;
        if (cachep->flags & SLAB_POISON) {
 #ifdef CONFIG_DEBUG_PAGEALLOC
-               if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
+               if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
                        kernel_map_pages(virt_to_page(objp),
-                                        cachep->objsize / PAGE_SIZE, 1);
+                                        cachep->buffer_size / PAGE_SIZE, 1);
                else
                        check_poison_obj(cachep, objp);
 #else
@@ -2553,7 +2559,7 @@ static void *cache_alloc_debugcheck_after(kmem_cache_t *cachep, gfp_t flags,
                *dbg_redzone1(cachep, objp) = RED_ACTIVE;
                *dbg_redzone2(cachep, objp) = RED_ACTIVE;
        }
-       objp += obj_dbghead(cachep);
+       objp += obj_offset(cachep);
        if (cachep->ctor && cachep->flags & SLAB_POISON) {
                unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR;
 
@@ -2648,7 +2654,7 @@ static void *__cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid)
        BUG_ON(slabp->inuse == cachep->num);
 
        /* get obj pointer */
-       obj = slabp->s_mem + slabp->free * cachep->objsize;
+       obj = slabp->s_mem + slabp->free * cachep->buffer_size;
        slabp->inuse++;
        next = slab_bufctl(slabp)[slabp->free];
 #if DEBUG
@@ -2699,7 +2705,7 @@ static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects,
                slabp = page_get_slab(virt_to_page(objp));
                l3 = cachep->nodelists[node];
                list_del(&slabp->list);
-               objnr = (unsigned)(objp - slabp->s_mem) / cachep->objsize;
+               objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size;
                check_spinlock_acquired_node(cachep, node);
                check_slabp(cachep, slabp);
 
@@ -2881,7 +2887,7 @@ int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr)
        unsigned long addr = (unsigned long)ptr;
        unsigned long min_addr = PAGE_OFFSET;
        unsigned long align_mask = BYTES_PER_WORD - 1;
-       unsigned long size = cachep->objsize;
+       unsigned long size = cachep->buffer_size;
        struct page *page;
 
        if (unlikely(addr < min_addr))
@@ -3083,7 +3089,7 @@ void kfree(const void *objp)
        local_irq_save(flags);
        kfree_debugcheck(objp);
        c = page_get_cache(virt_to_page(objp));
-       mutex_debug_check_no_locks_freed(objp, obj_reallen(c));
+       mutex_debug_check_no_locks_freed(objp, obj_size(c));
        __cache_free(c, (void *)objp);
        local_irq_restore(flags);
 }
@@ -3114,7 +3120,7 @@ EXPORT_SYMBOL(free_percpu);
 
 unsigned int kmem_cache_size(kmem_cache_t *cachep)
 {
-       return obj_reallen(cachep);
+       return obj_size(cachep);
 }
 EXPORT_SYMBOL(kmem_cache_size);
 
@@ -3258,13 +3264,13 @@ static void enable_cpucache(kmem_cache_t *cachep)
         * The numbers are guessed, we should auto-tune as described by
         * Bonwick.
         */
-       if (cachep->objsize > 131072)
+       if (cachep->buffer_size > 131072)
                limit = 1;
-       else if (cachep->objsize > PAGE_SIZE)
+       else if (cachep->buffer_size > PAGE_SIZE)
                limit = 8;
-       else if (cachep->objsize > 1024)
+       else if (cachep->buffer_size > 1024)
                limit = 24;
-       else if (cachep->objsize > 256)
+       else if (cachep->buffer_size > 256)
                limit = 54;
        else
                limit = 120;
@@ -3279,7 +3285,7 @@ static void enable_cpucache(kmem_cache_t *cachep)
         */
        shared = 0;
 #ifdef CONFIG_SMP
-       if (cachep->objsize <= PAGE_SIZE)
+       if (cachep->buffer_size <= PAGE_SIZE)
                shared = 8;
 #endif
 
@@ -3528,7 +3534,7 @@ static int s_show(struct seq_file *m, void *p)
                printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
 
        seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
-                  name, active_objs, num_objs, cachep->objsize,
+                  name, active_objs, num_objs, cachep->buffer_size,
                   cachep->num, (1 << cachep->gfporder));
        seq_printf(m, " : tunables %4u %4u %4u",
                   cachep->limit, cachep->batchcount, cachep->shared);
@@ -3656,5 +3662,5 @@ unsigned int ksize(const void *objp)
        if (unlikely(objp == NULL))
                return 0;
 
-       return obj_reallen(page_get_cache(virt_to_page(objp)));
+       return obj_size(page_get_cache(virt_to_page(objp)));
 }