Merge branches 'slab/cleanups', 'slab/failslab', 'slab/fixes' and 'slub/percpu' into...

diff --cc mm/slab.c
Simple merge

diff --cc mm/slub.c
index 8d71aaf,8d71aaf,cab5288,8d71aaf,bd4a9e9..3525a4e
--- 1/mm/slub.c
--- 2/mm/slub.c
--- 3/mm/slub.c
--- 4/mm/slub.c
--- 5/mm/slub.c
+++ b/mm/slub.c
@@@@@@ -2095,130 -2095,130 -2099,130 -2095,130 -2062,24 +2066,24 @@@@@@ init_kmem_cache_node(struct kmem_cache_
     #endif
     }
     
---- #ifdef CONFIG_SMP
---- /*
----  * Per cpu array for per cpu structures.
----  *
----  * The per cpu array places all kmem_cache_cpu structures from one processor
----  * close together meaning that it becomes possible that multiple per cpu
----  * structures are contained in one cacheline. This may be particularly
----  * beneficial for the kmalloc caches.
----  *
----  * A desktop system typically has around 60-80 slabs. With 100 here we are
----  * likely able to get per cpu structures for all caches from the array defined
----  * here. We must be able to cover all kmalloc caches during bootstrap.
----  *
----  * If the per cpu array is exhausted then fall back to kmalloc
----  * of individual cachelines. No sharing is possible then.
----  */
---- #define NR_KMEM_CACHE_CPU 100
---- 
---- static DEFINE_PER_CPU(struct kmem_cache_cpu [NR_KMEM_CACHE_CPU],
----                  kmem_cache_cpu);
---- 
---- static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free);
---- static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS);
---- 
---- static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s,
----                                                    int cpu, gfp_t flags)
---- {
----    struct kmem_cache_cpu *c = per_cpu(kmem_cache_cpu_free, cpu);
---- 
----    if (c)
----            per_cpu(kmem_cache_cpu_free, cpu) =
----                            (void *)c->freelist;
----    else {
----            /* Table overflow: So allocate ourselves */
----            c = kmalloc_node(
----                    ALIGN(sizeof(struct kmem_cache_cpu), cache_line_size()),
----                    flags, cpu_to_node(cpu));
----            if (!c)
----                    return NULL;
----    }
---- 
----    init_kmem_cache_cpu(s, c);
----    return c;
---- }
---- 
---- static void free_kmem_cache_cpu(struct kmem_cache_cpu *c, int cpu)
---- {
----    if (c < per_cpu(kmem_cache_cpu, cpu) ||
----                    c >= per_cpu(kmem_cache_cpu, cpu) + NR_KMEM_CACHE_CPU) {
----            kfree(c);
----            return;
----    }
----    c->freelist = (void *)per_cpu(kmem_cache_cpu_free, cpu);
----    per_cpu(kmem_cache_cpu_free, cpu) = c;
---- }
---- 
---- static void free_kmem_cache_cpus(struct kmem_cache *s)
---- {
----    int cpu;
---- 
----    for_each_online_cpu(cpu) {
----            struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
---- 
----            if (c) {
----                    s->cpu_slab[cpu] = NULL;
----                    free_kmem_cache_cpu(c, cpu);
----            }
----    }
---- }
---- 
---- static int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
---- {
----    int cpu;
---- 
----    for_each_online_cpu(cpu) {
----            struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
-- - 
-- -            if (c)
-- -                    continue;
-- - 
-- -            c = alloc_kmem_cache_cpu(s, cpu, flags);
-- -            if (!c) {
-- -                    free_kmem_cache_cpus(s);
-- -                    return 0;
-- -            }
-- -            s->cpu_slab[cpu] = c;
-- -    }
-- -    return 1;
-- - }
-- - 
-- - /*
-- -  * Initialize the per cpu array.
-- -  */
-- - static void init_alloc_cpu_cpu(int cpu)
-- - {
-- -    int i;
++++ static DEFINE_PER_CPU(struct kmem_cache_cpu, kmalloc_percpu[KMALLOC_CACHES]);
     
  -             if (c)
  -                     continue;
  -  
  -             c = alloc_kmem_cache_cpu(s, cpu, flags);
  -             if (!c) {
  -                     free_kmem_cache_cpus(s);
  -                     return 0;
  -             }
  -             s->cpu_slab[cpu] = c;
  -     }
  -     return 1;
  -  }
  -  
  -  /*
  -   * Initialize the per cpu array.
  -   */
  -  static void init_alloc_cpu_cpu(int cpu)
  -  {
  -     int i;
  -  
----    if (cpumask_test_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once)))
----            return;
---- 
----    for (i = NR_KMEM_CACHE_CPU - 1; i >= 0; i--)
----            free_kmem_cache_cpu(&per_cpu(kmem_cache_cpu, cpu)[i], cpu);
---- 
----    cpumask_set_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once));
---- }
---- 
---- static void __init init_alloc_cpu(void)
++++ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
     {
----    int cpu;
---- 
----    for_each_online_cpu(cpu)
----            init_alloc_cpu_cpu(cpu);
----   }
++++    if (s < kmalloc_caches + KMALLOC_CACHES && s >= kmalloc_caches)
++++            /*
++++             * Boot time creation of the kmalloc array. Use static per cpu data
++++             * since the per cpu allocator is not available yet.
++++             */
++++            s->cpu_slab = per_cpu_var(kmalloc_percpu) + (s - kmalloc_caches);
++++    else
++++            s->cpu_slab =  alloc_percpu(struct kmem_cache_cpu);
     
---- #else
---- static inline void free_kmem_cache_cpus(struct kmem_cache *s) {}
---- static inline void init_alloc_cpu(void) {}
++++    if (!s->cpu_slab)
++++            return 0;
     
---- static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
---- {
----    init_kmem_cache_cpu(s, &s->cpu_slab);
        return 1;
     }
---- #endif
     
     #ifdef CONFIG_NUMA
     /*