futexes: Increase hash table size for better performance

[platform/adaptation/renesas_rcar/renesas_kernel.git] / kernel / futex.c

diff --git a/kernel/futex.c b/kernel/futex.c
index 085f5fa..577481d 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -63,6 +63,7 @@
 #include <linux/sched/rt.h>
 #include <linux/hugetlb.h>
 #include <linux/freezer.h>
+#include <linux/bootmem.h>
 
 #include <asm/futex.h>
 
@@ -70,8 +71,6 @@
 
 int __read_mostly futex_cmpxchg_enabled;
 
-#define FUTEX_HASHBITS (CONFIG_BASE_SMALL ? 4 : 8)
-
 /*
  * Futex flags used to encode options to functions and preserve them across
  * restarts.
@@ -149,9 +148,11 @@ static const struct futex_q futex_q_init = {
 struct futex_hash_bucket {
        spinlock_t lock;
        struct plist_head chain;
-};
+} ____cacheline_aligned_in_smp;
 
-static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS];
+static unsigned long __read_mostly futex_hashsize;
+
+static struct futex_hash_bucket *futex_queues;
 
 /*
  * We hash on the keys returned from get_futex_key (see below).
@@ -161,7 +162,7 @@ static struct futex_hash_bucket *hash_futex(union futex_key *key)
        u32 hash = jhash2((u32*)&key->both.word,
                          (sizeof(key->both.word)+sizeof(key->both.ptr))/4,
                          key->both.offset);
-       return &futex_queues[hash & ((1 << FUTEX_HASHBITS)-1)];
+       return &futex_queues[hash & (futex_hashsize - 1)];
 }
 
 /*
@@ -2719,7 +2720,18 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
 static int __init futex_init(void)
 {
        u32 curval;
-       int i;
+       unsigned long i;
+
+#if CONFIG_BASE_SMALL
+       futex_hashsize = 16;
+#else
+       futex_hashsize = roundup_pow_of_two(256 * num_possible_cpus());
+#endif
+
+       futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),
+                                              futex_hashsize, 0,
+                                              futex_hashsize < 256 ? HASH_SMALL : 0,
+                                              NULL, NULL, futex_hashsize, futex_hashsize);
 
        /*
         * This will fail and we want it. Some arch implementations do
@@ -2734,7 +2746,7 @@ static int __init futex_init(void)
        if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
                futex_cmpxchg_enabled = 1;
 
-       for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {
+       for (i = 0; i < futex_hashsize; i++) {
                plist_head_init(&futex_queues[i].chain);
                spin_lock_init(&futex_queues[i].lock);
        }