futexes: Increase hash table size for better performance

[platform/adaptation/renesas_rcar/renesas_kernel.git] / kernel / futex.c

diff --git a/kernel/futex.c b/kernel/futex.c
index f6ff019..577481d 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -63,6 +63,7 @@
 #include <linux/sched/rt.h>
 #include <linux/hugetlb.h>
 #include <linux/freezer.h>
+#include <linux/bootmem.h>
 
 #include <asm/futex.h>
 
@@ -70,8 +71,6 @@
 
 int __read_mostly futex_cmpxchg_enabled;
 
-#define FUTEX_HASHBITS (CONFIG_BASE_SMALL ? 4 : 8)
-
 /*
  * Futex flags used to encode options to functions and preserve them across
  * restarts.
@@ -149,9 +148,11 @@ static const struct futex_q futex_q_init = {
 struct futex_hash_bucket {
        spinlock_t lock;
        struct plist_head chain;
-};
+} ____cacheline_aligned_in_smp;
+
+static unsigned long __read_mostly futex_hashsize;
 
-static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS];
+static struct futex_hash_bucket *futex_queues;
 
 /*
  * We hash on the keys returned from get_futex_key (see below).
@@ -161,7 +162,7 @@ static struct futex_hash_bucket *hash_futex(union futex_key *key)
        u32 hash = jhash2((u32*)&key->both.word,
                          (sizeof(key->both.word)+sizeof(key->both.ptr))/4,
                          key->both.offset);
-       return &futex_queues[hash & ((1 << FUTEX_HASHBITS)-1)];
+       return &futex_queues[hash & (futex_hashsize - 1)];
 }
 
 /*
@@ -598,13 +599,10 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
 {
        struct futex_pi_state *pi_state = NULL;
        struct futex_q *this, *next;
-       struct plist_head *head;
        struct task_struct *p;
        pid_t pid = uval & FUTEX_TID_MASK;
 
-       head = &hb->chain;
-
-       plist_for_each_entry_safe(this, next, head, list) {
+       plist_for_each_entry_safe(this, next, &hb->chain, list) {
                if (match_futex(&this->key, key)) {
                        /*
                         * Another waiter already exists - bump up
@@ -986,7 +984,6 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
 {
        struct futex_hash_bucket *hb;
        struct futex_q *this, *next;
-       struct plist_head *head;
        union futex_key key = FUTEX_KEY_INIT;
        int ret;
 
@@ -999,9 +996,8 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
 
        hb = hash_futex(&key);
        spin_lock(&hb->lock);
-       head = &hb->chain;
 
-       plist_for_each_entry_safe(this, next, head, list) {
+       plist_for_each_entry_safe(this, next, &hb->chain, list) {
                if (match_futex (&this->key, &key)) {
                        if (this->pi_state || this->rt_waiter) {
                                ret = -EINVAL;
@@ -1034,7 +1030,6 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
 {
        union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
        struct futex_hash_bucket *hb1, *hb2;
-       struct plist_head *head;
        struct futex_q *this, *next;
        int ret, op_ret;
 
@@ -1082,9 +1077,7 @@ retry_private:
                goto retry;
        }
 
-       head = &hb1->chain;
-
-       plist_for_each_entry_safe(this, next, head, list) {
+       plist_for_each_entry_safe(this, next, &hb1->chain, list) {
                if (match_futex (&this->key, &key1)) {
                        if (this->pi_state || this->rt_waiter) {
                                ret = -EINVAL;
@@ -1097,10 +1090,8 @@ retry_private:
        }
 
        if (op_ret > 0) {
-               head = &hb2->chain;
-
                op_ret = 0;
-               plist_for_each_entry_safe(this, next, head, list) {
+               plist_for_each_entry_safe(this, next, &hb2->chain, list) {
                        if (match_futex (&this->key, &key2)) {
                                if (this->pi_state || this->rt_waiter) {
                                        ret = -EINVAL;
@@ -1270,7 +1261,6 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags,
        int drop_count = 0, task_count = 0, ret;
        struct futex_pi_state *pi_state = NULL;
        struct futex_hash_bucket *hb1, *hb2;
-       struct plist_head *head1;
        struct futex_q *this, *next;
        u32 curval2;
 
@@ -1393,8 +1383,7 @@ retry_private:
                }
        }
 
-       head1 = &hb1->chain;
-       plist_for_each_entry_safe(this, next, head1, list) {
+       plist_for_each_entry_safe(this, next, &hb1->chain, list) {
                if (task_count - nr_wake >= nr_requeue)
                        break;
 
@@ -1494,7 +1483,7 @@ static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)
 }
 
 static inline void
-queue_unlock(struct futex_q *q, struct futex_hash_bucket *hb)
+queue_unlock(struct futex_hash_bucket *hb)
        __releases(&hb->lock)
 {
        spin_unlock(&hb->lock);
@@ -1867,7 +1856,7 @@ retry_private:
        ret = get_futex_value_locked(&uval, uaddr);
 
        if (ret) {
-               queue_unlock(q, *hb);
+               queue_unlock(*hb);
 
                ret = get_user(uval, uaddr);
                if (ret)
@@ -1881,7 +1870,7 @@ retry_private:
        }
 
        if (uval != val) {
-               queue_unlock(q, *hb);
+               queue_unlock(*hb);
                ret = -EWOULDBLOCK;
        }
 
@@ -2029,7 +2018,7 @@ retry_private:
                         * Task is exiting and we just wait for the
                         * exit to complete.
                         */
-                       queue_unlock(&q, hb);
+                       queue_unlock(hb);
                        put_futex_key(&q.key);
                        cond_resched();
                        goto retry;
@@ -2081,7 +2070,7 @@ retry_private:
        goto out_put_key;
 
 out_unlock_put_key:
-       queue_unlock(&q, hb);
+       queue_unlock(hb);
 
 out_put_key:
        put_futex_key(&q.key);
@@ -2091,7 +2080,7 @@ out:
        return ret != -EINTR ? ret : -ERESTARTNOINTR;
 
 uaddr_faulted:
-       queue_unlock(&q, hb);
+       queue_unlock(hb);
 
        ret = fault_in_user_writeable(uaddr);
        if (ret)
@@ -2113,7 +2102,6 @@ static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
 {
        struct futex_hash_bucket *hb;
        struct futex_q *this, *next;
-       struct plist_head *head;
        union futex_key key = FUTEX_KEY_INIT;
        u32 uval, vpid = task_pid_vnr(current);
        int ret;
@@ -2153,9 +2141,7 @@ retry:
         * Ok, other tasks may need to be woken up - check waiters
         * and do the wakeup if necessary:
         */
-       head = &hb->chain;
-
-       plist_for_each_entry_safe(this, next, head, list) {
+       plist_for_each_entry_safe(this, next, &hb->chain, list) {
                if (!match_futex (&this->key, &key))
                        continue;
                ret = wake_futex_pi(uaddr, uval, this);
@@ -2734,7 +2720,18 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
 static int __init futex_init(void)
 {
        u32 curval;
-       int i;
+       unsigned long i;
+
+#if CONFIG_BASE_SMALL
+       futex_hashsize = 16;
+#else
+       futex_hashsize = roundup_pow_of_two(256 * num_possible_cpus());
+#endif
+
+       futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),
+                                              futex_hashsize, 0,
+                                              futex_hashsize < 256 ? HASH_SMALL : 0,
+                                              NULL, NULL, futex_hashsize, futex_hashsize);
 
        /*
         * This will fail and we want it. Some arch implementations do
@@ -2749,7 +2746,7 @@ static int __init futex_init(void)
        if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
                futex_cmpxchg_enabled = 1;
 
-       for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {
+       for (i = 0; i < futex_hashsize; i++) {
                plist_head_init(&futex_queues[i].chain);
                spin_lock_init(&futex_queues[i].lock);
        }