futex: Restructure futex_requeue()

No point in taking two more 'requeue_pi' conditionals just to get to the
requeue. Same for the requeue_pi case just the other way round.

No functional change.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210815211305.468835790@linutronix.de

diff --git a/kernel/futex.c b/kernel/futex.c
index 5439742..6cb6b5d 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -2104,20 +2104,17 @@ retry_private:
                        break;
                }
 
-               /*
-                * Wake nr_wake waiters.  For requeue_pi, if we acquired the
-                * lock, we already woke the top_waiter.  If not, it will be
-                * woken by futex_unlock_pi().
-                */
-               if (++task_count <= nr_wake && !requeue_pi) {
-                       mark_wake_futex(&wake_q, this);
+               /* Plain futexes just wake or requeue and are done */
+               if (!requeue_pi) {
+                       if (++task_count <= nr_wake)
+                               mark_wake_futex(&wake_q, this);
+                       else
+                               requeue_futex(this, hb1, hb2, &key2);
                        continue;
                }
 
                /* Ensure we requeue to the expected futex for requeue_pi. */
-               if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) {
-                       /* Don't account for it */
-                       task_count--;
+               if (!match_futex(this->requeue_pi_key, &key2)) {
                        ret = -EINVAL;
                        break;
                }
@@ -2125,50 +2122,45 @@ retry_private:
                /*
                 * Requeue nr_requeue waiters and possibly one more in the case
                 * of requeue_pi if we couldn't acquire the lock atomically.
+                *
+                * Prepare the waiter to take the rt_mutex. Take a refcount
+                * on the pi_state and store the pointer in the futex_q
+                * object of the waiter.
                 */
-               if (requeue_pi) {
+               get_pi_state(pi_state);
+               this->pi_state = pi_state;
+               ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
+                                               this->rt_waiter, this->task);
+               if (ret == 1) {
                        /*
-                        * Prepare the waiter to take the rt_mutex. Take a
-                        * refcount on the pi_state and store the pointer in
-                        * the futex_q object of the waiter.
+                        * We got the lock. We do neither drop the refcount
+                        * on pi_state nor clear this->pi_state because the
+                        * waiter needs the pi_state for cleaning up the
+                        * user space value. It will drop the refcount
+                        * after doing so.
                         */
-                       get_pi_state(pi_state);
-                       this->pi_state = pi_state;
-                       ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
-                                                       this->rt_waiter,
-                                                       this->task);
-                       if (ret == 1) {
-                               /*
-                                * We got the lock. We do neither drop the
-                                * refcount on pi_state nor clear
-                                * this->pi_state because the waiter needs the
-                                * pi_state for cleaning up the user space
-                                * value. It will drop the refcount after
-                                * doing so.
-                                */
-                               requeue_pi_wake_futex(this, &key2, hb2);
-                               continue;
-                       } else if (ret) {
-                               /*
-                                * rt_mutex_start_proxy_lock() detected a
-                                * potential deadlock when we tried to queue
-                                * that waiter. Drop the pi_state reference
-                                * which we took above and remove the pointer
-                                * to the state from the waiters futex_q
-                                * object.
-                                */
-                               this->pi_state = NULL;
-                               put_pi_state(pi_state);
-                               /* Don't account for it */
-                               task_count--;
-                               /*
-                                * We stop queueing more waiters and let user
-                                * space deal with the mess.
-                                */
-                               break;
-                       }
+                       requeue_pi_wake_futex(this, &key2, hb2);
+                       task_count++;
+                       continue;
+               } else if (ret) {
+                       /*
+                        * rt_mutex_start_proxy_lock() detected a potential
+                        * deadlock when we tried to queue that waiter.
+                        * Drop the pi_state reference which we took above
+                        * and remove the pointer to the state from the
+                        * waiters futex_q object.
+                        */
+                       this->pi_state = NULL;
+                       put_pi_state(pi_state);
+                       /*
+                        * We stop queueing more waiters and let user space
+                        * deal with the mess.
+                        */
+                       break;
                }
+               /* Waiter is queued, move it to hb2 */
                requeue_futex(this, hb1, hb2, &key2);
+               task_count++;
        }
 
        /*