locking/rtmutex: Guard regular sleeping locks specific functions

Guard the regular sleeping lock specific functionality, which is used for
rtmutex on non-RT enabled kernels and for mutex, rtmutex and semaphores on
RT enabled kernels so the code can be reused for the RT specific
implementation of spinlocks and rwlocks in a different compilation unit.

No functional change.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210815211303.311535693@linutronix.de

diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 8b0d38d..949781a 100644 (file)
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1075,10 +1075,139 @@ static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
        raw_spin_unlock(&current->pi_lock);
 }
 
+static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
+{
+       int ret = try_to_take_rt_mutex(lock, current, NULL);
+
+       /*
+        * try_to_take_rt_mutex() sets the lock waiters bit
+        * unconditionally. Clean this up.
+        */
+       fixup_rt_mutex_waiters(lock);
+
+       return ret;
+}
+
+/*
+ * Slow path try-lock function:
+ */
+static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)
+{
+       unsigned long flags;
+       int ret;
+
+       /*
+        * If the lock already has an owner we fail to get the lock.
+        * This can be done without taking the @lock->wait_lock as
+        * it is only being read, and this is a trylock anyway.
+        */
+       if (rt_mutex_owner(lock))
+               return 0;
+
+       /*
+        * The mutex has currently no owner. Lock the wait lock and try to
+        * acquire the lock. We use irqsave here to support early boot calls.
+        */
+       raw_spin_lock_irqsave(&lock->wait_lock, flags);
+
+       ret = __rt_mutex_slowtrylock(lock);
+
+       raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+
+       return ret;
+}
+
+static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)
+{
+       if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
+               return 1;
+
+       return rt_mutex_slowtrylock(lock);
+}
+
+/*
+ * Slow path to release a rt-mutex.
+ */
+static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)
+{
+       DEFINE_RT_WAKE_Q(wqh);
+       unsigned long flags;
+
+       /* irqsave required to support early boot calls */
+       raw_spin_lock_irqsave(&lock->wait_lock, flags);
+
+       debug_rt_mutex_unlock(lock);
+
+       /*
+        * We must be careful here if the fast path is enabled. If we
+        * have no waiters queued we cannot set owner to NULL here
+        * because of:
+        *
+        * foo->lock->owner = NULL;
+        *                      rtmutex_lock(foo->lock);   <- fast path
+        *                      free = atomic_dec_and_test(foo->refcnt);
+        *                      rtmutex_unlock(foo->lock); <- fast path
+        *                      if (free)
+        *                              kfree(foo);
+        * raw_spin_unlock(foo->lock->wait_lock);
+        *
+        * So for the fastpath enabled kernel:
+        *
+        * Nothing can set the waiters bit as long as we hold
+        * lock->wait_lock. So we do the following sequence:
+        *
+        *      owner = rt_mutex_owner(lock);
+        *      clear_rt_mutex_waiters(lock);
+        *      raw_spin_unlock(&lock->wait_lock);
+        *      if (cmpxchg(&lock->owner, owner, 0) == owner)
+        *              return;
+        *      goto retry;
+        *
+        * The fastpath disabled variant is simple as all access to
+        * lock->owner is serialized by lock->wait_lock:
+        *
+        *      lock->owner = NULL;
+        *      raw_spin_unlock(&lock->wait_lock);
+        */
+       while (!rt_mutex_has_waiters(lock)) {
+               /* Drops lock->wait_lock ! */
+               if (unlock_rt_mutex_safe(lock, flags) == true)
+                       return;
+               /* Relock the rtmutex and try again */
+               raw_spin_lock_irqsave(&lock->wait_lock, flags);
+       }
+
+       /*
+        * The wakeup next waiter path does not suffer from the above
+        * race. See the comments there.
+        *
+        * Queue the next waiter for wakeup once we release the wait_lock.
+        */
+       mark_wakeup_next_waiter(&wqh, lock);
+       raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+
+       rt_mutex_wake_up_q(&wqh);
+}
+
+static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)
+{
+       if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
+               return;
+
+       rt_mutex_slowunlock(lock);
+}
+
+#ifdef RT_MUTEX_BUILD_MUTEX
+/*
+ * Functions required for:
+ *     - rtmutex, futex on all kernels
+ *     - mutex and rwsem substitutions on RT kernels
+ */
+
 /*
  * Remove a waiter from a lock and give up
  *
- * Must be called with lock->wait_lock held and interrupts disabled. I must
+ * Must be called with lock->wait_lock held and interrupts disabled. It must
  * have just failed to try_to_take_rt_mutex().
  */
 static void __sched remove_waiter(struct rt_mutex_base *lock,
@@ -1286,125 +1415,4 @@ static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
 
        return rt_mutex_slowlock(lock, state);
 }
-
-static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
-{
-       int ret = try_to_take_rt_mutex(lock, current, NULL);
-
-       /*
-        * try_to_take_rt_mutex() sets the lock waiters bit
-        * unconditionally. Clean this up.
-        */
-       fixup_rt_mutex_waiters(lock);
-
-       return ret;
-}
-
-/*
- * Slow path try-lock function:
- */
-static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)
-{
-       unsigned long flags;
-       int ret;
-
-       /*
-        * If the lock already has an owner we fail to get the lock.
-        * This can be done without taking the @lock->wait_lock as
-        * it is only being read, and this is a trylock anyway.
-        */
-       if (rt_mutex_owner(lock))
-               return 0;
-
-       /*
-        * The mutex has currently no owner. Lock the wait lock and try to
-        * acquire the lock. We use irqsave here to support early boot calls.
-        */
-       raw_spin_lock_irqsave(&lock->wait_lock, flags);
-
-       ret = __rt_mutex_slowtrylock(lock);
-
-       raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
-
-       return ret;
-}
-
-static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)
-{
-       if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
-               return 1;
-
-       return rt_mutex_slowtrylock(lock);
-}
-
-/*
- * Slow path to release a rt-mutex.
- */
-static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)
-{
-       DEFINE_RT_WAKE_Q(wqh);
-       unsigned long flags;
-
-       /* irqsave required to support early boot calls */
-       raw_spin_lock_irqsave(&lock->wait_lock, flags);
-
-       debug_rt_mutex_unlock(lock);
-
-       /*
-        * We must be careful here if the fast path is enabled. If we
-        * have no waiters queued we cannot set owner to NULL here
-        * because of:
-        *
-        * foo->lock->owner = NULL;
-        *                      rtmutex_lock(foo->lock);   <- fast path
-        *                      free = atomic_dec_and_test(foo->refcnt);
-        *                      rtmutex_unlock(foo->lock); <- fast path
-        *                      if (free)
-        *                              kfree(foo);
-        * raw_spin_unlock(foo->lock->wait_lock);
-        *
-        * So for the fastpath enabled kernel:
-        *
-        * Nothing can set the waiters bit as long as we hold
-        * lock->wait_lock. So we do the following sequence:
-        *
-        *      owner = rt_mutex_owner(lock);
-        *      clear_rt_mutex_waiters(lock);
-        *      raw_spin_unlock(&lock->wait_lock);
-        *      if (cmpxchg(&lock->owner, owner, 0) == owner)
-        *              return;
-        *      goto retry;
-        *
-        * The fastpath disabled variant is simple as all access to
-        * lock->owner is serialized by lock->wait_lock:
-        *
-        *      lock->owner = NULL;
-        *      raw_spin_unlock(&lock->wait_lock);
-        */
-       while (!rt_mutex_has_waiters(lock)) {
-               /* Drops lock->wait_lock ! */
-               if (unlock_rt_mutex_safe(lock, flags) == true)
-                       return;
-               /* Relock the rtmutex and try again */
-               raw_spin_lock_irqsave(&lock->wait_lock, flags);
-       }
-
-       /*
-        * The wakeup next waiter path does not suffer from the above
-        * race. See the comments there.
-        *
-        * Queue the next waiter for wakeup once we release the wait_lock.
-        */
-       mark_wakeup_next_waiter(&wqh, lock);
-       raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
-
-       rt_mutex_wake_up_q(&wqh);
-}
-
-static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)
-{
-       if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
-               return;
-
-       rt_mutex_slowunlock(lock);
-}
+#endif /* RT_MUTEX_BUILD_MUTEX */

diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c
index 56403dc..7f3ac09 100644 (file)
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -5,6 +5,7 @@
 #include <linux/spinlock.h>
 #include <linux/export.h>
 
+#define RT_MUTEX_BUILD_MUTEX
 #include "rtmutex.c"
 
 /*

diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index c017f9f..2847833 100644 (file)
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -1347,6 +1347,7 @@ static inline void __downgrade_write(struct rw_semaphore *sem)
 
 #else /* !CONFIG_PREEMPT_RT */
 
+#define RT_MUTEX_BUILD_MUTEX
 #include "rtmutex.c"
 
 #define rwbase_set_and_save_current_state(state)       \