Add __builtin_expect where appropriate.
authorUlrich Drepper <drepper@redhat.com>
Tue, 18 Mar 2003 00:31:30 +0000 (00:31 +0000)
committerUlrich Drepper <drepper@redhat.com>
Tue, 18 Mar 2003 00:31:30 +0000 (00:31 +0000)
nptl/sysdeps/ia64/bits/atomic.h
nptl/sysdeps/ia64/pthread_spin_unlock.c
nptl/sysdeps/unix/sysv/linux/lowlevellock.c
nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_init.c [new file with mode: 0644]
nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_unlock.S [new file with mode: 0644]

index cfccc6a..fab2307 100644 (file)
@@ -76,20 +76,22 @@ typedef uintmax_t uatomic_max_t;
     __val = *__memp;                                                         \
     if (sizeof (*mem) == 4)                                                  \
       do                                                                     \
-       __oldval = __val;                                                     \
-      while ((__val                                                          \
-             = __arch_compare_and_exchange_32_val_acq (__memp,               \
-                                                       __oldval + __value,   \
-                                                       __oldval))            \
-            != __oldval);                                                    \
+       {                                                                     \
+         __oldval = __val;                                                   \
+         __val = __arch_compare_and_exchange_32_val_acq (__memp,             \
+                                                         __oldval + __value, \
+                                                         __oldval);          \
+       }                                                                     \
+      while (__builtin_expect (__val != __oldval, 0));                       \
     else if (sizeof (*mem) == 8)                                             \
       do                                                                     \
-       __oldval = __val;                                                     \
-      while ((__val                                                          \
-             = __arch_compare_and_exchange_64_val_acq (__memp,               \
-                                                       __oldval + __value,   \
-                                                       __oldval))            \
-            != __oldval);                                                    \
+       {                                                                     \
+         __oldval = __val;                                                   \
+         __val = __arch_compare_and_exchange_64_val_acq (__memp,             \
+                                                         __oldval + __value, \
+                                                         __oldval);          \
+       }                                                                     \
+      while (__builtin_expect (__val != __oldval, 0));                       \
     else                                                                     \
       abort ();                                                                      \
     __oldval + __value; })
index b3f1ff1..9019188 100644 (file)
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
-/* Ugly hack to avoid the declaration of pthread_spin_init.  */
-#define pthread_spin_init pthread_spin_init_XXX
 #include "pthreadP.h"
-#undef pthread_spin_init
+
 
 int
 pthread_spin_unlock (pthread_spinlock_t *lock)
 {
-  *lock = 0;
+  __sync_lock_release_si ((int *) lock);
   return 0;
 }
 strong_alias (pthread_spin_unlock, pthread_spin_init)
index b80e153..4ce4235 100644 (file)
@@ -27,9 +27,9 @@
 void
 __lll_lock_wait (int *futex, int val)
 {
-  do {
-      lll_futex_wait (futex, val+1);
-  while ((val = __lll_add (futex, 1)) != 0);
+  do
+      lll_futex_wait (futex, val + 1);
+  while ((val = __lll_add (futex, 1)) != 0);
   *futex = 2;
 }
 hidden_proto (__lll_lock_wait)
@@ -38,8 +38,6 @@ hidden_proto (__lll_lock_wait)
 int
 __lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
 {
-  int err;
-
   /* Reject invalid timeouts.  */
   if (abstime->tv_nsec >= 1000000000)
     return EINVAL;
@@ -48,33 +46,31 @@ __lll_timedlock_wait (int *futex, int val, const struct timespec *abstime)
     {
       struct timeval tv;
       struct timespec rt;
-      int sec, nsec;
 
       /* Get the current time.  */
-      __gettimeofday (&tv, NULL);
+      (void) __gettimeofday (&tv, NULL);
 
       /* Compute relative timeout.  */
-      sec = abstime->tv_sec - tv.tv_sec;
-      nsec = abstime->tv_nsec - tv.tv_usec * 1000;
-      if (nsec < 0)
+      rt.tv_sec = abstime->tv_sec - tv.tv_sec;
+      rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
+      if (rt.tv_nsec < 0)
        {
-         nsec += 1000000000;
-         --sec;
+         rt.tv_nsec += 1000000000;
+         --rt.tv_sec;
        }
 
       /* Already timed out?  */
-      err = -ETIMEDOUT;
-      if (sec < 0)
-       break;
+      if (rt.tv_sec < 0)
+       return ETIMEDOUT;
 
       /* Wait.  */
-      rt.tv_sec = sec;
-      rt.tv_nsec = nsec;
-      err = lll_futex_timed_wait (futex, val+1, &rt);
-    } while (err == 0 && (val = __lll_add (futex, 1)) != 0);
+      if (lll_futex_timed_wait (futex, val + 1, &rt) == -ETIMEDOUT)
+       return ETIMEDOUT;
+    }
+  while ((val = __lll_add (futex, 1)) != 0);
 
   *futex = 2;
-  return -err;
+  return 0;
 }
 hidden_proto (__lll_timedlock_wait)
 
@@ -84,7 +80,9 @@ hidden_proto (__lll_timedlock_wait)
 int
 lll_unlock_wake_cb (int *futex)
 {
-  __lll_add (futex, 1);
+  if (__lll_add (futex, 1) + 1 != 0)
+    lll_futex_wake (futex, 1);
+
   return 0;
 }
 hidden_proto (lll_unlock_wake_cb)
@@ -94,7 +92,6 @@ int
 __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
 {
   int tid;
-  int err = 0;
 
   if (abstime == NULL || abstime->tv_nsec >= 1000000000)
     return EINVAL;
@@ -104,36 +101,29 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
     {
       struct timeval tv;
       struct timespec rt;
-      int sec, nsec;
 
       /* Get the current time.  */
-      __gettimeofday (&tv, NULL);
+      (void) __gettimeofday (&tv, NULL);
 
       /* Compute relative timeout.  */
-      sec = abstime->tv_sec - tv.tv_sec;
-      nsec = abstime->tv_nsec - tv.tv_usec * 1000;
-      if (nsec < 0)
+      rt.tv_sec = abstime->tv_sec - tv.tv_sec;
+      rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
+      if (rt.tv_nsec < 0)
        {
-         nsec += 1000000000;
-         --sec;
+         rt.tv_nsec += 1000000000;
+         --rt.tv_sec;
        }
 
       /* Already timed out?  */
-      err = -ETIMEDOUT;
-      if (sec < 0)
-       break;
-
-      /* Wait.  */
-      rt.tv_sec = sec;
-      rt.tv_nsec = nsec;
+      if (rt.tv_sec < 0)
+       return ETIMEDOUT;
 
       /* Wait until thread terminates.  */
-      err = lll_futex_timed_wait (tidp, tid, &rt);
-      if (err != 0)
-       break;
+      if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT)
+       return ETIMEDOUT;
     }
 
-  return -err;
+  return 0;
 }
 
 hidden_proto (__lll_timedwait_tid)
index 3295b63..bfd109f 100644 (file)
 /* Initializer for compatibility lock. */
 #define LLL_MUTEX_LOCK_INITIALIZER (0)
 
-#define lll_futex_wait(futexp, val)                            \
-  ({                                                           \
-    INTERNAL_SYSCALL_DECL (__err);                             \
-    long int __ret;                                            \
-                                                               \
-    __ret = INTERNAL_SYSCALL (futex, __err, 4,                 \
-                             (futexp), FUTEX_WAIT, (val), 0);  \
-    INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0;                \
+#define lll_futex_wait(futexp, val) \
+  ({                                                                         \
+    INTERNAL_SYSCALL_DECL (__err);                                           \
+    long int __ret;                                                          \
+                                                                             \
+    __ret = INTERNAL_SYSCALL (futex, __err, 4,                               \
+                             (futexp), FUTEX_WAIT, (val), 0);                \
+    INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0;                              \
   })
 
-#define lll_futex_timed_wait(futexp, val, timespec)                    \
-  ({                                                                   \
-    INTERNAL_SYSCALL_DECL (__err);                                     \
-    long int __ret;                                                    \
-                                                                       \
-    __ret = INTERNAL_SYSCALL (futex, __err, 4,                         \
-                             (futexp), FUTEX_WAIT, (val), (timespec)); \
-    INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0;                        \
+#define lll_futex_timed_wait(futexp, val, timespec) \
+  ({                                                                         \
+    INTERNAL_SYSCALL_DECL (__err);                                           \
+    long int __ret;                                                          \
+                                                                             \
+    __ret = INTERNAL_SYSCALL (futex, __err, 4,                               \
+                             (futexp), FUTEX_WAIT, (val), (timespec));       \
+    INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0;                              \
   })
 
-#define lll_futex_wake(futexp, nr)                             \
-  ({                                                           \
-    INTERNAL_SYSCALL_DECL (__err);                             \
-    long int __ret;                                            \
-                                                               \
-    __ret = INTERNAL_SYSCALL (futex, __err, 4,                 \
-                             (futexp), FUTEX_WAKE, (nr), 0);   \
-    INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0;                \
+#define lll_futex_wake(futexp, nr) \
+  ({                                                                         \
+    INTERNAL_SYSCALL_DECL (__err);                                           \
+    long int __ret;                                                          \
+                                                                             \
+    __ret = INTERNAL_SYSCALL (futex, __err, 4,                               \
+                             (futexp), FUTEX_WAKE, (nr), 0);                 \
+    INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret: 0;                              \
   })
 
 #ifdef UP
-#define __lll_acq_instr        ""
-#define __lll_rel_instr        ""
+# define __lll_acq_instr       ""
+# define __lll_rel_instr       ""
 #else
-#define __lll_acq_instr        "isync"
-#define __lll_rel_instr        "sync"
+# define __lll_acq_instr       "isync"
+# define __lll_rel_instr       "sync"
 #endif
 
 /* Set *futex to 1 if it is 0, atomically.  Returns the old value */
-#define __lll_trylock(futex)                           \
-  ({ int __val;                                                        \
-     __asm __volatile ("1:     lwarx   %0,0,%2\n"              \
-                      "        cmpwi   0,%0,0\n"               \
-                      "        bne     2f\n"                   \
-                      "        stwcx.  %3,0,%2\n"              \
-                      "        bne-    1b\n"                   \
-                      "2:      " __lll_acq_instr               \
-                      : "=&r" (__val), "=m" (*futex)           \
-                      : "r" (futex), "r" (1), "1" (*futex)     \
-                      : "cr0", "memory");                      \
-     __val;                                                    \
+#define __lll_trylock(futex) \
+  ({ int __val;                                                                      \
+     __asm __volatile ("1:     lwarx   %0,0,%2\n"                            \
+                      "        cmpwi   0,%0,0\n"                             \
+                      "        bne     2f\n"                                 \
+                      "        stwcx.  %3,0,%2\n"                            \
+                      "        bne-    1b\n"                                 \
+                      "2:      " __lll_acq_instr                             \
+                      : "=&r" (__val), "=m" (*futex)                         \
+                      : "r" (futex), "r" (1), "1" (*futex)                   \
+                      : "cr0", "memory");                                    \
+     __val;                                                                  \
   })
 
 #define lll_mutex_trylock(lock)        __lll_trylock(&(lock))
 
 /* Add inc to *futex atomically and return the old value. */
-#define __lll_add(futex, inc)                                          \
-  ({ int __val, __tmp;                                                 \
-     __asm __volatile ("1:     lwarx   %0,0,%3\n"                      \
-                      "        addi    %1,%0,%4\n"                     \
-                      "        stwcx.  %1,0,%3\n"                      \
-                      "        bne-    1b"                             \
-                      : "=&b" (__val), "=&r" (__tmp), "=m" (*futex)    \
-                      : "r" (futex), "I" (inc), "2" (*futex)           \
-                      : "cr0");                                        \
-     __val;                                                            \
+#define __lll_add(futex, inc) \
+  ({ int __val, __tmp;                                                       \
+     __asm __volatile ("1:     lwarx   %0,0,%3\n"                            \
+                      "        addi    %1,%0,%4\n"                           \
+                      "        stwcx.  %1,0,%3\n"                            \
+                      "        bne-    1b"                                   \
+                      : "=&b" (__val), "=&r" (__tmp), "=m" (*futex)          \
+                      : "r" (futex), "I" (inc), "2" (*futex)                 \
+                      : "cr0");                                              \
+     __val;                                                                  \
   })
 
 extern void __lll_lock_wait (int *futex, int val) attribute_hidden;
 
-#define lll_mutex_lock(lock)                           \
-  (void) ({                                            \
-    int *__futex = &(lock);                            \
-    int __val = __lll_add (__futex, 1);                        \
-    __asm __volatile (__lll_acq_instr ::: "memory");   \
-    if (__builtin_expect (__val != 0, 0))              \
-      __lll_lock_wait (__futex, __val);                        \
+#define lll_mutex_lock(lock) \
+  (void) ({                                                                  \
+    int *__futex = &(lock);                                                  \
+    int __val = __lll_add (__futex, 1);                                              \
+    __asm __volatile (__lll_acq_instr ::: "memory");                         \
+    if (__builtin_expect (__val != 0, 0))                                    \
+      __lll_lock_wait (__futex, __val);                                              \
   })
 
 extern int __lll_timedlock_wait
        (int *futex, int val, const struct timespec *) attribute_hidden;
 
-#define lll_mutex_timedlock(lock, abstime)                     \
-  ({ int *__futex = &(lock);                                           \
-     int __val = __lll_add (__futex, 1);                               \
-     __asm __volatile (__lll_acq_instr ::: "memory");                  \
-     if (__builtin_expect (__val != 0, 0))                             \
-       __val = __lll_timedlock_wait (__futex, __val, (abstime));       \
-     __val;                                                            \
+#define lll_mutex_timedlock(lock, abstime) \
+  ({ int *__futex = &(lock);                                                 \
+     int __val = __lll_add (__futex, 1);                                     \
+     __asm __volatile (__lll_acq_instr ::: "memory");                        \
+     if (__builtin_expect (__val != 0, 0))                                   \
+       __val = __lll_timedlock_wait (__futex, __val, (abstime));             \
+     __val;                                                                  \
   })
 
-#define lll_mutex_unlock(lock)                         \
-  (void) ({                                            \
-    int *__futex = &(lock);                            \
-     __asm __volatile (__lll_rel_instr ::: "memory");  \
-    int __val = __lll_add (__futex, -1);               \
-    if (__builtin_expect (__val != 1, 0))              \
-      {                                                        \
-       *__futex = 0;                                   \
-       lll_futex_wake (__futex, 1);                    \
-      }                                                        \
+#define lll_mutex_unlock(lock) \
+  (void) ({                                                                  \
+    int *__futex = &(lock);                                                  \
+     __asm __volatile (__lll_rel_instr ::: "memory");                        \
+    int __val = __lll_add (__futex, -1);                                     \
+    if (__builtin_expect (__val != 1, 0))                                    \
+      {                                                                              \
+       *__futex = 0;                                                         \
+       lll_futex_wake (__futex, 1);                                          \
+      }                                                                              \
   })
 
 #define lll_mutex_islocked(futex) \
@@ -164,39 +164,39 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
    wakeup when the clone terminates.  The memory location contains the
    thread ID while the clone is running and is reset to zero
    afterwards. */
-#define lll_wait_tid(tid)                      \
-  do {                                         \
-    __typeof (tid) __tid;                      \
-    while ((__tid = (tid)) != 0)               \
-      lll_futex_wait (&(tid), __tid);          \
+#define lll_wait_tid(tid) \
+  do {                                                                       \
+    __typeof (tid) __tid;                                                    \
+    while ((__tid = (tid)) != 0)                                             \
+      lll_futex_wait (&(tid), __tid);                                        \
   } while (0)
 
 extern int __lll_timedwait_tid (int *, const struct timespec *)
      attribute_hidden;
 
-#define lll_timedwait_tid(tid, abstime)                        \
-  ({                                                   \
-    int __res = 0;                                     \
-    if ((tid) != 0)                                    \
-      __res = __lll_timedwait_tid (&(tid), (abstime)); \
-    __res;                                             \
+#define lll_timedwait_tid(tid, abstime) \
+  ({                                                                         \
+    int __res = 0;                                                           \
+    if ((tid) != 0)                                                          \
+      __res = __lll_timedwait_tid (&(tid), (abstime));                       \
+    __res;                                                                   \
   })
 
 
 /* Decrement *futex if it is > 0, and return the old value */
-#define __lll_dec_if_positive(futex)                                   \
-  ({ int __val, __tmp;                                                 \
-     __asm __volatile ("1:     lwarx   %0,0,%3\n"                      \
-                      "        cmpwi   0,%0,0\n"                       \
-                      "        addi    %1,%0,-1\n"                     \
-                      "        ble     2f\n"                           \
-                      "        stwcx.  %1,0,%3\n"                      \
-                      "        bne-    1b\n"                           \
-                      "2:      " __lll_acq_instr                       \
-                      : "=&b" (__val), "=&r" (__tmp), "=m" (*futex)    \
-                      : "r" (futex), "2" (*futex)                      \
-                      : "cr0");                                        \
-     __val;                                                            \
+#define __lll_dec_if_positive(futex) \
+  ({ int __val, __tmp;                                                       \
+     __asm __volatile ("1:     lwarx   %0,0,%3\n"                            \
+                      "        cmpwi   0,%0,0\n"                             \
+                      "        addi    %1,%0,-1\n"                           \
+                      "        ble     2f\n"                                 \
+                      "        stwcx.  %1,0,%3\n"                            \
+                      "        bne-    1b\n"                                 \
+                      "2:      " __lll_acq_instr                             \
+                      : "=&b" (__val), "=&r" (__tmp), "=m" (*futex)          \
+                      : "r" (futex), "2" (*futex)                            \
+                      : "cr0");                                              \
+     __val;                                                                  \
   })
 
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_init.c b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_init.c
new file mode 100644 (file)
index 0000000..dc81178
--- /dev/null
@@ -0,0 +1 @@
+#include <nptl/sysdeps/x86_64/pthread_spin_init.c>
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_spin_unlock.S
new file mode 100644 (file)
index 0000000..7da1c75
--- /dev/null
@@ -0,0 +1 @@
+#include <nptl/sysdeps/x86_64/pthread_spin_unlock.S>