nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h

   1 /* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26 #include <atomic.h>
  27
  28 #ifndef __NR_futex
  29 # define __NR_futex             221
  30 #endif
  31 #define FUTEX_WAIT              0
  32 #define FUTEX_WAKE              1
  33 #define FUTEX_REQUEUE           3
  34 #define FUTEX_CMP_REQUEUE       4
  35 #define FUTEX_WAKE_OP           5
  36 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  37 #define FUTEX_LOCK_PI           6
  38 #define FUTEX_UNLOCK_PI         7
  39 #define FUTEX_TRYLOCK_PI        8
  40 #define FUTEX_PRIVATE_FLAG      128
  41
  42 /* Values for 'private' parameter of locking macros.  Yes, the
  43    definition seems to be backwards.  But it is not.  The bit will be
  44    reversed before passing to the system call.  */
  45 #define LLL_PRIVATE     0
  46 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  47
  48 #if !defined NOT_IN_libc || defined IS_IN_rtld
  49 /* In libc.so or ld.so all futexes are private.  */
  50 # ifdef __ASSUME_PRIVATE_FUTEX
  51 #  define __lll_private_flag(fl, private) \
  52   ((fl) | FUTEX_PRIVATE_FLAG)
  53 # else
  54 #  define __lll_private_flag(fl, private) \
  55   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  56 # endif
  57 #else
  58 # ifdef __ASSUME_PRIVATE_FUTEX
  59 #  define __lll_private_flag(fl, private) \
  60   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  61 # else
  62 #  define __lll_private_flag(fl, private) \
  63   (__builtin_constant_p (private)                                             \
  64    ? ((private) == 0                                                          \
  65       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  66       : (fl))                                                                 \
  67    : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG)                                \
  68               & THREAD_GETMEM (THREAD_SELF, header.private_futex))))
  69 # endif
  70 #endif
  71
  72 /* Initializer for compatibility lock.  */
  73 #define LLL_MUTEX_LOCK_INITIALIZER (0)
  74
  75 #define lll_futex_wait(futexp, val, private) \
  76   lll_futex_timed_wait (futexp, val, NULL, private)
  77
  78 #define lll_futex_timed_wait(futexp, val, timespec, private) \
  79   ({                                                                          \
  80     INTERNAL_SYSCALL_DECL (__err);                                            \
  81     long int __ret;                                                           \
  82                                                                               \
  83     __ret = INTERNAL_SYSCALL (futex, __err, 4, (futexp),                      \
  84                               __lll_private_flag (FUTEX_WAIT, private),       \
  85                               (val), (timespec));                             \
  86     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  87   })
  88
  89 #define lll_futex_wake(futexp, nr, private) \
  90   ({                                                                          \
  91     INTERNAL_SYSCALL_DECL (__err);                                            \
  92     long int __ret;                                                           \
  93                                                                               \
  94     __ret = INTERNAL_SYSCALL (futex, __err, 4, (futexp),                      \
  95                               __lll_private_flag (FUTEX_WAKE, private),       \
  96                               (nr), 0);                                       \
  97     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  98   })
  99
 100 #define lll_robust_mutex_dead(futexv) \
 101   do                                                                          \
 102     {                                                                         \
 103       INTERNAL_SYSCALL_DECL (__err);                                          \
 104       int *__futexp = &(futexv);                                              \
 105                                                                               \
 106       atomic_or (__futexp, FUTEX_OWNER_DIED);                                 \
 107       INTERNAL_SYSCALL (futex, __err, 4, __futexp, FUTEX_WAKE, 1, 0);         \
 108     }                                                                         \
 109   while (0)
 110
 111 /* Returns non-zero if error happened, zero if success.  */
 112 #define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
 113   ({                                                                          \
 114     INTERNAL_SYSCALL_DECL (__err);                                            \
 115     long int __ret;                                                           \
 116                                                                               \
 117     __ret = INTERNAL_SYSCALL (futex, __err, 6,                                \
 118                               (futexp), FUTEX_CMP_REQUEUE, (nr_wake),         \
 119                               (nr_move), (mutex), (val));                     \
 120     INTERNAL_SYSCALL_ERROR_P (__ret, __err);                                  \
 121   })
 122
 123 /* Returns non-zero if error happened, zero if success.  */
 124 #define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
 125   ({                                                                          \
 126     INTERNAL_SYSCALL_DECL (__err);                                            \
 127     long int __ret;                                                           \
 128                                                                               \
 129     __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),                      \
 130                               __lll_private_flag (FUTEX_WAKE_OP, private),    \
 131                               (nr_wake), (nr_wake2), (futexp2),               \
 132                               FUTEX_OP_CLEAR_WAKE_IF_GT_ONE);                 \
 133     INTERNAL_SYSCALL_ERROR_P (__ret, __err);                                  \
 134   })
 135
 136
 137 #define lll_private_futex_wait(futexp, val) \
 138   lll_futex_timed_wait (futexp, val, NULL, LLL_PRIVATE)
 139
 140 #define lll_private_futex_timed_wait(futexp, val, timeout) \
 141   lll_futex_timed_wait (futexp, val, timeout, LLL_PRIVATE)
 142
 143 #define lll_private_futex_wake(futexp, val) \
 144   lll_futex_wake (futexp, val, LLL_PRIVATE)
 145
 146 #ifdef UP
 147 # define __lll_acq_instr        ""
 148 # define __lll_rel_instr        ""
 149 #else
 150 # define __lll_acq_instr        "isync"
 151 # ifdef _ARCH_PWR4
 152 /*
 153  * Newer powerpc64 processors support the new "light weight" sync (lwsync)
 154  * So if the build is using -mcpu=[power4,power5,power5+,970] we can
 155  * safely use lwsync.
 156  */
 157 #  define __lll_rel_instr       "lwsync"
 158 # else
 159 /*
 160  * Older powerpc32 processors don't support the new "light weight"
 161  * sync (lwsync).  So the only safe option is to use normal sync
 162  * for all powerpc32 applications.
 163  */
 164 #  define __lll_rel_instr       "sync"
 165 # endif
 166 #endif
 167
 168 /* Set *futex to ID if it is 0, atomically.  Returns the old value */
 169 #define __lll_robust_trylock(futex, id) \
 170   ({ int __val;                                                               \
 171      __asm __volatile ("1:      lwarx   %0,0,%2" MUTEX_HINT_ACQ "\n"          \
 172                        "        cmpwi   0,%0,0\n"                             \
 173                        "        bne     2f\n"                                 \
 174                        "        stwcx.  %3,0,%2\n"                            \
 175                        "        bne-    1b\n"                                 \
 176                        "2:      " __lll_acq_instr                             \
 177                        : "=&r" (__val), "=m" (*futex)                         \
 178                        : "r" (futex), "r" (id), "m" (*futex)                  \
 179                        : "cr0", "memory");                                    \
 180      __val;                                                                   \
 181   })
 182
 183 #define lll_robust_mutex_trylock(lock, id) __lll_robust_trylock (&(lock), id)
 184
 185 /* Set *futex to 1 if it is 0, atomically.  Returns the old value */
 186 #define __lll_trylock(futex) __lll_robust_trylock (futex, 1)
 187
 188 #define lll_mutex_trylock(lock) __lll_trylock (&(lock))
 189
 190 /* Set *futex to 2 if it is 0, atomically.  Returns the old value */
 191 #define __lll_cond_trylock(futex) __lll_robust_trylock (futex, 2)
 192
 193 #define lll_mutex_cond_trylock(lock)    __lll_cond_trylock (&(lock))
 194
 195
 196 extern void __lll_lock_wait (int *futex) attribute_hidden;
 197 extern int __lll_robust_lock_wait (int *futex) attribute_hidden;
 198
 199 #define lll_mutex_lock(lock) \
 200   (void) ({                                                                   \
 201     int *__futex = &(lock);                                                   \
 202     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 1, 0),\
 203                           0) != 0)                                            \
 204       __lll_lock_wait (__futex);                                              \
 205   })
 206
 207 #define lll_robust_mutex_lock(lock, id) \
 208   ({                                                                          \
 209     int *__futex = &(lock);                                                   \
 210     int __val = 0;                                                            \
 211     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id,  \
 212                                                                 0), 0))       \
 213       __val = __lll_robust_lock_wait (__futex);                               \
 214     __val;                                                                    \
 215   })
 216
 217 #define lll_mutex_cond_lock(lock) \
 218   (void) ({                                                                   \
 219     int *__futex = &(lock);                                                   \
 220     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 2, 0),\
 221                           0) != 0)                                            \
 222       __lll_lock_wait (__futex);                                              \
 223   })
 224
 225 #define lll_robust_mutex_cond_lock(lock, id) \
 226   ({                                                                          \
 227     int *__futex = &(lock);                                                   \
 228     int __val = 0;                                                            \
 229     int __id = id | FUTEX_WAITERS;                                            \
 230     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, __id,\
 231                                                                 0), 0))       \
 232       __val = __lll_robust_lock_wait (__futex);                               \
 233     __val;                                                                    \
 234   })
 235
 236
 237 extern int __lll_timedlock_wait
 238   (int *futex, const struct timespec *) attribute_hidden;
 239 extern int __lll_robust_timedlock_wait
 240   (int *futex, const struct timespec *) attribute_hidden;
 241
 242 #define lll_mutex_timedlock(lock, abstime) \
 243   ({                                                                          \
 244     int *__futex = &(lock);                                                   \
 245     int __val = 0;                                                            \
 246     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 1, 0),\
 247                           0) != 0)                                            \
 248       __val = __lll_timedlock_wait (__futex, abstime);                        \
 249     __val;                                                                    \
 250   })
 251
 252 #define lll_robust_mutex_timedlock(lock, abstime, id) \
 253   ({                                                                          \
 254     int *__futex = &(lock);                                                   \
 255     int __val = 0;                                                            \
 256     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id,  \
 257                                                                 0), 0))       \
 258       __val = __lll_robust_timedlock_wait (__futex, abstime);                 \
 259     __val;                                                                    \
 260   })
 261
 262 #define lll_mutex_unlock(lock) \
 263   ((void) ({                                                                  \
 264     int *__futex = &(lock);                                                   \
 265     int __val = atomic_exchange_rel (__futex, 0);                             \
 266     if (__builtin_expect (__val > 1, 0))                                      \
 267       lll_futex_wake (__futex, 1, LLL_SHARED);                                \
 268   }))
 269
 270 #define lll_robust_mutex_unlock(lock) \
 271   ((void) ({                                                                  \
 272     int *__futex = &(lock);                                                   \
 273     int __val = atomic_exchange_rel (__futex, 0);                             \
 274     if (__builtin_expect (__val & FUTEX_WAITERS, 0))                          \
 275       lll_futex_wake (__futex, 1, LLL_SHARED);                                \
 276   }))
 277
 278 #define lll_mutex_unlock_force(lock) \
 279   ((void) ({                                                                  \
 280     int *__futex = &(lock);                                                   \
 281     *__futex = 0;                                                             \
 282     __asm __volatile (__lll_rel_instr ::: "memory");                          \
 283     lll_futex_wake (__futex, 1, LLL_SHARED);                                  \
 284   }))
 285
 286 #define lll_mutex_islocked(futex) \
 287   (futex != 0)
 288
 289
 290 /* Our internal lock implementation is identical to the binary-compatible
 291    mutex implementation. */
 292
 293 /* Type for lock object.  */
 294 typedef int lll_lock_t;
 295
 296 /* Initializers for lock.  */
 297 #define LLL_LOCK_INITIALIZER            (0)
 298 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 299
 300 /* The states of a lock are:
 301     0  -  untaken
 302     1  -  taken by one user
 303    >1  -  taken by more users */
 304
 305 #define lll_trylock(lock)       lll_mutex_trylock (lock)
 306 #define lll_lock(lock)          lll_mutex_lock (lock)
 307 #define lll_unlock(lock)        lll_mutex_unlock (lock)
 308 #define lll_islocked(lock)      lll_mutex_islocked (lock)
 309
 310 /* The kernel notifies a process which uses CLONE_CLEARTID via futex
 311    wakeup when the clone terminates.  The memory location contains the
 312    thread ID while the clone is running and is reset to zero
 313    afterwards.  */
 314 #define lll_wait_tid(tid) \
 315   do {                                                                        \
 316     __typeof (tid) __tid;                                                     \
 317     while ((__tid = (tid)) != 0)                                              \
 318       lll_futex_wait (&(tid), __tid, LLL_SHARED);                             \
 319   } while (0)
 320
 321 extern int __lll_timedwait_tid (int *, const struct timespec *)
 322      attribute_hidden;
 323
 324 #define lll_timedwait_tid(tid, abstime) \
 325   ({                                                                          \
 326     int __res = 0;                                                            \
 327     if ((tid) != 0)                                                           \
 328       __res = __lll_timedwait_tid (&(tid), (abstime));                        \
 329     __res;                                                                    \
 330   })
 331
 332
 333 /* Conditional variable handling.  */
 334
 335 extern void __lll_cond_wait (pthread_cond_t *cond)
 336      attribute_hidden;
 337 extern int __lll_cond_timedwait (pthread_cond_t *cond,
 338                                  const struct timespec *abstime)
 339      attribute_hidden;
 340 extern void __lll_cond_wake (pthread_cond_t *cond)
 341      attribute_hidden;
 342 extern void __lll_cond_broadcast (pthread_cond_t *cond)
 343      attribute_hidden;
 344
 345 #define lll_cond_wait(cond) \
 346   __lll_cond_wait (cond)
 347 #define lll_cond_timedwait(cond, abstime) \
 348   __lll_cond_timedwait (cond, abstime)
 349 #define lll_cond_wake(cond) \
 350   __lll_cond_wake (cond)
 351 #define lll_cond_broadcast(cond) \
 352   __lll_cond_broadcast (cond)
 353
 354 #endif  /* lowlevellock.h */