nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h

   1 /* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26 #include <atomic.h>
  27
  28 #ifndef __NR_futex
  29 # define __NR_futex             221
  30 #endif
  31 #define FUTEX_WAIT              0
  32 #define FUTEX_WAKE              1
  33 #define FUTEX_REQUEUE           3
  34 #define FUTEX_CMP_REQUEUE       4
  35 #define FUTEX_WAKE_OP           5
  36 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  37 #define FUTEX_LOCK_PI           6
  38 #define FUTEX_UNLOCK_PI         7
  39 #define FUTEX_TRYLOCK_PI        8
  40 #define FUTEX_PRIVATE_FLAG      128
  41
  42 /* Values for 'private' parameter of locking macros.  Yes, the
  43    definition seems to be backwards.  But it is not.  The bit will be
  44    reversed before passing to the system call.  */
  45 #define LLL_PRIVATE     0
  46 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  47
  48
  49 /* Initializer for compatibility lock.  */
  50 #define LLL_MUTEX_LOCK_INITIALIZER (0)
  51
  52 #define lll_futex_wait(futexp, val, private) \
  53   ({                                                                          \
  54     INTERNAL_SYSCALL_DECL (__err);                                            \
  55     long int opt_flags = (FUTEX_WAIT | LLL_SHARED) ^ private;                 \
  56     long int __ret;                                                           \
  57                                                                               \
  58     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
  59                               (futexp), opt_flags, (val), 0);                 \
  60     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  61   })
  62
  63 #define lll_futex_timed_wait(futexp, val, timespec, private) \
  64   ({                                                                          \
  65     INTERNAL_SYSCALL_DECL (__err);                                            \
  66     long int opt_flags = (FUTEX_WAIT | LLL_SHARED) ^ private;                 \
  67     long int __ret;                                                           \
  68                                                                               \
  69     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
  70                               (futexp), opt_flags, (val), (timespec));        \
  71     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  72   })
  73
  74 #define lll_futex_wake(futexp, nr, private) \
  75   ({                                                                          \
  76     INTERNAL_SYSCALL_DECL (__err);                                            \
  77     long int opt_flags = (FUTEX_WAKE | LLL_SHARED) ^ private;                 \
  78     long int __ret;                                                           \
  79                                                                               \
  80     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
  81                               (futexp), opt_flags, (nr), 0);                  \
  82     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
  83   })
  84
  85 #define lll_robust_mutex_dead(futexv) \
  86   do                                                                          \
  87     {                                                                         \
  88       INTERNAL_SYSCALL_DECL (__err);                                          \
  89       int *__futexp = &(futexv);                                              \
  90                                                                               \
  91       atomic_or (__futexp, FUTEX_OWNER_DIED);                                 \
  92       INTERNAL_SYSCALL (futex, __err, 4, __futexp, FUTEX_WAKE, 1, 0);         \
  93     }                                                                         \
  94   while (0)
  95
  96 /* Returns non-zero if error happened, zero if success.  */
  97 #define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
  98   ({                                                                          \
  99     INTERNAL_SYSCALL_DECL (__err);                                            \
 100     long int __ret;                                                           \
 101                                                                               \
 102     __ret = INTERNAL_SYSCALL (futex, __err, 6,                                \
 103                               (futexp), FUTEX_CMP_REQUEUE, (nr_wake),         \
 104                               (nr_move), (mutex), (val));                     \
 105     INTERNAL_SYSCALL_ERROR_P (__ret, __err);                                  \
 106   })
 107
 108 /* Returns non-zero if error happened, zero if success.  */
 109 #define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \
 110   ({                                                                          \
 111     INTERNAL_SYSCALL_DECL (__err);                                            \
 112     long int opt_flags = (FUTEX_WAKE_OP | LLL_SHARED) ^ private;              \
 113     long int opt_flag2 = (FUTEX_OP_CLEAR_WAKE_IF_GT_ONE | LLL_SHARED)         \
 114                           ^ private;                                          \
 115     long int __ret;                                                           \
 116                                                                               \
 117     __ret = INTERNAL_SYSCALL (futex, __err, 6,                                \
 118                               (futexp), opt_flags, (nr_wake),                 \
 119                               (nr_wake2), (futexp2),                          \
 120                               opt_flag2);                                     \
 121     INTERNAL_SYSCALL_ERROR_P (__ret, __err);                                  \
 122   })
 123
 124
 125 #define lll_private_futex_wait(futexp, val) \
 126   lll_private_futex_timed_wait (futexp, val, NULL)
 127
 128
 129 #ifdef __ASSUME_PRIVATE_FUTEX
 130 # define lll_private_futex_timed_wait(futexp, val, timeout) \
 131   ({                                                                          \
 132     INTERNAL_SYSCALL_DECL (__err);                                            \
 133     long int __ret;                                                           \
 134                                                                               \
 135     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
 136                               (futexp), (FUTEX_WAIT | FUTEX_PRIVATE_FLAG),    \
 137                               (val), (timeout));                              \
 138     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
 139   })
 140
 141 # define lll_private_futex_wake(futexp, val) \
 142   ({                                                                          \
 143     INTERNAL_SYSCALL_DECL (__err);                                            \
 144     long int __ret;                                                           \
 145                                                                               \
 146     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
 147                               (futexp), (FUTEX_WAKE | FUTEX_PRIVATE_FLAG),    \
 148                               (val), 0);                                      \
 149     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
 150   })
 151 #else
 152 # define lll_private_futex_timed_wait(futexp, val, timeout) \
 153   ({                                                                          \
 154     INTERNAL_SYSCALL_DECL (__err);                                            \
 155     long int __ret;                                                           \
 156                                                                               \
 157     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
 158                               (futexp), FUTEX_WAIT, (val), (timeout));        \
 159     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
 160   })
 161
 162 # define lll_private_futex_wake(futexp, val) \
 163   ({                                                                          \
 164     INTERNAL_SYSCALL_DECL (__err);                                            \
 165     long int __ret;                                                           \
 166                                                                               \
 167     __ret = INTERNAL_SYSCALL (futex, __err, 4,                                \
 168                               (futexp), FUTEX_WAKE, (val), 0);                \
 169     INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;                 \
 170   })
 171 #endif
 172
 173 #ifdef UP
 174 # define __lll_acq_instr        ""
 175 # define __lll_rel_instr        ""
 176 #else
 177 # define __lll_acq_instr        "isync"
 178 # ifdef _ARCH_PWR4
 179 /*
 180  * Newer powerpc64 processors support the new "light weight" sync (lwsync)
 181  * So if the build is using -mcpu=[power4,power5,power5+,970] we can
 182  * safely use lwsync.
 183  */
 184 #  define __lll_rel_instr       "lwsync"
 185 # else
 186 /*
 187  * Older powerpc32 processors don't support the new "light weight"
 188  * sync (lwsync).  So the only safe option is to use normal sync
 189  * for all powerpc32 applications.
 190  */
 191 #  define __lll_rel_instr       "sync"
 192 # endif
 193 #endif
 194
 195 /* Set *futex to ID if it is 0, atomically.  Returns the old value */
 196 #define __lll_robust_trylock(futex, id) \
 197   ({ int __val;                                                               \
 198      __asm __volatile ("1:      lwarx   %0,0,%2" MUTEX_HINT_ACQ "\n"          \
 199                        "        cmpwi   0,%0,0\n"                             \
 200                        "        bne     2f\n"                                 \
 201                        "        stwcx.  %3,0,%2\n"                            \
 202                        "        bne-    1b\n"                                 \
 203                        "2:      " __lll_acq_instr                             \
 204                        : "=&r" (__val), "=m" (*futex)                         \
 205                        : "r" (futex), "r" (id), "m" (*futex)                  \
 206                        : "cr0", "memory");                                    \
 207      __val;                                                                   \
 208   })
 209
 210 #define lll_robust_mutex_trylock(lock, id) __lll_robust_trylock (&(lock), id)
 211
 212 /* Set *futex to 1 if it is 0, atomically.  Returns the old value */
 213 #define __lll_trylock(futex) __lll_robust_trylock (futex, 1)
 214
 215 #define lll_mutex_trylock(lock) __lll_trylock (&(lock))
 216
 217 /* Set *futex to 2 if it is 0, atomically.  Returns the old value */
 218 #define __lll_cond_trylock(futex) __lll_robust_trylock (futex, 2)
 219
 220 #define lll_mutex_cond_trylock(lock)    __lll_cond_trylock (&(lock))
 221
 222
 223 extern void __lll_lock_wait (int *futex) attribute_hidden;
 224 extern int __lll_robust_lock_wait (int *futex) attribute_hidden;
 225
 226 #define lll_mutex_lock(lock) \
 227   (void) ({                                                                   \
 228     int *__futex = &(lock);                                                   \
 229     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 1, 0),\
 230                           0) != 0)                                            \
 231       __lll_lock_wait (__futex);                                              \
 232   })
 233
 234 #define lll_robust_mutex_lock(lock, id) \
 235   ({                                                                          \
 236     int *__futex = &(lock);                                                   \
 237     int __val = 0;                                                            \
 238     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id,  \
 239                                                                 0), 0))       \
 240       __val = __lll_robust_lock_wait (__futex);                               \
 241     __val;                                                                    \
 242   })
 243
 244 #define lll_mutex_cond_lock(lock) \
 245   (void) ({                                                                   \
 246     int *__futex = &(lock);                                                   \
 247     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 2, 0),\
 248                           0) != 0)                                            \
 249       __lll_lock_wait (__futex);                                              \
 250   })
 251
 252 #define lll_robust_mutex_cond_lock(lock, id) \
 253   ({                                                                          \
 254     int *__futex = &(lock);                                                   \
 255     int __val = 0;                                                            \
 256     int __id = id | FUTEX_WAITERS;                                            \
 257     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, __id,\
 258                                                                 0), 0))       \
 259       __val = __lll_robust_lock_wait (__futex);                               \
 260     __val;                                                                    \
 261   })
 262
 263
 264 extern int __lll_timedlock_wait
 265   (int *futex, const struct timespec *) attribute_hidden;
 266 extern int __lll_robust_timedlock_wait
 267   (int *futex, const struct timespec *) attribute_hidden;
 268
 269 #define lll_mutex_timedlock(lock, abstime) \
 270   ({                                                                          \
 271     int *__futex = &(lock);                                                   \
 272     int __val = 0;                                                            \
 273     if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, 1, 0),\
 274                           0) != 0)                                            \
 275       __val = __lll_timedlock_wait (__futex, abstime);                        \
 276     __val;                                                                    \
 277   })
 278
 279 #define lll_robust_mutex_timedlock(lock, abstime, id) \
 280   ({                                                                          \
 281     int *__futex = &(lock);                                                   \
 282     int __val = 0;                                                            \
 283     if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id,  \
 284                                                                 0), 0))       \
 285       __val = __lll_robust_timedlock_wait (__futex, abstime);                 \
 286     __val;                                                                    \
 287   })
 288
 289 #define lll_mutex_unlock(lock) \
 290   ((void) ({                                                                  \
 291     int *__futex = &(lock);                                                   \
 292     int __val = atomic_exchange_rel (__futex, 0);                             \
 293     if (__builtin_expect (__val > 1, 0))                                      \
 294       lll_futex_wake (__futex, 1, LLL_SHARED);                                \
 295   }))
 296
 297 #define lll_robust_mutex_unlock(lock) \
 298   ((void) ({                                                                  \
 299     int *__futex = &(lock);                                                   \
 300     int __val = atomic_exchange_rel (__futex, 0);                             \
 301     if (__builtin_expect (__val & FUTEX_WAITERS, 0))                          \
 302       lll_futex_wake (__futex, 1, LLL_SHARED);                                \
 303   }))
 304
 305 #define lll_mutex_unlock_force(lock) \
 306   ((void) ({                                                                  \
 307     int *__futex = &(lock);                                                   \
 308     *__futex = 0;                                                             \
 309     __asm __volatile (__lll_rel_instr ::: "memory");                          \
 310     lll_futex_wake (__futex, 1, LLL_SHARED);                                  \
 311   }))
 312
 313 #define lll_mutex_islocked(futex) \
 314   (futex != 0)
 315
 316
 317 /* Our internal lock implementation is identical to the binary-compatible
 318    mutex implementation. */
 319
 320 /* Type for lock object.  */
 321 typedef int lll_lock_t;
 322
 323 /* Initializers for lock.  */
 324 #define LLL_LOCK_INITIALIZER            (0)
 325 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 326
 327 /* The states of a lock are:
 328     0  -  untaken
 329     1  -  taken by one user
 330    >1  -  taken by more users */
 331
 332 #define lll_trylock(lock)       lll_mutex_trylock (lock)
 333 #define lll_lock(lock)          lll_mutex_lock (lock)
 334 #define lll_unlock(lock)        lll_mutex_unlock (lock)
 335 #define lll_islocked(lock)      lll_mutex_islocked (lock)
 336
 337 /* The kernel notifies a process which uses CLONE_CLEARTID via futex
 338    wakeup when the clone terminates.  The memory location contains the
 339    thread ID while the clone is running and is reset to zero
 340    afterwards.  */
 341 #define lll_wait_tid(tid) \
 342   do {                                                                        \
 343     __typeof (tid) __tid;                                                     \
 344     while ((__tid = (tid)) != 0)                                              \
 345       lll_futex_wait (&(tid), __tid, LLL_SHARED);                             \
 346   } while (0)
 347
 348 extern int __lll_timedwait_tid (int *, const struct timespec *)
 349      attribute_hidden;
 350
 351 #define lll_timedwait_tid(tid, abstime) \
 352   ({                                                                          \
 353     int __res = 0;                                                            \
 354     if ((tid) != 0)                                                           \
 355       __res = __lll_timedwait_tid (&(tid), (abstime));                        \
 356     __res;                                                                    \
 357   })
 358
 359
 360 /* Conditional variable handling.  */
 361
 362 extern void __lll_cond_wait (pthread_cond_t *cond)
 363      attribute_hidden;
 364 extern int __lll_cond_timedwait (pthread_cond_t *cond,
 365                                  const struct timespec *abstime)
 366      attribute_hidden;
 367 extern void __lll_cond_wake (pthread_cond_t *cond)
 368      attribute_hidden;
 369 extern void __lll_cond_broadcast (pthread_cond_t *cond)
 370      attribute_hidden;
 371
 372 #define lll_cond_wait(cond) \
 373   __lll_cond_wait (cond)
 374 #define lll_cond_timedwait(cond, abstime) \
 375   __lll_cond_timedwait (cond, abstime)
 376 #define lll_cond_wake(cond) \
 377   __lll_cond_wake (cond)
 378 #define lll_cond_broadcast(cond) \
 379   __lll_cond_broadcast (cond)
 380
 381 #endif  /* lowlevellock.h */