nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h

   1 /* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #include <time.h>
  24 #include <sys/param.h>
  25 #include <bits/pthreadtypes.h>
  26
  27 #ifndef LOCK_INSTR
  28 # ifdef UP
  29 #  define LOCK_INSTR    /* nothing */
  30 # else
  31 #  define LOCK_INSTR "lock;"
  32 # endif
  33 #endif
  34
  35 #define SYS_futex               202
  36 #define FUTEX_WAIT              0
  37 #define FUTEX_WAKE              1
  38
  39
  40 /* Initializer for compatibility lock.  */
  41 #define LLL_MUTEX_LOCK_INITIALIZER (0)
  42
  43
  44 #define lll_futex_wait(futex, val) \
  45   do {                                                                        \
  46     int __ignore;                                                             \
  47     register __typeof (val) _val asm ("edx") = (val);                         \
  48     __asm __volatile ("xorq %%r10, %%r10\n\t"                                 \
  49                       "syscall"                                               \
  50                       : "=a" (__ignore)                                       \
  51                       : "0" (SYS_futex), "D" (&futex), "S" (FUTEX_WAIT),      \
  52                         "d" (_val)                                            \
  53                       : "memory", "cc", "r10", "r11", "cx");                  \
  54   } while (0)
  55
  56
  57 #define lll_futex_wake(futex, nr) \
  58   do {                                                                        \
  59     int __ignore;                                                             \
  60     register __typeof (nr) _nr asm ("edx") = (nr);                            \
  61     __asm __volatile ("syscall"                                               \
  62                       : "=a" (__ignore)                                       \
  63                       : "0" (SYS_futex), "D" (&futex), "S" (FUTEX_WAKE),      \
  64                         "d" (_nr)                                             \
  65                       : "memory", "cc", "r10", "r11", "cx");                  \
  66   } while (0)
  67
  68
  69 /* Does not preserve %eax and %ecx.  */
  70 extern int __lll_mutex_lock_wait (int *__futex, int __val) attribute_hidden;
  71 /* Does not preserver %eax, %ecx, and %edx.  */
  72 extern int __lll_mutex_timedlock_wait (int *__futex, int __val,
  73                                        const struct timespec *__abstime)
  74      attribute_hidden;
  75 /* Preserves all registers but %eax.  */
  76 extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
  77
  78
  79 #define lll_mutex_trylock(futex) \
  80   ({ unsigned char ret;                                                       \
  81      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0"                 \
  82                        : "=a" (ret), "=m" (futex)                             \
  83                        : "r" (1), "1" (futex), "0" (0)                        \
  84                        : "memory");                                           \
  85      ret; })
  86
  87
  88 #define lll_mutex_lock(futex) \
  89   (void) ({ int ignore1, ignore2;                                             \
  90             __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"                   \
  91                               "testl %0, %0\n\t"                              \
  92                               "jne 1f\n\t"                                    \
  93                               ".subsection 1\n"                               \
  94                               "1:\tleaq %2, %%rdi\n\t"                        \
  95                               "subq $128, %%rsp\n\t"                          \
  96                               "callq __lll_mutex_lock_wait\n\t"               \
  97                               "addq $128, %%rsp\n\t"                          \
  98                               "jmp 2f\n\t"                                    \
  99                               ".previous\n"                                   \
 100                               "2:"                                            \
 101                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
 102                               : "0" (1), "2" (futex)                          \
 103                               : "ax", "cx", "r11", "cc", "memory"); })
 104
 105
 106 #define lll_mutex_cond_lock(futex) \
 107   (void) ({ int ignore1, ignore2;                                             \
 108             __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"                   \
 109                               "testl %0, %0\n\t"                              \
 110                               "jne 1f\n\t"                                    \
 111                               ".subsection 1\n"                               \
 112                               "1:\tleaq %2, %%rdi\n\t"                        \
 113                               "subq $128, %%rsp\n\t"                          \
 114                               "callq __lll_mutex_lock_wait\n\t"               \
 115                               "addq $128, %%rsp\n\t"                          \
 116                               "jmp 2f\n\t"                                    \
 117                               ".previous\n"                                   \
 118                               "2:"                                            \
 119                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
 120                               : "0" (2), "2" (futex)                          \
 121                               : "ax", "cx", "r11", "cc", "memory"); })
 122
 123
 124 #define lll_mutex_timedlock(futex, timeout) \
 125   ({ int result, ignore1, ignore2, ignore3;                                   \
 126      __asm __volatile (LOCK_INSTR "xaddl %0, %4\n\t"                          \
 127                        "testl %0, %0\n\t"                                     \
 128                        "jne 1f\n\t"                                           \
 129                        ".subsection 1\n"                                      \
 130                        "1:\tleaq %4, %%rdi\n\t"                               \
 131                        "movq %7, %%rdx\n\t"                                   \
 132                        "subq $128, %%rsp\n\t"                                 \
 133                        "callq __lll_mutex_timedlock_wait\n\t"                 \
 134                        "addq $128, %%rsp\n\t"                                 \
 135                        "jmp 2f\n\t"                                           \
 136                        ".previous\n"                                          \
 137                        "2:"                                                   \
 138                        : "=a" (result), "=&D" (ignore1), "=&S" (ignore2),     \
 139                          "=&d" (ignore3), "=m" (futex)                        \
 140                        : "0" (1), "4" (futex), "m" (timeout)                  \
 141                        : "memory", "cx", "cc", "r10", "r11");                 \
 142      result; })
 143
 144
 145 #define lll_mutex_unlock(futex) \
 146   (void) ({ int ignore;                                                       \
 147             __asm __volatile (LOCK_INSTR "decl %0\n\t"                        \
 148                               "jne 1f\n\t"                                    \
 149                               ".subsection 1\n"                               \
 150                               "1:\tleaq %0, %%rdi\n\t"                        \
 151                               "subq $128, %%rsp\n\t"                          \
 152                               "callq __lll_mutex_unlock_wake\n\t"             \
 153                               "addq $128, %%rsp\n\t"                          \
 154                               "jmp 2f\n\t"                                    \
 155                               ".previous\n"                                   \
 156                               "2:"                                            \
 157                               : "=m" (futex), "=&D" (ignore)                  \
 158                               : "0" (futex)                                   \
 159                               : "ax", "cx", "r11", "cc", "memory"); })
 160
 161
 162 #define lll_mutex_islocked(futex) \
 163   (futex != 0)
 164
 165
 166 /* We have a separate internal lock implementation which is not tied
 167    to binary compatibility.  */
 168
 169 /* Type for lock object.  */
 170 typedef int lll_lock_t;
 171
 172 /* Initializers for lock.  */
 173 #define LLL_LOCK_INITIALIZER            (1)
 174 #define LLL_LOCK_INITIALIZER_LOCKED     (0)
 175
 176
 177 extern int __lll_lock_wait (int *__futex, int val) attribute_hidden;
 178 extern int __lll_unlock_wake (int *__futex) attribute_hidden;
 179 extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 180
 181
 182 /* The states of a lock are:
 183     1  -  untaken
 184     0  -  taken by one user
 185    <0  -  taken by more users */
 186
 187
 188 #if defined NOT_IN_libc || defined UP
 189 # define lll_trylock(futex) \
 190   ({ unsigned char ret;                                                       \
 191      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1; setne %0"                 \
 192                        : "=a" (ret), "=m" (futex)                             \
 193                        : "r" (0), "1" (futex), "0" (1)                        \
 194                        : "memory");                                           \
 195      ret; })
 196
 197
 198 # define lll_lock(futex) \
 199   (void) ({ int ignore1, ignore2;                                             \
 200             __asm __volatile (LOCK_INSTR "xaddl %0, %2\n\t"                   \
 201                               "jne 1f\n\t"                                    \
 202                               ".subsection 1\n"                               \
 203                               "1:\tleaq %2, %%rdi\n\t"                        \
 204                               "subq $128, %%rsp\n\t"                          \
 205                               "callq __lll_lock_wait\n\t"                     \
 206                               "addq $128, %%rsp\n\t"                          \
 207                               "jmp 2f\n\t"                                    \
 208                               ".previous\n"                                   \
 209                               "2:"                                            \
 210                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
 211                               : "0" (-1), "2" (futex)                         \
 212                               : "ax", "cx", "r11", "cc", "memory"); })
 213
 214
 215 # define lll_unlock(futex) \
 216   (void) ({ int ignore;                                                       \
 217             __asm __volatile (LOCK_INSTR "incl %0\n\t"                        \
 218                               "jng 1f\n\t"                                    \
 219                               ".subsection 1\n"                               \
 220                               "1:\tleaq %0, %%rdi\n\t"                        \
 221                               "subq $128, %%rsp\n\t"                          \
 222                               "callq __lll_unlock_wake\n\t"                   \
 223                               "addq $128, %%rsp\n\t"                          \
 224                               "jmp 2f\n\t"                                    \
 225                               ".previous\n"                                   \
 226                               "2:"                                            \
 227                               : "=m" (futex), "=&D" (ignore)                  \
 228                               : "0" (futex)                                   \
 229                               : "ax", "cx", "r11", "cc", "memory"); })
 230 #else
 231 /* Special versions of the macros for use in libc itself.  They avoid
 232    the lock prefix when the thread library is not used.
 233
 234    XXX In future we might even want to avoid it on UP machines.  */
 235
 236 # define lll_trylock(futex) \
 237   ({ unsigned char ret;                                                       \
 238      __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"          \
 239                        "je 0f\n\t"                                            \
 240                        "lock\n"                                               \
 241                        "0:\tcmpxchgl %2, %1; setne %0"                        \
 242                        : "=a" (ret), "=m" (futex)                             \
 243                        : "r" (0), "1" (futex), "0" (1)                        \
 244                        : "memory");                                           \
 245      ret; })
 246
 247
 248 # define lll_lock(futex) \
 249   (void) ({ int ignore1, ignore2;                                             \
 250             __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 251                               "je 0f\n\t"                                     \
 252                               "lock\n"                                        \
 253                               "0:\txaddl %0, %2\n\t"                          \
 254                               "jne 1f\n\t"                                    \
 255                               ".subsection 1\n"                               \
 256                               "1:\tleaq %2, %%rdi\n\t"                        \
 257                               "subq $128, %%rsp\n\t"                          \
 258                               "callq __lll_lock_wait\n\t"                     \
 259                               "addq $128, %%rsp\n\t"                          \
 260                               "jmp 2f\n\t"                                    \
 261                               ".previous\n"                                   \
 262                               "2:"                                            \
 263                               : "=S" (ignore1), "=&D" (ignore2), "=m" (futex) \
 264                               : "0" (-1), "2" (futex)                         \
 265                               : "ax", "cx", "r11", "cc", "memory"); })
 266
 267
 268 # define lll_unlock(futex) \
 269   (void) ({ int ignore;                                                       \
 270             __asm __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 271                               "je 0f\n\t"                                     \
 272                               "lock\n"                                        \
 273                               "0:\tincl %0\n\t"                       \
 274                               "jng 1f\n\t"                                    \
 275                               ".subsection 1\n"                               \
 276                               "1:\tleaq %0, %%rdi\n\t"                        \
 277                               "subq $128, %%rsp\n\t"                          \
 278                               "callq __lll_unlock_wake\n\t"                   \
 279                               "addq $128, %%rsp\n\t"                          \
 280                               "jmp 2f\n\t"                                    \
 281                               ".previous\n"                                   \
 282                               "2:"                                            \
 283                               : "=m" (futex), "=&D" (ignore)                  \
 284                               : "0" (futex)                                   \
 285                               : "ax", "cx", "r11", "cc", "memory"); })
 286 #endif
 287
 288
 289 #define lll_islocked(futex) \
 290   (futex != 0)
 291
 292
 293 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 294    wakeup when the clone terminates.  The memory location contains the
 295    thread ID while the clone is running and is reset to zero
 296    afterwards.
 297
 298    The macro parameter must not have any side effect.  */
 299 #define lll_wait_tid(tid) \
 300   do {                                                                        \
 301     int __ignore;                                                             \
 302     register __typeof (tid) _tid asm ("edx") = (tid);                         \
 303     if (_tid != 0)                                                            \
 304       __asm __volatile ("xorq %%r10, %%r10\n\t"                               \
 305                         "1:\tmovq %2, %%rax\n\t"                              \
 306                         "syscall\n\t"                                         \
 307                         "cmpl $0, (%%rdi)\n\t"                                \
 308                         "jne 1b"                                              \
 309                         : "=&a" (__ignore)                                    \
 310                         : "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid),      \
 311                           "d" (_tid)                                          \
 312                         : "memory", "cc", "r10", "r11", "cx");                \
 313   } while (0)
 314
 315 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 316      attribute_hidden;
 317 #define lll_timedwait_tid(tid, abstime) \
 318   ({                                                                          \
 319     int __result = 0;                                                         \
 320     if (tid != 0)                                                             \
 321       {                                                                       \
 322         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
 323           __result = EINVAL;                                                  \
 324         else                                                                  \
 325           __result = __lll_timedwait_tid (&tid, abstime);                     \
 326       }                                                                       \
 327     __result; })
 328
 329
 330 /* Conditional variable handling.  */
 331
 332 extern void __lll_cond_wait (pthread_cond_t *cond) attribute_hidden;
 333 extern int __lll_cond_timedwait (pthread_cond_t *cond,
 334                                  const struct timespec *abstime)
 335      attribute_hidden;
 336 extern void __lll_cond_wake (pthread_cond_t *cond) attribute_hidden;
 337 extern void __lll_cond_broadcast (pthread_cond_t *cond) attribute_hidden;
 338
 339
 340 #define lll_cond_wait(cond) \
 341   __lll_cond_wait (cond)
 342 #define lll_cond_timedwait(cond, abstime) \
 343   __lll_cond_timedwait (cond, abstime)
 344 #define lll_cond_wake(cond) \
 345   __lll_cond_wake (cond)
 346 #define lll_cond_broadcast(cond) \
 347   __lll_cond_broadcast (cond)
 348
 349
 350 #endif  /* lowlevellock.h */