nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h

   1 /* Copyright (C) 2002-2004, 2006-2008, 2009, 2012
   2    Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #ifndef __ASSEMBLER__
  24 # include <time.h>
  25 # include <sys/param.h>
  26 # include <bits/pthreadtypes.h>
  27 # include <kernel-features.h>
  28 # include <tcb-offsets.h>
  29
  30 # ifndef LOCK_INSTR
  31 #  ifdef UP
  32 #   define LOCK_INSTR   /* nothing */
  33 #  else
  34 #   define LOCK_INSTR "lock;"
  35 #  endif
  36 # endif
  37 #else
  38 # ifndef LOCK
  39 #  ifdef UP
  40 #   define LOCK
  41 #  else
  42 #   define LOCK lock
  43 #  endif
  44 # endif
  45 #endif
  46
  47 #define SYS_futex               __NR_futex
  48 #define FUTEX_WAIT              0
  49 #define FUTEX_WAKE              1
  50 #define FUTEX_CMP_REQUEUE       4
  51 #define FUTEX_WAKE_OP           5
  52 #define FUTEX_LOCK_PI           6
  53 #define FUTEX_UNLOCK_PI         7
  54 #define FUTEX_TRYLOCK_PI        8
  55 #define FUTEX_WAIT_BITSET       9
  56 #define FUTEX_WAKE_BITSET       10
  57 #define FUTEX_WAIT_REQUEUE_PI   11
  58 #define FUTEX_CMP_REQUEUE_PI    12
  59 #define FUTEX_PRIVATE_FLAG      128
  60 #define FUTEX_CLOCK_REALTIME    256
  61
  62 #define FUTEX_BITSET_MATCH_ANY  0xffffffff
  63
  64 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  65
  66 /* Values for 'private' parameter of locking macros.  Yes, the
  67    definition seems to be backwards.  But it is not.  The bit will be
  68    reversed before passing to the system call.  */
  69 #define LLL_PRIVATE     0
  70 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  71
  72 #ifndef __ASSEMBLER__
  73
  74 #if !defined NOT_IN_libc || defined IS_IN_rtld
  75 /* In libc.so or ld.so all futexes are private.  */
  76 # ifdef __ASSUME_PRIVATE_FUTEX
  77 #  define __lll_private_flag(fl, private) \
  78   ((fl) | FUTEX_PRIVATE_FLAG)
  79 # else
  80 #  define __lll_private_flag(fl, private) \
  81   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  82 # endif
  83 #else
  84 # ifdef __ASSUME_PRIVATE_FUTEX
  85 #  define __lll_private_flag(fl, private) \
  86   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  87 # else
  88 #  define __lll_private_flag(fl, private) \
  89   (__builtin_constant_p (private)                                             \
  90    ? ((private) == 0                                                          \
  91       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  92       : (fl))                                                                 \
  93    : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);                 \
  94         asm ("andl %%fs:%P1, %0" : "+r" (__fl)                                \
  95              : "i" (offsetof (struct pthread, header.private_futex)));        \
  96         __fl | (fl); }))
  97 # endif
  98 #endif
  99
 100 /* Initializer for lock.  */
 101 #define LLL_LOCK_INITIALIZER            (0)
 102 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 103 #define LLL_LOCK_INITIALIZER_WAITERS    (2)
 104
 105 /* Delay in spinlock loop.  */
 106 #define BUSY_WAIT_NOP     asm ("rep; nop")
 107
 108
 109 #define LLL_STUB_UNWIND_INFO_START \
 110         ".section       .eh_frame,\"a\",@progbits\n"            \
 111 "7:\t"  ".long  9f-8f   # Length of Common Information Entry\n" \
 112 "8:\t"  ".long  0x0     # CIE Identifier Tag\n\t"               \
 113         ".byte  0x1     # CIE Version\n\t"                      \
 114         ".ascii \"zR\\0\"       # CIE Augmentation\n\t"         \
 115         ".uleb128 0x1   # CIE Code Alignment Factor\n\t"        \
 116         ".sleb128 -8    # CIE Data Alignment Factor\n\t"        \
 117         ".byte  0x10    # CIE RA Column\n\t"                    \
 118         ".uleb128 0x1   # Augmentation size\n\t"                \
 119         ".byte  0x1b    # FDE Encoding (pcrel sdata4)\n\t"      \
 120         ".byte  0x12    # DW_CFA_def_cfa_sf\n\t"                \
 121         ".uleb128 0x7\n\t"                                      \
 122         ".sleb128 16\n\t"                                       \
 123         ".align " LP_SIZE "\n"                                  \
 124 "9:\t"  ".long  23f-10f # FDE Length\n"                         \
 125 "10:\t" ".long  10b-7b  # FDE CIE offset\n\t"                   \
 126         ".long  1b-.    # FDE initial location\n\t"             \
 127         ".long  6b-1b   # FDE address range\n\t"                \
 128         ".uleb128 0x0   # Augmentation size\n\t"                \
 129         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 130         ".uleb128 0x10\n\t"                                     \
 131         ".uleb128 12f-11f\n"                                    \
 132 "11:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 133         ".sleb128 4b-1b\n"
 134 #define LLL_STUB_UNWIND_INFO_END \
 135         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 136         ".uleb128 0x10\n\t"                                     \
 137         ".uleb128 14f-13f\n"                                    \
 138 "13:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 139         ".sleb128 4b-2b\n"                                      \
 140 "14:\t" ".byte  0x40 + (3b-2b) # DW_CFA_advance_loc\n\t"        \
 141         ".byte  0x0e    # DW_CFA_def_cfa_offset\n\t"            \
 142         ".uleb128 0\n\t"                                        \
 143         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 144         ".uleb128 0x10\n\t"                                     \
 145         ".uleb128 16f-15f\n"                                    \
 146 "15:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 147         ".sleb128 4b-3b\n"                                      \
 148 "16:\t" ".byte  0x40 + (4b-3b-1) # DW_CFA_advance_loc\n\t"      \
 149         ".byte  0x0e    # DW_CFA_def_cfa_offset\n\t"            \
 150         ".uleb128 128\n\t"                                      \
 151         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 152         ".uleb128 0x10\n\t"                                     \
 153         ".uleb128 20f-17f\n"                                    \
 154 "17:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 155         ".sleb128 19f-18f\n\t"                                  \
 156         ".byte  0x0d    # DW_OP_const4s\n"                      \
 157 "18:\t" ".4byte 4b-.\n\t"                                       \
 158         ".byte  0x1c    # DW_OP_minus\n\t"                      \
 159         ".byte  0x0d    # DW_OP_const4s\n"                      \
 160 "19:\t" ".4byte 24f-.\n\t"                                      \
 161         ".byte  0x22    # DW_OP_plus\n"                         \
 162 "20:\t" ".byte  0x40 + (5b-4b+1) # DW_CFA_advance_loc\n\t"      \
 163         ".byte  0x13    # DW_CFA_def_cfa_offset_sf\n\t"         \
 164         ".sleb128 16\n\t"                                       \
 165         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 166         ".uleb128 0x10\n\t"                                     \
 167         ".uleb128 22f-21f\n"                                    \
 168 "21:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 169         ".sleb128 4b-5b\n"                                      \
 170 "22:\t" ".align " LP_SIZE "\n"                                  \
 171 "23:\t" ".previous\n"
 172
 173 /* Unwind info for
 174    1: leaq ..., %rdi
 175    2: subq $128, %rsp
 176    3: callq ...
 177    4: addq $128, %rsp
 178    5: jmp 24f
 179    6:
 180    snippet.  */
 181 #define LLL_STUB_UNWIND_INFO_5 \
 182 LLL_STUB_UNWIND_INFO_START                                      \
 183 "12:\t" ".byte  0x40 + (2b-1b) # DW_CFA_advance_loc\n\t"        \
 184 LLL_STUB_UNWIND_INFO_END
 185
 186 /* Unwind info for
 187    1: leaq ..., %rdi
 188    0: movq ..., %rdx
 189    2: subq $128, %rsp
 190    3: callq ...
 191    4: addq $128, %rsp
 192    5: jmp 24f
 193    6:
 194    snippet.  */
 195 #define LLL_STUB_UNWIND_INFO_6 \
 196 LLL_STUB_UNWIND_INFO_START                                      \
 197 "12:\t" ".byte  0x40 + (0b-1b) # DW_CFA_advance_loc\n\t"        \
 198         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 199         ".uleb128 0x10\n\t"                                     \
 200         ".uleb128 26f-25f\n"                                    \
 201 "25:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 202         ".sleb128 4b-0b\n"                                      \
 203 "26:\t" ".byte  0x40 + (2b-0b) # DW_CFA_advance_loc\n\t"        \
 204 LLL_STUB_UNWIND_INFO_END
 205
 206
 207 #define lll_futex_wait(futex, val, private) \
 208   lll_futex_timed_wait(futex, val, NULL, private)
 209
 210
 211 #define lll_futex_timed_wait(futex, val, timeout, private) \
 212   ({                                                                          \
 213     register const struct timespec *__to __asm ("r10") = timeout;             \
 214     int __status;                                                             \
 215     register __typeof (val) _val __asm ("edx") = (val);                       \
 216     __asm __volatile ("syscall"                                               \
 217                       : "=a" (__status)                                       \
 218                       : "0" (SYS_futex), "D" (futex),                         \
 219                         "S" (__lll_private_flag (FUTEX_WAIT, private)),       \
 220                         "d" (_val), "r" (__to)                                \
 221                       : "memory", "cc", "r11", "cx");                         \
 222     __status;                                                                 \
 223   })
 224
 225
 226 #define lll_futex_wake(futex, nr, private) \
 227   do {                                                                        \
 228     int __ignore;                                                             \
 229     register __typeof (nr) _nr __asm ("edx") = (nr);                          \
 230     __asm __volatile ("syscall"                                               \
 231                       : "=a" (__ignore)                                       \
 232                       : "0" (SYS_futex), "D" (futex),                         \
 233                         "S" (__lll_private_flag (FUTEX_WAKE, private)),       \
 234                         "d" (_nr)                                             \
 235                       : "memory", "cc", "r10", "r11", "cx");                  \
 236   } while (0)
 237
 238
 239 /* NB: in the lll_trylock macro we simply return the value in %eax
 240    after the cmpxchg instruction.  In case the operation succeded this
 241    value is zero.  In case the operation failed, the cmpxchg instruction
 242    has loaded the current value of the memory work which is guaranteed
 243    to be nonzero.  */
 244 #if defined NOT_IN_libc || defined UP
 245 # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
 246 #else
 247 # define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t"      \
 248                            "je 0f\n\t"                                        \
 249                            "lock; cmpxchgl %2, %1\n\t"                        \
 250                            "jmp 1f\n\t"                                       \
 251                            "0:\tcmpxchgl %2, %1\n\t"                          \
 252                            "1:"
 253 #endif
 254
 255 #define lll_trylock(futex) \
 256   ({ int ret;                                                                 \
 257      __asm __volatile (__lll_trylock_asm                                      \
 258                        : "=a" (ret), "=m" (futex)                             \
 259                        : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
 260                          "0" (LLL_LOCK_INITIALIZER)                           \
 261                        : "memory");                                           \
 262      ret; })
 263
 264 #define lll_robust_trylock(futex, id) \
 265   ({ int ret;                                                                 \
 266      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
 267                        : "=a" (ret), "=m" (futex)                             \
 268                        : "r" (id), "m" (futex), "0" (LLL_LOCK_INITIALIZER)    \
 269                        : "memory");                                           \
 270      ret; })
 271
 272 #define lll_cond_trylock(futex) \
 273   ({ int ret;                                                                 \
 274      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
 275                        : "=a" (ret), "=m" (futex)                             \
 276                        : "r" (LLL_LOCK_INITIALIZER_WAITERS),                  \
 277                          "m" (futex), "0" (LLL_LOCK_INITIALIZER)              \
 278                        : "memory");                                           \
 279      ret; })
 280
 281 #if defined NOT_IN_libc || defined UP
 282 # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t"                \
 283                               "jnz 1f\n\t"
 284 #else
 285 # define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 286                               "je 0f\n\t"                                     \
 287                               "lock; cmpxchgl %4, %2\n\t"                     \
 288                               "jnz 1f\n\t"                                    \
 289                               "jmp 24f\n"                                     \
 290                               "0:\tcmpxchgl %4, %2\n\t"                       \
 291                               "jnz 1f\n\t"
 292 #endif
 293
 294 #define lll_lock(futex, private) \
 295   (void)                                                                      \
 296     ({ int ignore1, ignore2, ignore3;                                         \
 297        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 298          __asm __volatile (__lll_lock_asm_start                               \
 299                            ".subsection 1\n\t"                                \
 300                            ".type _L_lock_%=, @function\n"                    \
 301                            "_L_lock_%=:\n"                                    \
 302                            "1:\tlea %2, %%" RDI_LP "\n"                       \
 303                            "2:\tsub $128, %%" RSP_LP "\n"                     \
 304                            "3:\tcallq __lll_lock_wait_private\n"              \
 305                            "4:\tadd $128, %%" RSP_LP "\n"                     \
 306                            "5:\tjmp 24f\n"                                    \
 307                            "6:\t.size _L_lock_%=, 6b-1b\n\t"                  \
 308                            ".previous\n"                                      \
 309                            LLL_STUB_UNWIND_INFO_5                             \
 310                            "24:"                                              \
 311                            : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),   \
 312                              "=a" (ignore3)                                   \
 313                            : "0" (1), "m" (futex), "3" (0)                    \
 314                            : "cx", "r11", "cc", "memory");                    \
 315        else                                                                   \
 316          __asm __volatile (__lll_lock_asm_start                               \
 317                            ".subsection 1\n\t"                                \
 318                            ".type _L_lock_%=, @function\n"                    \
 319                            "_L_lock_%=:\n"                                    \
 320                            "1:\tlea %2, %%" RDI_LP "\n"                       \
 321                            "2:\tsub $128, %%" RSP_LP "\n"                     \
 322                            "3:\tcallq __lll_lock_wait\n"                      \
 323                            "4:\tadd $128, %%" RSP_LP "\n"                     \
 324                            "5:\tjmp 24f\n"                                    \
 325                            "6:\t.size _L_lock_%=, 6b-1b\n\t"                  \
 326                            ".previous\n"                                      \
 327                            LLL_STUB_UNWIND_INFO_5                             \
 328                            "24:"                                              \
 329                            : "=S" (ignore1), "=D" (ignore2), "=m" (futex),    \
 330                              "=a" (ignore3)                                   \
 331                            : "1" (1), "m" (futex), "3" (0), "0" (private)     \
 332                            : "cx", "r11", "cc", "memory");                    \
 333     })                                                                        \
 334
 335 #define lll_robust_lock(futex, id, private) \
 336   ({ int result, ignore1, ignore2;                                            \
 337     __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"                        \
 338                       "jnz 1f\n\t"                                            \
 339                       ".subsection 1\n\t"                                     \
 340                       ".type _L_robust_lock_%=, @function\n"                  \
 341                       "_L_robust_lock_%=:\n"                                  \
 342                       "1:\tlea %2, %%" RDI_LP "\n"                            \
 343                       "2:\tsub $128, %%" RSP_LP "\n"                          \
 344                       "3:\tcallq __lll_robust_lock_wait\n"                    \
 345                       "4:\tadd $128, %%" RSP_LP "\n"                          \
 346                       "5:\tjmp 24f\n"                                         \
 347                       "6:\t.size _L_robust_lock_%=, 6b-1b\n\t"                \
 348                       ".previous\n"                                           \
 349                       LLL_STUB_UNWIND_INFO_5                                  \
 350                       "24:"                                                   \
 351                       : "=S" (ignore1), "=D" (ignore2), "=m" (futex),         \
 352                         "=a" (result)                                         \
 353                       : "1" (id), "m" (futex), "3" (0), "0" (private)         \
 354                       : "cx", "r11", "cc", "memory");                         \
 355     result; })
 356
 357 #define lll_cond_lock(futex, private) \
 358   (void)                                                                      \
 359     ({ int ignore1, ignore2, ignore3;                                         \
 360        __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"                     \
 361                          "jnz 1f\n\t"                                         \
 362                          ".subsection 1\n\t"                                  \
 363                          ".type _L_cond_lock_%=, @function\n"                 \
 364                          "_L_cond_lock_%=:\n"                                 \
 365                          "1:\tlea %2, %%" RDI_LP "\n"                         \
 366                          "2:\tsub $128, %%" RSP_LP "\n"                       \
 367                          "3:\tcallq __lll_lock_wait\n"                        \
 368                          "4:\tadd $128, %%" RSP_LP "\n"                       \
 369                          "5:\tjmp 24f\n"                                      \
 370                          "6:\t.size _L_cond_lock_%=, 6b-1b\n\t"               \
 371                          ".previous\n"                                        \
 372                          LLL_STUB_UNWIND_INFO_5                               \
 373                          "24:"                                                \
 374                          : "=S" (ignore1), "=D" (ignore2), "=m" (futex),      \
 375                            "=a" (ignore3)                                     \
 376                          : "1" (2), "m" (futex), "3" (0), "0" (private)       \
 377                          : "cx", "r11", "cc", "memory");                      \
 378     })
 379
 380 #define lll_robust_cond_lock(futex, id, private) \
 381   ({ int result, ignore1, ignore2;                                            \
 382     __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t"                        \
 383                       "jnz 1f\n\t"                                            \
 384                       ".subsection 1\n\t"                                     \
 385                       ".type _L_robust_cond_lock_%=, @function\n"             \
 386                       "_L_robust_cond_lock_%=:\n"                             \
 387                       "1:\tlea %2, %%" RDI_LP "\n"                            \
 388                       "2:\tsub $128, %%" RSP_LP "\n"                          \
 389                       "3:\tcallq __lll_robust_lock_wait\n"                    \
 390                       "4:\tadd $128, %%" RSP_LP "\n"                          \
 391                       "5:\tjmp 24f\n"                                         \
 392                       "6:\t.size _L_robust_cond_lock_%=, 6b-1b\n\t"           \
 393                       ".previous\n"                                           \
 394                       LLL_STUB_UNWIND_INFO_5                                  \
 395                       "24:"                                                   \
 396                       : "=S" (ignore1), "=D" (ignore2), "=m" (futex),         \
 397                         "=a" (result)                                         \
 398                       : "1" (id | FUTEX_WAITERS), "m" (futex), "3" (0),       \
 399                         "0" (private)                                         \
 400                       : "cx", "r11", "cc", "memory");                         \
 401     result; })
 402
 403 #define lll_timedlock(futex, timeout, private) \
 404   ({ int result, ignore1, ignore2, ignore3;                                   \
 405      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t"                       \
 406                        "jnz 1f\n\t"                                           \
 407                        ".subsection 1\n\t"                                    \
 408                        ".type _L_timedlock_%=, @function\n"                   \
 409                        "_L_timedlock_%=:\n"                                   \
 410                        "1:\tlea %4, %%" RDI_LP "\n"                           \
 411                        "0:\tmov %8, %%" RDX_LP "\n"                           \
 412                        "2:\tsub $128, %%" RSP_LP "\n"                         \
 413                        "3:\tcallq __lll_timedlock_wait\n"                     \
 414                        "4:\tadd $128, %%" RSP_LP "\n"                         \
 415                        "5:\tjmp 24f\n"                                        \
 416                        "6:\t.size _L_timedlock_%=, 6b-1b\n\t"                 \
 417                        ".previous\n"                                          \
 418                        LLL_STUB_UNWIND_INFO_6                                 \
 419                        "24:"                                                  \
 420                        : "=a" (result), "=D" (ignore1), "=S" (ignore2),       \
 421                          "=&d" (ignore3), "=m" (futex)                        \
 422                        : "0" (0), "1" (1), "m" (futex), "m" (timeout),        \
 423                          "2" (private)                                        \
 424                        : "memory", "cx", "cc", "r10", "r11");                 \
 425      result; })
 426
 427 #define lll_robust_timedlock(futex, timeout, id, private) \
 428   ({ int result, ignore1, ignore2, ignore3;                                   \
 429      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t"                       \
 430                        "jnz 1f\n\t"                                           \
 431                        ".subsection 1\n\t"                                    \
 432                        ".type _L_robust_timedlock_%=, @function\n"            \
 433                        "_L_robust_timedlock_%=:\n"                            \
 434                        "1:\tlea %4, %%" RDI_LP "\n"                           \
 435                        "0:\tmov %8, %%" RDX_LP "\n"                           \
 436                        "2:\tsub $128, %%" RSP_LP "\n"                         \
 437                        "3:\tcallq __lll_robust_timedlock_wait\n"              \
 438                        "4:\tadd $128, %%" RSP_LP "\n"                         \
 439                        "5:\tjmp 24f\n"                                        \
 440                        "6:\t.size _L_robust_timedlock_%=, 6b-1b\n\t"          \
 441                        ".previous\n"                                          \
 442                        LLL_STUB_UNWIND_INFO_6                                 \
 443                        "24:"                                                  \
 444                        : "=a" (result), "=D" (ignore1), "=S" (ignore2),       \
 445                          "=&d" (ignore3), "=m" (futex)                        \
 446                        : "0" (0), "1" (id), "m" (futex), "m" (timeout),       \
 447                          "2" (private)                                        \
 448                        : "memory", "cx", "cc", "r10", "r11");                 \
 449      result; })
 450
 451 #if defined NOT_IN_libc || defined UP
 452 # define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t"                      \
 453                                 "jne 1f\n\t"
 454 #else
 455 # define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
 456                                 "je 0f\n\t"                                   \
 457                                 "lock; decl %0\n\t"                           \
 458                                 "jne 1f\n\t"                                  \
 459                                 "jmp 24f\n\t"                                 \
 460                                 "0:\tdecl %0\n\t"                             \
 461                                 "jne 1f\n\t"
 462 #endif
 463
 464 #define lll_unlock(futex, private) \
 465   (void)                                                                      \
 466     ({ int ignore;                                                            \
 467        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 468          __asm __volatile (__lll_unlock_asm_start                             \
 469                            ".subsection 1\n\t"                                \
 470                            ".type _L_unlock_%=, @function\n"                  \
 471                            "_L_unlock_%=:\n"                                  \
 472                            "1:\tlea %0, %%" RDI_LP "\n"                       \
 473                            "2:\tsub $128, %%" RSP_LP "\n"                     \
 474                            "3:\tcallq __lll_unlock_wake_private\n"            \
 475                            "4:\tadd $128, %%" RSP_LP "\n"                     \
 476                            "5:\tjmp 24f\n"                                    \
 477                            "6:\t.size _L_unlock_%=, 6b-1b\n\t"                \
 478                            ".previous\n"                                      \
 479                            LLL_STUB_UNWIND_INFO_5                             \
 480                            "24:"                                              \
 481                            : "=m" (futex), "=&D" (ignore)                     \
 482                            : "m" (futex)                                      \
 483                            : "ax", "cx", "r11", "cc", "memory");              \
 484        else                                                                   \
 485          __asm __volatile (__lll_unlock_asm_start                             \
 486                            ".subsection 1\n\t"                                \
 487                            ".type _L_unlock_%=, @function\n"                  \
 488                            "_L_unlock_%=:\n"                                  \
 489                            "1:\tlea %0, %%" RDI_LP "\n"                       \
 490                            "2:\tsub $128, %%" RSP_LP "\n"                     \
 491                            "3:\tcallq __lll_unlock_wake\n"                    \
 492                            "4:\tadd $128, %%" RSP_LP "\n"                     \
 493                            "5:\tjmp 24f\n"                                    \
 494                            "6:\t.size _L_unlock_%=, 6b-1b\n\t"                \
 495                            ".previous\n"                                      \
 496                            LLL_STUB_UNWIND_INFO_5                             \
 497                            "24:"                                              \
 498                            : "=m" (futex), "=&D" (ignore)                     \
 499                            : "m" (futex), "S" (private)                       \
 500                            : "ax", "cx", "r11", "cc", "memory");              \
 501     })
 502
 503 #define lll_robust_unlock(futex, private) \
 504   do                                                                          \
 505     {                                                                         \
 506       int ignore;                                                             \
 507       __asm __volatile (LOCK_INSTR "andl %2, %0\n\t"                          \
 508                         "jne 1f\n\t"                                          \
 509                         ".subsection 1\n\t"                                   \
 510                         ".type _L_robust_unlock_%=, @function\n"              \
 511                         "_L_robust_unlock_%=:\n"                              \
 512                         "1:\tlea %0, %%" RDI_LP "\n"                          \
 513                         "2:\tsub $128, %%" RSP_LP "\n"                        \
 514                         "3:\tcallq __lll_unlock_wake\n"                       \
 515                         "4:\tadd $128, %%" RSP_LP "\n"                        \
 516                         "5:\tjmp 24f\n"                                       \
 517                         "6:\t.size _L_robust_unlock_%=, 6b-1b\n\t"            \
 518                         ".previous\n"                                         \
 519                         LLL_STUB_UNWIND_INFO_5                                \
 520                         "24:"                                                 \
 521                         : "=m" (futex), "=&D" (ignore)                        \
 522                         : "i" (FUTEX_WAITERS), "m" (futex),                   \
 523                           "S" (private)                                       \
 524                         : "ax", "cx", "r11", "cc", "memory");                 \
 525     }                                                                         \
 526   while (0)
 527
 528 #define lll_robust_dead(futex, private) \
 529   do                                                                          \
 530     {                                                                         \
 531       int ignore;                                                             \
 532       __asm __volatile (LOCK_INSTR "orl %3, (%2)\n\t"                         \
 533                         "syscall"                                             \
 534                         : "=m" (futex), "=a" (ignore)                         \
 535                         : "D" (&(futex)), "i" (FUTEX_OWNER_DIED),             \
 536                           "S" (__lll_private_flag (FUTEX_WAKE, private)),     \
 537                           "1" (__NR_futex), "d" (1)                           \
 538                         : "cx", "r11", "cc", "memory");                       \
 539     }                                                                         \
 540   while (0)
 541
 542 /* Returns non-zero if error happened, zero if success.  */
 543 #define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
 544   ({ int __res;                                                               \
 545      register int __nr_move __asm ("r10") = nr_move;                          \
 546      register void *__mutex __asm ("r8") = mutex;                             \
 547      register int __val __asm ("r9") = val;                                   \
 548      __asm __volatile ("syscall"                                              \
 549                        : "=a" (__res)                                         \
 550                        : "0" (__NR_futex), "D" ((void *) ftx),                \
 551                          "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,          \
 552                                                   private)), "d" (nr_wake),   \
 553                          "r" (__nr_move), "r" (__mutex), "r" (__val)          \
 554                        : "cx", "r11", "cc", "memory");                        \
 555      __res < 0; })
 556
 557 #define lll_islocked(futex) \
 558   (futex != LLL_LOCK_INITIALIZER)
 559
 560
 561 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 562    wakeup when the clone terminates.  The memory location contains the
 563    thread ID while the clone is running and is reset to zero
 564    afterwards.
 565
 566    The macro parameter must not have any side effect.  */
 567 #define lll_wait_tid(tid) \
 568   do {                                                                        \
 569     int __ignore;                                                             \
 570     register __typeof (tid) _tid asm ("edx") = (tid);                         \
 571     if (_tid != 0)                                                            \
 572       __asm __volatile ("xorq %%r10, %%r10\n\t"                               \
 573                         "1:\tmovq %2, %%rax\n\t"                              \
 574                         "syscall\n\t"                                         \
 575                         "cmpl $0, (%%rdi)\n\t"                                \
 576                         "jne 1b"                                              \
 577                         : "=&a" (__ignore)                                    \
 578                         : "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid),      \
 579                           "d" (_tid)                                          \
 580                         : "memory", "cc", "r10", "r11", "cx");                \
 581   } while (0)
 582
 583 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 584      attribute_hidden;
 585 #define lll_timedwait_tid(tid, abstime) \
 586   ({                                                                          \
 587     int __result = 0;                                                         \
 588     if (tid != 0)                                                             \
 589       {                                                                       \
 590         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
 591           __result = EINVAL;                                                  \
 592         else                                                                  \
 593           __result = __lll_timedwait_tid (&tid, abstime);                     \
 594       }                                                                       \
 595     __result; })
 596
 597 #endif  /* !__ASSEMBLER__ */
 598
 599 #endif  /* lowlevellock.h */