include/atomic.h

   1 /* Internal macros for atomic operations for GNU C Library.
   2    Copyright (C) 2002-2023 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #ifndef _ATOMIC_H
  20 #define _ATOMIC_H       1
  21
  22 /* This header defines three types of macros:
  23
  24    - atomic arithmetic and logic operation on memory.  They all
  25      have the prefix "atomic_".
  26
  27    - conditionally atomic operations of the same kinds.  These
  28      always behave identical but can be faster when atomicity
  29      is not really needed since only one thread has access to
  30      the memory location.  In that case the code is slower in
  31      the multi-thread case.  The interfaces have the prefix
  32      "catomic_".
  33
  34    - support functions like barriers.  They also have the prefix
  35      "atomic_".
  36
  37    Architectures must provide a few lowlevel macros (the compare
  38    and exchange definitions).  All others are optional.  They
  39    should only be provided if the architecture has specific
  40    support for the operation.
  41
  42    As <atomic.h> macros are usually heavily nested and often use local
  43    variables to make sure side-effects are evaluated properly, use for
  44    macro local variables a per-macro unique prefix.  This file uses
  45    __atgN_ prefix where N is different in each macro.  */
  46
  47 #include <stdlib.h>
  48
  49 #include <atomic-machine.h>
  50
  51 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
  52    bit width of *MEM.  The calling macro puts parens around MEM
  53    and following args.  */
  54 #define __atomic_val_bysize(pre, post, mem, ...)                              \
  55   ({                                                                          \
  56     __typeof ((__typeof (*(mem))) *(mem)) __atg1_result;                      \
  57     if (sizeof (*mem) == 1)                                                   \
  58       __atg1_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  59     else if (sizeof (*mem) == 2)                                              \
  60       __atg1_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  61     else if (sizeof (*mem) == 4)                                              \
  62       __atg1_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  63     else if (sizeof (*mem) == 8)                                              \
  64       __atg1_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  65     else                                                                      \
  66       abort ();                                                               \
  67     __atg1_result;                                                            \
  68   })
  69 #define __atomic_bool_bysize(pre, post, mem, ...)                             \
  70   ({                                                                          \
  71     int __atg2_result;                                                        \
  72     if (sizeof (*mem) == 1)                                                   \
  73       __atg2_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  74     else if (sizeof (*mem) == 2)                                              \
  75       __atg2_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  76     else if (sizeof (*mem) == 4)                                              \
  77       __atg2_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  78     else if (sizeof (*mem) == 8)                                              \
  79       __atg2_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  80     else                                                                      \
  81       abort ();                                                               \
  82     __atg2_result;                                                            \
  83   })
  84
  85
  86 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  87    Return the old *MEM value.  */
  88 #if !defined atomic_compare_and_exchange_val_acq \
  89     && defined __arch_compare_and_exchange_val_32_acq
  90 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  91   __atomic_val_bysize (__arch_compare_and_exchange_val,acq,                   \
  92                        mem, newval, oldval)
  93 #endif
  94
  95
  96 #ifndef catomic_compare_and_exchange_val_acq
  97 # ifdef __arch_c_compare_and_exchange_val_32_acq
  98 #  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  99   __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq,                 \
 100                        mem, newval, oldval)
 101 # else
 102 #  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 103   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 104 # endif
 105 #endif
 106
 107
 108 #ifndef catomic_compare_and_exchange_val_rel
 109 # ifndef atomic_compare_and_exchange_val_rel
 110 #  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)           \
 111   catomic_compare_and_exchange_val_acq (mem, newval, oldval)
 112 # else
 113 #  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)           \
 114   atomic_compare_and_exchange_val_rel (mem, newval, oldval)
 115 # endif
 116 #endif
 117
 118
 119 #ifndef atomic_compare_and_exchange_val_rel
 120 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval)             \
 121   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 122 #endif
 123
 124
 125 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
 126    Return zero if *MEM was changed or non-zero if no exchange happened.  */
 127 #ifndef atomic_compare_and_exchange_bool_acq
 128 # ifdef __arch_compare_and_exchange_bool_32_acq
 129 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 130   __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq,                 \
 131                         mem, newval, oldval)
 132 # else
 133 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 134   ({ /* Cannot use __oldval here, because macros later in this file might     \
 135         call this macro with __oldval argument.  */                           \
 136      __typeof (oldval) __atg3_old = (oldval);                                 \
 137      atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old)            \
 138        != __atg3_old;                                                         \
 139   })
 140 # endif
 141 #endif
 142
 143
 144 #ifndef catomic_compare_and_exchange_bool_acq
 145 # ifdef __arch_c_compare_and_exchange_bool_32_acq
 146 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 147   __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq,               \
 148                         mem, newval, oldval)
 149 # else
 150 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 151   ({ /* Cannot use __oldval here, because macros later in this file might     \
 152         call this macro with __oldval argument.  */                           \
 153      __typeof (oldval) __atg4_old = (oldval);                                 \
 154      catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old)           \
 155        != __atg4_old;                                                         \
 156   })
 157 # endif
 158 #endif
 159
 160
 161 /* Store NEWVALUE in *MEM and return the old value.  */
 162 #ifndef atomic_exchange_acq
 163 # define atomic_exchange_acq(mem, newvalue) \
 164   ({ __typeof ((__typeof (*(mem))) *(mem)) __atg5_oldval;                     \
 165      __typeof (mem) __atg5_memp = (mem);                                      \
 166      __typeof ((__typeof (*(mem))) *(mem)) __atg5_value = (newvalue);         \
 167                                                                               \
 168      do                                                                       \
 169        __atg5_oldval = *__atg5_memp;                                          \
 170      while (__builtin_expect                                                  \
 171             (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
 172                                                    __atg5_oldval), 0));       \
 173                                                                               \
 174      __atg5_oldval; })
 175 #endif
 176
 177 #ifndef atomic_exchange_rel
 178 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
 179 #endif
 180
 181
 182 /* Add VALUE to *MEM and return the old value of *MEM.  */
 183 #ifndef atomic_exchange_and_add_acq
 184 # ifdef atomic_exchange_and_add
 185 #  define atomic_exchange_and_add_acq(mem, value) \
 186   atomic_exchange_and_add (mem, value)
 187 # else
 188 #  define atomic_exchange_and_add_acq(mem, value) \
 189   ({ __typeof (*(mem)) __atg6_oldval;                                         \
 190      __typeof (mem) __atg6_memp = (mem);                                      \
 191      __typeof (*(mem)) __atg6_value = (value);                                \
 192                                                                               \
 193      do                                                                       \
 194        __atg6_oldval = *__atg6_memp;                                          \
 195      while (__builtin_expect                                                  \
 196             (atomic_compare_and_exchange_bool_acq (__atg6_memp,               \
 197                                                    __atg6_oldval              \
 198                                                    + __atg6_value,            \
 199                                                    __atg6_oldval), 0));       \
 200                                                                               \
 201      __atg6_oldval; })
 202 # endif
 203 #endif
 204
 205 #ifndef atomic_exchange_and_add_rel
 206 # define atomic_exchange_and_add_rel(mem, value) \
 207   atomic_exchange_and_add_acq(mem, value)
 208 #endif
 209
 210 #ifndef atomic_exchange_and_add
 211 # define atomic_exchange_and_add(mem, value) \
 212   atomic_exchange_and_add_acq(mem, value)
 213 #endif
 214
 215 #ifndef catomic_exchange_and_add
 216 # define catomic_exchange_and_add(mem, value) \
 217   ({ __typeof (*(mem)) __atg7_oldv;                                           \
 218      __typeof (mem) __atg7_memp = (mem);                                      \
 219      __typeof (*(mem)) __atg7_value = (value);                                \
 220                                                                               \
 221      do                                                                       \
 222        __atg7_oldv = *__atg7_memp;                                            \
 223      while (__builtin_expect                                                  \
 224             (catomic_compare_and_exchange_bool_acq (__atg7_memp,              \
 225                                                     __atg7_oldv               \
 226                                                     + __atg7_value,           \
 227                                                     __atg7_oldv), 0));        \
 228                                                                               \
 229      __atg7_oldv; })
 230 #endif
 231
 232
 233 #ifndef atomic_max
 234 # define atomic_max(mem, value) \
 235   do {                                                                        \
 236     __typeof (*(mem)) __atg8_oldval;                                          \
 237     __typeof (mem) __atg8_memp = (mem);                                       \
 238     __typeof (*(mem)) __atg8_value = (value);                                 \
 239     do {                                                                      \
 240       __atg8_oldval = *__atg8_memp;                                           \
 241       if (__atg8_oldval >= __atg8_value)                                      \
 242         break;                                                                \
 243     } while (__builtin_expect                                                 \
 244              (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
 245                                                     __atg8_oldval), 0));      \
 246   } while (0)
 247 #endif
 248
 249
 250 #ifndef catomic_max
 251 # define catomic_max(mem, value) \
 252   do {                                                                        \
 253     __typeof (*(mem)) __atg9_oldv;                                            \
 254     __typeof (mem) __atg9_memp = (mem);                                       \
 255     __typeof (*(mem)) __atg9_value = (value);                                 \
 256     do {                                                                      \
 257       __atg9_oldv = *__atg9_memp;                                             \
 258       if (__atg9_oldv >= __atg9_value)                                        \
 259         break;                                                                \
 260     } while (__builtin_expect                                                 \
 261              (catomic_compare_and_exchange_bool_acq (__atg9_memp,             \
 262                                                      __atg9_value,            \
 263                                                      __atg9_oldv), 0));       \
 264   } while (0)
 265 #endif
 266
 267
 268 #ifndef atomic_min
 269 # define atomic_min(mem, value) \
 270   do {                                                                        \
 271     __typeof (*(mem)) __atg10_oldval;                                         \
 272     __typeof (mem) __atg10_memp = (mem);                                      \
 273     __typeof (*(mem)) __atg10_value = (value);                                \
 274     do {                                                                      \
 275       __atg10_oldval = *__atg10_memp;                                         \
 276       if (__atg10_oldval <= __atg10_value)                                    \
 277         break;                                                                \
 278     } while (__builtin_expect                                                 \
 279              (atomic_compare_and_exchange_bool_acq (__atg10_memp,             \
 280                                                     __atg10_value,            \
 281                                                     __atg10_oldval), 0));     \
 282   } while (0)
 283 #endif
 284
 285
 286 #ifndef atomic_add
 287 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
 288 #endif
 289
 290
 291 #ifndef catomic_add
 292 # define catomic_add(mem, value) \
 293   (void) catomic_exchange_and_add ((mem), (value))
 294 #endif
 295
 296
 297 #ifndef atomic_increment
 298 # define atomic_increment(mem) atomic_add ((mem), 1)
 299 #endif
 300
 301
 302 #ifndef catomic_increment
 303 # define catomic_increment(mem) catomic_add ((mem), 1)
 304 #endif
 305
 306
 307 #ifndef atomic_increment_val
 308 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
 309 #endif
 310
 311
 312 #ifndef catomic_increment_val
 313 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
 314 #endif
 315
 316
 317 /* Add one to *MEM and return true iff it's now zero.  */
 318 #ifndef atomic_increment_and_test
 319 # define atomic_increment_and_test(mem) \
 320   (atomic_exchange_and_add ((mem), 1) + 1 == 0)
 321 #endif
 322
 323
 324 #ifndef atomic_decrement
 325 # define atomic_decrement(mem) atomic_add ((mem), -1)
 326 #endif
 327
 328
 329 #ifndef catomic_decrement
 330 # define catomic_decrement(mem) catomic_add ((mem), -1)
 331 #endif
 332
 333
 334 #ifndef atomic_decrement_val
 335 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
 336 #endif
 337
 338
 339 #ifndef catomic_decrement_val
 340 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
 341 #endif
 342
 343
 344 /* Subtract 1 from *MEM and return true iff it's now zero.  */
 345 #ifndef atomic_decrement_and_test
 346 # define atomic_decrement_and_test(mem) \
 347   (atomic_exchange_and_add ((mem), -1) == 1)
 348 #endif
 349
 350
 351 /* Decrement *MEM if it is > 0, and return the old value.  */
 352 #ifndef atomic_decrement_if_positive
 353 # define atomic_decrement_if_positive(mem) \
 354   ({ __typeof (*(mem)) __atg11_oldval;                                        \
 355      __typeof (mem) __atg11_memp = (mem);                                     \
 356                                                                               \
 357      do                                                                       \
 358        {                                                                      \
 359          __atg11_oldval = *__atg11_memp;                                      \
 360          if (__glibc_unlikely (__atg11_oldval <= 0))                          \
 361            break;                                                             \
 362        }                                                                      \
 363      while (__builtin_expect                                                  \
 364             (atomic_compare_and_exchange_bool_acq (__atg11_memp,              \
 365                                                    __atg11_oldval - 1,        \
 366                                                    __atg11_oldval), 0));      \
 367      __atg11_oldval; })
 368 #endif
 369
 370
 371 #ifndef atomic_add_negative
 372 # define atomic_add_negative(mem, value)                                      \
 373   ({ __typeof (value) __atg12_value = (value);                                \
 374      atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
 375 #endif
 376
 377
 378 #ifndef atomic_add_zero
 379 # define atomic_add_zero(mem, value)                                          \
 380   ({ __typeof (value) __atg13_value = (value);                                \
 381      atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
 382 #endif
 383
 384
 385 #ifndef atomic_bit_set
 386 # define atomic_bit_set(mem, bit) \
 387   (void) atomic_bit_test_set(mem, bit)
 388 #endif
 389
 390
 391 #ifndef atomic_bit_test_set
 392 # define atomic_bit_test_set(mem, bit) \
 393   ({ __typeof (*(mem)) __atg14_old;                                           \
 394      __typeof (mem) __atg14_memp = (mem);                                     \
 395      __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit));       \
 396                                                                               \
 397      do                                                                       \
 398        __atg14_old = (*__atg14_memp);                                         \
 399      while (__builtin_expect                                                  \
 400             (atomic_compare_and_exchange_bool_acq (__atg14_memp,              \
 401                                                    __atg14_old | __atg14_mask,\
 402                                                    __atg14_old), 0));         \
 403                                                                               \
 404      __atg14_old & __atg14_mask; })
 405 #endif
 406
 407 /* Atomically *mem &= mask.  */
 408 #ifndef atomic_and
 409 # define atomic_and(mem, mask) \
 410   do {                                                                        \
 411     __typeof (*(mem)) __atg15_old;                                            \
 412     __typeof (mem) __atg15_memp = (mem);                                      \
 413     __typeof (*(mem)) __atg15_mask = (mask);                                  \
 414                                                                               \
 415     do                                                                        \
 416       __atg15_old = (*__atg15_memp);                                          \
 417     while (__builtin_expect                                                   \
 418            (atomic_compare_and_exchange_bool_acq (__atg15_memp,               \
 419                                                   __atg15_old & __atg15_mask, \
 420                                                   __atg15_old), 0));          \
 421   } while (0)
 422 #endif
 423
 424 #ifndef catomic_and
 425 # define catomic_and(mem, mask) \
 426   do {                                                                        \
 427     __typeof (*(mem)) __atg20_old;                                            \
 428     __typeof (mem) __atg20_memp = (mem);                                      \
 429     __typeof (*(mem)) __atg20_mask = (mask);                                  \
 430                                                                               \
 431     do                                                                        \
 432       __atg20_old = (*__atg20_memp);                                          \
 433     while (__builtin_expect                                                   \
 434            (catomic_compare_and_exchange_bool_acq (__atg20_memp,              \
 435                                                    __atg20_old & __atg20_mask,\
 436                                                    __atg20_old), 0));         \
 437   } while (0)
 438 #endif
 439
 440 /* Atomically *mem &= mask and return the old value of *mem.  */
 441 #ifndef atomic_and_val
 442 # define atomic_and_val(mem, mask) \
 443   ({ __typeof (*(mem)) __atg16_old;                                           \
 444      __typeof (mem) __atg16_memp = (mem);                                     \
 445      __typeof (*(mem)) __atg16_mask = (mask);                                 \
 446                                                                               \
 447      do                                                                       \
 448        __atg16_old = (*__atg16_memp);                                         \
 449      while (__builtin_expect                                                  \
 450             (atomic_compare_and_exchange_bool_acq (__atg16_memp,              \
 451                                                    __atg16_old & __atg16_mask,\
 452                                                    __atg16_old), 0));         \
 453                                                                               \
 454      __atg16_old; })
 455 #endif
 456
 457 /* Atomically *mem |= mask and return the old value of *mem.  */
 458 #ifndef atomic_or
 459 # define atomic_or(mem, mask) \
 460   do {                                                                        \
 461     __typeof (*(mem)) __atg17_old;                                            \
 462     __typeof (mem) __atg17_memp = (mem);                                      \
 463     __typeof (*(mem)) __atg17_mask = (mask);                                  \
 464                                                                               \
 465     do                                                                        \
 466       __atg17_old = (*__atg17_memp);                                          \
 467     while (__builtin_expect                                                   \
 468            (atomic_compare_and_exchange_bool_acq (__atg17_memp,               \
 469                                                   __atg17_old | __atg17_mask, \
 470                                                   __atg17_old), 0));          \
 471   } while (0)
 472 #endif
 473
 474 #ifndef catomic_or
 475 # define catomic_or(mem, mask) \
 476   do {                                                                        \
 477     __typeof (*(mem)) __atg18_old;                                            \
 478     __typeof (mem) __atg18_memp = (mem);                                      \
 479     __typeof (*(mem)) __atg18_mask = (mask);                                  \
 480                                                                               \
 481     do                                                                        \
 482       __atg18_old = (*__atg18_memp);                                          \
 483     while (__builtin_expect                                                   \
 484            (catomic_compare_and_exchange_bool_acq (__atg18_memp,              \
 485                                                    __atg18_old | __atg18_mask,\
 486                                                    __atg18_old), 0));         \
 487   } while (0)
 488 #endif
 489
 490 /* Atomically *mem |= mask and return the old value of *mem.  */
 491 #ifndef atomic_or_val
 492 # define atomic_or_val(mem, mask) \
 493   ({ __typeof (*(mem)) __atg19_old;                                           \
 494      __typeof (mem) __atg19_memp = (mem);                                     \
 495      __typeof (*(mem)) __atg19_mask = (mask);                                 \
 496                                                                               \
 497      do                                                                       \
 498        __atg19_old = (*__atg19_memp);                                         \
 499      while (__builtin_expect                                                  \
 500             (atomic_compare_and_exchange_bool_acq (__atg19_memp,              \
 501                                                    __atg19_old | __atg19_mask,\
 502                                                    __atg19_old), 0));         \
 503                                                                               \
 504      __atg19_old; })
 505 #endif
 506
 507 #ifndef atomic_full_barrier
 508 # define atomic_full_barrier() __asm ("" ::: "memory")
 509 #endif
 510
 511
 512 #ifndef atomic_read_barrier
 513 # define atomic_read_barrier() atomic_full_barrier ()
 514 #endif
 515
 516
 517 #ifndef atomic_write_barrier
 518 # define atomic_write_barrier() atomic_full_barrier ()
 519 #endif
 520
 521
 522 #ifndef atomic_forced_read
 523 # define atomic_forced_read(x) \
 524   ({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; })
 525 #endif
 526
 527 /* This is equal to 1 iff the architecture supports 64b atomic operations.  */
 528 #ifndef __HAVE_64B_ATOMICS
 529 #error Unable to determine if 64-bit atomics are present.
 530 #endif
 531
 532 /* The following functions are a subset of the atomic operations provided by
 533    C11.  Usually, a function named atomic_OP_MO(args) is equivalent to C11's
 534    atomic_OP_explicit(args, memory_order_MO); exceptions noted below.  */
 535
 536 /* Each arch can request to use compiler built-ins for C11 atomics.  If it
 537    does, all atomics will be based on these.  */
 538 #if USE_ATOMIC_COMPILER_BUILTINS
 539
 540 /* We require 32b atomic operations; some archs also support 64b atomic
 541    operations.  */
 542 void __atomic_link_error (void);
 543 # if __HAVE_64B_ATOMICS == 1
 544 #  define __atomic_check_size(mem) \
 545    if ((sizeof (*mem) != 4) && (sizeof (*mem) != 8))                          \
 546      __atomic_link_error ();
 547 # else
 548 #  define __atomic_check_size(mem) \
 549    if (sizeof (*mem) != 4)                                                    \
 550      __atomic_link_error ();
 551 # endif
 552 /* We additionally provide 8b and 16b atomic loads and stores; we do not yet
 553    need other atomic operations of such sizes, and restricting the support to
 554    loads and stores makes this easier for archs that do not have native
 555    support for atomic operations to less-than-word-sized data.  */
 556 # if __HAVE_64B_ATOMICS == 1
 557 #  define __atomic_check_size_ls(mem) \
 558    if ((sizeof (*mem) != 1) && (sizeof (*mem) != 2) && (sizeof (*mem) != 4)   \
 559        && (sizeof (*mem) != 8))                                               \
 560      __atomic_link_error ();
 561 # else
 562 #  define __atomic_check_size_ls(mem) \
 563    if ((sizeof (*mem) != 1) && (sizeof (*mem) != 2) && sizeof (*mem) != 4)    \
 564      __atomic_link_error ();
 565 # endif
 566
 567 # define atomic_thread_fence_acquire() \
 568   __atomic_thread_fence (__ATOMIC_ACQUIRE)
 569 # define atomic_thread_fence_release() \
 570   __atomic_thread_fence (__ATOMIC_RELEASE)
 571 # define atomic_thread_fence_seq_cst() \
 572   __atomic_thread_fence (__ATOMIC_SEQ_CST)
 573
 574 # define atomic_load_relaxed(mem) \
 575   ({ __atomic_check_size_ls((mem));                                           \
 576      __atomic_load_n ((mem), __ATOMIC_RELAXED); })
 577 # define atomic_load_acquire(mem) \
 578   ({ __atomic_check_size_ls((mem));                                           \
 579      __atomic_load_n ((mem), __ATOMIC_ACQUIRE); })
 580
 581 # define atomic_store_relaxed(mem, val) \
 582   do {                                                                        \
 583     __atomic_check_size_ls((mem));                                            \
 584     __atomic_store_n ((mem), (val), __ATOMIC_RELAXED);                        \
 585   } while (0)
 586 # define atomic_store_release(mem, val) \
 587   do {                                                                        \
 588     __atomic_check_size_ls((mem));                                            \
 589     __atomic_store_n ((mem), (val), __ATOMIC_RELEASE);                        \
 590   } while (0)
 591
 592 /* On failure, this CAS has memory_order_relaxed semantics.  */
 593 # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \
 594   ({ __atomic_check_size((mem));                                              \
 595   __atomic_compare_exchange_n ((mem), (expected), (desired), 1,               \
 596     __ATOMIC_RELAXED, __ATOMIC_RELAXED); })
 597 # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \
 598   ({ __atomic_check_size((mem));                                              \
 599   __atomic_compare_exchange_n ((mem), (expected), (desired), 1,               \
 600     __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); })
 601 # define atomic_compare_exchange_weak_release(mem, expected, desired) \
 602   ({ __atomic_check_size((mem));                                              \
 603   __atomic_compare_exchange_n ((mem), (expected), (desired), 1,               \
 604     __ATOMIC_RELEASE, __ATOMIC_RELAXED); })
 605
 606 # define atomic_exchange_relaxed(mem, desired) \
 607   ({ __atomic_check_size((mem));                                              \
 608   __atomic_exchange_n ((mem), (desired), __ATOMIC_RELAXED); })
 609 # define atomic_exchange_acquire(mem, desired) \
 610   ({ __atomic_check_size((mem));                                              \
 611   __atomic_exchange_n ((mem), (desired), __ATOMIC_ACQUIRE); })
 612 # define atomic_exchange_release(mem, desired) \
 613   ({ __atomic_check_size((mem));                                              \
 614   __atomic_exchange_n ((mem), (desired), __ATOMIC_RELEASE); })
 615
 616 # define atomic_fetch_add_relaxed(mem, operand) \
 617   ({ __atomic_check_size((mem));                                              \
 618   __atomic_fetch_add ((mem), (operand), __ATOMIC_RELAXED); })
 619 # define atomic_fetch_add_acquire(mem, operand) \
 620   ({ __atomic_check_size((mem));                                              \
 621   __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQUIRE); })
 622 # define atomic_fetch_add_release(mem, operand) \
 623   ({ __atomic_check_size((mem));                                              \
 624   __atomic_fetch_add ((mem), (operand), __ATOMIC_RELEASE); })
 625 # define atomic_fetch_add_acq_rel(mem, operand) \
 626   ({ __atomic_check_size((mem));                                              \
 627   __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQ_REL); })
 628
 629 # define atomic_fetch_and_relaxed(mem, operand) \
 630   ({ __atomic_check_size((mem));                                              \
 631   __atomic_fetch_and ((mem), (operand), __ATOMIC_RELAXED); })
 632 # define atomic_fetch_and_acquire(mem, operand) \
 633   ({ __atomic_check_size((mem));                                              \
 634   __atomic_fetch_and ((mem), (operand), __ATOMIC_ACQUIRE); })
 635 # define atomic_fetch_and_release(mem, operand) \
 636   ({ __atomic_check_size((mem));                                              \
 637   __atomic_fetch_and ((mem), (operand), __ATOMIC_RELEASE); })
 638
 639 # define atomic_fetch_or_relaxed(mem, operand) \
 640   ({ __atomic_check_size((mem));                                              \
 641   __atomic_fetch_or ((mem), (operand), __ATOMIC_RELAXED); })
 642 # define atomic_fetch_or_acquire(mem, operand) \
 643   ({ __atomic_check_size((mem));                                              \
 644   __atomic_fetch_or ((mem), (operand), __ATOMIC_ACQUIRE); })
 645 # define atomic_fetch_or_release(mem, operand) \
 646   ({ __atomic_check_size((mem));                                              \
 647   __atomic_fetch_or ((mem), (operand), __ATOMIC_RELEASE); })
 648
 649 # define atomic_fetch_xor_release(mem, operand) \
 650   ({ __atomic_check_size((mem));                                              \
 651   __atomic_fetch_xor ((mem), (operand), __ATOMIC_RELEASE); })
 652
 653 #else /* !USE_ATOMIC_COMPILER_BUILTINS  */
 654
 655 /* By default, we assume that read, write, and full barriers are equivalent
 656    to acquire, release, and seq_cst barriers.  Archs for which this does not
 657    hold have to provide custom definitions of the fences.  */
 658 # ifndef atomic_thread_fence_acquire
 659 #  define atomic_thread_fence_acquire() atomic_read_barrier ()
 660 # endif
 661 # ifndef atomic_thread_fence_release
 662 #  define atomic_thread_fence_release() atomic_write_barrier ()
 663 # endif
 664 # ifndef atomic_thread_fence_seq_cst
 665 #  define atomic_thread_fence_seq_cst() atomic_full_barrier ()
 666 # endif
 667
 668 # ifndef atomic_load_relaxed
 669 #  define atomic_load_relaxed(mem) \
 670    ({ __typeof ((__typeof (*(mem))) *(mem)) __atg100_val;                     \
 671    __asm ("" : "=r" (__atg100_val) : "0" (*(mem)));                           \
 672    __atg100_val; })
 673 # endif
 674 # ifndef atomic_load_acquire
 675 #  define atomic_load_acquire(mem) \
 676    ({ __typeof (*(mem)) __atg101_val = atomic_load_relaxed (mem);             \
 677    atomic_thread_fence_acquire ();                                            \
 678    __atg101_val; })
 679 # endif
 680
 681 # ifndef atomic_store_relaxed
 682 /* XXX Use inline asm here?  */
 683 #  define atomic_store_relaxed(mem, val) do { *(mem) = (val); } while (0)
 684 # endif
 685 # ifndef atomic_store_release
 686 #  define atomic_store_release(mem, val) \
 687    do {                                                                       \
 688      atomic_thread_fence_release ();                                          \
 689      atomic_store_relaxed ((mem), (val));                                     \
 690    } while (0)
 691 # endif
 692
 693 /* On failure, this CAS has memory_order_relaxed semantics.  */
 694 /* XXX This potentially has one branch more than necessary, but archs
 695    currently do not define a CAS that returns both the previous value and
 696    the success flag.  */
 697 # ifndef atomic_compare_exchange_weak_acquire
 698 #  define atomic_compare_exchange_weak_acquire(mem, expected, desired) \
 699    ({ typeof (*(expected)) __atg102_expected = *(expected);                   \
 700    *(expected) =                                                              \
 701      atomic_compare_and_exchange_val_acq ((mem), (desired), *(expected));     \
 702    *(expected) == __atg102_expected; })
 703 # endif
 704 # ifndef atomic_compare_exchange_weak_relaxed
 705 /* XXX Fall back to CAS with acquire MO because archs do not define a weaker
 706    CAS.  */
 707 #  define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \
 708    atomic_compare_exchange_weak_acquire ((mem), (expected), (desired))
 709 # endif
 710 # ifndef atomic_compare_exchange_weak_release
 711 #  define atomic_compare_exchange_weak_release(mem, expected, desired) \
 712    ({ typeof (*(expected)) __atg103_expected = *(expected);                   \
 713    *(expected) =                                                              \
 714      atomic_compare_and_exchange_val_rel ((mem), (desired), *(expected));     \
 715    *(expected) == __atg103_expected; })
 716 # endif
 717
 718 /* XXX Fall back to acquire MO because archs do not define a weaker
 719    atomic_exchange.  */
 720 # ifndef atomic_exchange_relaxed
 721 #  define atomic_exchange_relaxed(mem, val) \
 722    atomic_exchange_acq ((mem), (val))
 723 # endif
 724 # ifndef atomic_exchange_acquire
 725 #  define atomic_exchange_acquire(mem, val) \
 726    atomic_exchange_acq ((mem), (val))
 727 # endif
 728 # ifndef atomic_exchange_release
 729 #  define atomic_exchange_release(mem, val) \
 730    atomic_exchange_rel ((mem), (val))
 731 # endif
 732
 733 # ifndef atomic_fetch_add_acquire
 734 #  define atomic_fetch_add_acquire(mem, operand) \
 735    atomic_exchange_and_add_acq ((mem), (operand))
 736 # endif
 737 # ifndef atomic_fetch_add_relaxed
 738 /* XXX Fall back to acquire MO because the MO semantics of
 739    atomic_exchange_and_add are not documented; the generic version falls back
 740    to atomic_exchange_and_add_acq if atomic_exchange_and_add is not defined,
 741    and vice versa.  */
 742 #  define atomic_fetch_add_relaxed(mem, operand) \
 743    atomic_fetch_add_acquire ((mem), (operand))
 744 # endif
 745 # ifndef atomic_fetch_add_release
 746 #  define atomic_fetch_add_release(mem, operand) \
 747    atomic_exchange_and_add_rel ((mem), (operand))
 748 # endif
 749 # ifndef atomic_fetch_add_acq_rel
 750 #  define atomic_fetch_add_acq_rel(mem, operand) \
 751    ({ atomic_thread_fence_release ();                                         \
 752    atomic_exchange_and_add_acq ((mem), (operand)); })
 753 # endif
 754
 755 /* XXX Fall back to acquire MO because archs do not define a weaker
 756    atomic_and_val.  */
 757 # ifndef atomic_fetch_and_relaxed
 758 #  define atomic_fetch_and_relaxed(mem, operand) \
 759    atomic_fetch_and_acquire ((mem), (operand))
 760 # endif
 761 /* XXX The default for atomic_and_val has acquire semantics, but this is not
 762    documented.  */
 763 # ifndef atomic_fetch_and_acquire
 764 #  define atomic_fetch_and_acquire(mem, operand) \
 765    atomic_and_val ((mem), (operand))
 766 # endif
 767 # ifndef atomic_fetch_and_release
 768 /* XXX This unnecessarily has acquire MO.  */
 769 #  define atomic_fetch_and_release(mem, operand) \
 770    ({ atomic_thread_fence_release ();                                         \
 771    atomic_and_val ((mem), (operand)); })
 772 # endif
 773
 774 /* XXX The default for atomic_or_val has acquire semantics, but this is not
 775    documented.  */
 776 # ifndef atomic_fetch_or_acquire
 777 #  define atomic_fetch_or_acquire(mem, operand) \
 778    atomic_or_val ((mem), (operand))
 779 # endif
 780 /* XXX Fall back to acquire MO because archs do not define a weaker
 781    atomic_or_val.  */
 782 # ifndef atomic_fetch_or_relaxed
 783 #  define atomic_fetch_or_relaxed(mem, operand) \
 784    atomic_fetch_or_acquire ((mem), (operand))
 785 # endif
 786 /* XXX Contains an unnecessary acquire MO because archs do not define a weaker
 787    atomic_or_val.  */
 788 # ifndef atomic_fetch_or_release
 789 #  define atomic_fetch_or_release(mem, operand) \
 790    ({ atomic_thread_fence_release ();                                         \
 791    atomic_fetch_or_acquire ((mem), (operand)); })
 792 # endif
 793
 794 # ifndef atomic_fetch_xor_release
 795 /* Failing the atomic_compare_exchange_weak_release reloads the value in
 796    __atg104_expected, so we need only do the XOR again and retry.  */
 797 # define atomic_fetch_xor_release(mem, operand) \
 798   ({ __typeof (mem) __atg104_memp = (mem);                                    \
 799      __typeof (*(mem)) __atg104_expected = (*__atg104_memp);                  \
 800      __typeof (*(mem)) __atg104_desired;                                      \
 801      __typeof (*(mem)) __atg104_op = (operand);                               \
 802                                                                               \
 803      do                                                                       \
 804        __atg104_desired = __atg104_expected ^ __atg104_op;                    \
 805      while (__glibc_unlikely                                                  \
 806             (atomic_compare_exchange_weak_release (                           \
 807                __atg104_memp, &__atg104_expected, __atg104_desired)           \
 808              == 0));                                                          \
 809      __atg104_expected; })
 810 #endif
 811
 812 #endif /* !USE_ATOMIC_COMPILER_BUILTINS  */
 813
 814 /* This operation does not affect synchronization semantics but can be used
 815    in the body of a spin loop to potentially improve its efficiency.  */
 816 #ifndef atomic_spin_nop
 817 # define atomic_spin_nop() do { /* nothing */ } while (0)
 818 #endif
 819
 820 /* ATOMIC_EXCHANGE_USES_CAS is non-zero if atomic_exchange operations
 821    are implemented based on a CAS loop; otherwise, this is zero and we assume
 822    that the atomic_exchange operations could provide better performance
 823    than a CAS loop.  */
 824 #ifndef ATOMIC_EXCHANGE_USES_CAS
 825 # error ATOMIC_EXCHANGE_USES_CAS has to be defined.
 826 #endif
 827
 828 #endif  /* atomic.h */