include/atomic.h

   1 /* Internal macros for atomic operations for GNU C Library.
   2    Copyright (C) 2002-2014 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #ifndef _ATOMIC_H
  21 #define _ATOMIC_H       1
  22
  23 /* This header defines three types of macros:
  24
  25    - atomic arithmetic and logic operation on memory.  They all
  26      have the prefix "atomic_".
  27
  28    - conditionally atomic operations of the same kinds.  These
  29      always behave identical but can be faster when atomicity
  30      is not really needed since only one thread has access to
  31      the memory location.  In that case the code is slower in
  32      the multi-thread case.  The interfaces have the prefix
  33      "catomic_".
  34
  35    - support functions like barriers.  They also have the prefix
  36      "atomic_".
  37
  38    Architectures must provide a few lowlevel macros (the compare
  39    and exchange definitions).  All others are optional.  They
  40    should only be provided if the architecture has specific
  41    support for the operation.
  42
  43    As <atomic.h> macros are usually heavily nested and often use local
  44    variables to make sure side-effects are evaluated properly, use for
  45    macro local variables a per-macro unique prefix.  This file uses
  46    __atgN_ prefix where N is different in each macro.  */
  47
  48 #include <stdlib.h>
  49
  50 #include <bits/atomic.h>
  51
  52 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
  53    bit width of *MEM.  The calling macro puts parens around MEM
  54    and following args.  */
  55 #define __atomic_val_bysize(pre, post, mem, ...)                              \
  56   ({                                                                          \
  57     __typeof (*mem) __atg1_result;                                            \
  58     if (sizeof (*mem) == 1)                                                   \
  59       __atg1_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  60     else if (sizeof (*mem) == 2)                                              \
  61       __atg1_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  62     else if (sizeof (*mem) == 4)                                              \
  63       __atg1_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  64     else if (sizeof (*mem) == 8)                                              \
  65       __atg1_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  66     else                                                                      \
  67       abort ();                                                               \
  68     __atg1_result;                                                            \
  69   })
  70 #define __atomic_bool_bysize(pre, post, mem, ...)                             \
  71   ({                                                                          \
  72     int __atg2_result;                                                        \
  73     if (sizeof (*mem) == 1)                                                   \
  74       __atg2_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  75     else if (sizeof (*mem) == 2)                                              \
  76       __atg2_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  77     else if (sizeof (*mem) == 4)                                              \
  78       __atg2_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  79     else if (sizeof (*mem) == 8)                                              \
  80       __atg2_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  81     else                                                                      \
  82       abort ();                                                               \
  83     __atg2_result;                                                            \
  84   })
  85
  86
  87 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  88    Return the old *MEM value.  */
  89 #if !defined atomic_compare_and_exchange_val_acq \
  90     && defined __arch_compare_and_exchange_val_32_acq
  91 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  92   __atomic_val_bysize (__arch_compare_and_exchange_val,acq,                   \
  93                        mem, newval, oldval)
  94 #endif
  95
  96
  97 #ifndef catomic_compare_and_exchange_val_acq
  98 # ifdef __arch_c_compare_and_exchange_val_32_acq
  99 #  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 100   __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq,                 \
 101                        mem, newval, oldval)
 102 # else
 103 #  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 104   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 105 # endif
 106 #endif
 107
 108
 109 #ifndef catomic_compare_and_exchange_val_rel
 110 # ifndef atomic_compare_and_exchange_val_rel
 111 #  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)           \
 112   catomic_compare_and_exchange_val_acq (mem, newval, oldval)
 113 # else
 114 #  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)           \
 115   atomic_compare_and_exchange_val_rel (mem, newval, oldval)
 116 # endif
 117 #endif
 118
 119
 120 #ifndef atomic_compare_and_exchange_val_rel
 121 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval)             \
 122   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 123 #endif
 124
 125
 126 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
 127    Return zero if *MEM was changed or non-zero if no exchange happened.  */
 128 #ifndef atomic_compare_and_exchange_bool_acq
 129 # ifdef __arch_compare_and_exchange_bool_32_acq
 130 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 131   __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq,                 \
 132                         mem, newval, oldval)
 133 # else
 134 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 135   ({ /* Cannot use __oldval here, because macros later in this file might     \
 136         call this macro with __oldval argument.  */                           \
 137      __typeof (oldval) __atg3_old = (oldval);                                 \
 138      atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old)            \
 139        != __atg3_old;                                                         \
 140   })
 141 # endif
 142 #endif
 143
 144
 145 #ifndef catomic_compare_and_exchange_bool_acq
 146 # ifdef __arch_c_compare_and_exchange_bool_32_acq
 147 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 148   __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq,               \
 149                         mem, newval, oldval)
 150 # else
 151 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 152   ({ /* Cannot use __oldval here, because macros later in this file might     \
 153         call this macro with __oldval argument.  */                           \
 154      __typeof (oldval) __atg4_old = (oldval);                                 \
 155      catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old)           \
 156        != __atg4_old;                                                         \
 157   })
 158 # endif
 159 #endif
 160
 161
 162 #ifndef catomic_compare_and_exchange_bool_rel
 163 # ifndef atomic_compare_and_exchange_bool_rel
 164 #  define catomic_compare_and_exchange_bool_rel(mem, newval, oldval)          \
 165   catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 166 # else
 167 #  define catomic_compare_and_exchange_bool_rel(mem, newval, oldval)          \
 168   atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
 169 # endif
 170 #endif
 171
 172
 173 #ifndef atomic_compare_and_exchange_bool_rel
 174 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
 175   atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 176 #endif
 177
 178
 179 /* Store NEWVALUE in *MEM and return the old value.  */
 180 #ifndef atomic_exchange_acq
 181 # define atomic_exchange_acq(mem, newvalue) \
 182   ({ __typeof (*(mem)) __atg5_oldval;                                         \
 183      __typeof (mem) __atg5_memp = (mem);                                      \
 184      __typeof (*(mem)) __atg5_value = (newvalue);                             \
 185                                                                               \
 186      do                                                                       \
 187        __atg5_oldval = *__atg5_memp;                                          \
 188      while (__builtin_expect                                                  \
 189             (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
 190                                                    __atg5_oldval), 0));       \
 191                                                                               \
 192      __atg5_oldval; })
 193 #endif
 194
 195 #ifndef atomic_exchange_rel
 196 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
 197 #endif
 198
 199
 200 /* Add VALUE to *MEM and return the old value of *MEM.  */
 201 #ifndef atomic_exchange_and_add_acq
 202 # ifdef atomic_exchange_and_add
 203 #  define atomic_exchange_and_add_acq(mem, value) \
 204   atomic_exchange_and_add (mem, value)
 205 # else
 206 #  define atomic_exchange_and_add_acq(mem, value) \
 207   ({ __typeof (*(mem)) __atg6_oldval;                                         \
 208      __typeof (mem) __atg6_memp = (mem);                                      \
 209      __typeof (*(mem)) __atg6_value = (value);                                \
 210                                                                               \
 211      do                                                                       \
 212        __atg6_oldval = *__atg6_memp;                                          \
 213      while (__builtin_expect                                                  \
 214             (atomic_compare_and_exchange_bool_acq (__atg6_memp,               \
 215                                                    __atg6_oldval              \
 216                                                    + __atg6_value,            \
 217                                                    __atg6_oldval), 0));       \
 218                                                                               \
 219      __atg6_oldval; })
 220 # endif
 221 #endif
 222
 223 #ifndef atomic_exchange_and_add_rel
 224 # define atomic_exchange_and_add_rel(mem, value) \
 225   atomic_exchange_and_add_acq(mem, value)
 226 #endif
 227
 228 #ifndef atomic_exchange_and_add
 229 # define atomic_exchange_and_add(mem, value) \
 230   atomic_exchange_and_add_acq(mem, value)
 231 #endif
 232
 233 #ifndef catomic_exchange_and_add
 234 # define catomic_exchange_and_add(mem, value) \
 235   ({ __typeof (*(mem)) __atg7_oldv;                                           \
 236      __typeof (mem) __atg7_memp = (mem);                                      \
 237      __typeof (*(mem)) __atg7_value = (value);                                \
 238                                                                               \
 239      do                                                                       \
 240        __atg7_oldv = *__atg7_memp;                                            \
 241      while (__builtin_expect                                                  \
 242             (catomic_compare_and_exchange_bool_acq (__atg7_memp,              \
 243                                                     __atg7_oldv               \
 244                                                     + __atg7_value,           \
 245                                                     __atg7_oldv), 0));        \
 246                                                                               \
 247      __atg7_oldv; })
 248 #endif
 249
 250
 251 #ifndef atomic_max
 252 # define atomic_max(mem, value) \
 253   do {                                                                        \
 254     __typeof (*(mem)) __atg8_oldval;                                          \
 255     __typeof (mem) __atg8_memp = (mem);                                       \
 256     __typeof (*(mem)) __atg8_value = (value);                                 \
 257     do {                                                                      \
 258       __atg8_oldval = *__atg8_memp;                                           \
 259       if (__atg8_oldval >= __atg8_value)                                      \
 260         break;                                                                \
 261     } while (__builtin_expect                                                 \
 262              (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
 263                                                     __atg8_oldval), 0));      \
 264   } while (0)
 265 #endif
 266
 267
 268 #ifndef catomic_max
 269 # define catomic_max(mem, value) \
 270   do {                                                                        \
 271     __typeof (*(mem)) __atg9_oldv;                                            \
 272     __typeof (mem) __atg9_memp = (mem);                                       \
 273     __typeof (*(mem)) __atg9_value = (value);                                 \
 274     do {                                                                      \
 275       __atg9_oldv = *__atg9_memp;                                             \
 276       if (__atg9_oldv >= __atg9_value)                                        \
 277         break;                                                                \
 278     } while (__builtin_expect                                                 \
 279              (catomic_compare_and_exchange_bool_acq (__atg9_memp,             \
 280                                                      __atg9_value,            \
 281                                                      __atg9_oldv), 0));       \
 282   } while (0)
 283 #endif
 284
 285
 286 #ifndef atomic_min
 287 # define atomic_min(mem, value) \
 288   do {                                                                        \
 289     __typeof (*(mem)) __atg10_oldval;                                         \
 290     __typeof (mem) __atg10_memp = (mem);                                      \
 291     __typeof (*(mem)) __atg10_value = (value);                                \
 292     do {                                                                      \
 293       __atg10_oldval = *__atg10_memp;                                         \
 294       if (__atg10_oldval <= __atg10_value)                                    \
 295         break;                                                                \
 296     } while (__builtin_expect                                                 \
 297              (atomic_compare_and_exchange_bool_acq (__atg10_memp,             \
 298                                                     __atg10_value,            \
 299                                                     __atg10_oldval), 0));     \
 300   } while (0)
 301 #endif
 302
 303
 304 #ifndef atomic_add
 305 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
 306 #endif
 307
 308
 309 #ifndef catomic_add
 310 # define catomic_add(mem, value) \
 311   (void) catomic_exchange_and_add ((mem), (value))
 312 #endif
 313
 314
 315 #ifndef atomic_increment
 316 # define atomic_increment(mem) atomic_add ((mem), 1)
 317 #endif
 318
 319
 320 #ifndef catomic_increment
 321 # define catomic_increment(mem) catomic_add ((mem), 1)
 322 #endif
 323
 324
 325 #ifndef atomic_increment_val
 326 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
 327 #endif
 328
 329
 330 #ifndef catomic_increment_val
 331 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
 332 #endif
 333
 334
 335 /* Add one to *MEM and return true iff it's now zero.  */
 336 #ifndef atomic_increment_and_test
 337 # define atomic_increment_and_test(mem) \
 338   (atomic_exchange_and_add ((mem), 1) + 1 == 0)
 339 #endif
 340
 341
 342 #ifndef atomic_decrement
 343 # define atomic_decrement(mem) atomic_add ((mem), -1)
 344 #endif
 345
 346
 347 #ifndef catomic_decrement
 348 # define catomic_decrement(mem) catomic_add ((mem), -1)
 349 #endif
 350
 351
 352 #ifndef atomic_decrement_val
 353 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
 354 #endif
 355
 356
 357 #ifndef catomic_decrement_val
 358 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
 359 #endif
 360
 361
 362 /* Subtract 1 from *MEM and return true iff it's now zero.  */
 363 #ifndef atomic_decrement_and_test
 364 # define atomic_decrement_and_test(mem) \
 365   (atomic_exchange_and_add ((mem), -1) == 1)
 366 #endif
 367
 368
 369 /* Decrement *MEM if it is > 0, and return the old value.  */
 370 #ifndef atomic_decrement_if_positive
 371 # define atomic_decrement_if_positive(mem) \
 372   ({ __typeof (*(mem)) __atg11_oldval;                                        \
 373      __typeof (mem) __atg11_memp = (mem);                                     \
 374                                                                               \
 375      do                                                                       \
 376        {                                                                      \
 377          __atg11_oldval = *__atg11_memp;                                      \
 378          if (__glibc_unlikely (__atg11_oldval <= 0))                          \
 379            break;                                                             \
 380        }                                                                      \
 381      while (__builtin_expect                                                  \
 382             (atomic_compare_and_exchange_bool_acq (__atg11_memp,              \
 383                                                    __atg11_oldval - 1,        \
 384                                                    __atg11_oldval), 0));      \
 385      __atg11_oldval; })
 386 #endif
 387
 388
 389 #ifndef atomic_add_negative
 390 # define atomic_add_negative(mem, value)                                      \
 391   ({ __typeof (value) __atg12_value = (value);                                \
 392      atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
 393 #endif
 394
 395
 396 #ifndef atomic_add_zero
 397 # define atomic_add_zero(mem, value)                                          \
 398   ({ __typeof (value) __atg13_value = (value);                                \
 399      atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
 400 #endif
 401
 402
 403 #ifndef atomic_bit_set
 404 # define atomic_bit_set(mem, bit) \
 405   (void) atomic_bit_test_set(mem, bit)
 406 #endif
 407
 408
 409 #ifndef atomic_bit_test_set
 410 # define atomic_bit_test_set(mem, bit) \
 411   ({ __typeof (*(mem)) __atg14_old;                                           \
 412      __typeof (mem) __atg14_memp = (mem);                                     \
 413      __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit));       \
 414                                                                               \
 415      do                                                                       \
 416        __atg14_old = (*__atg14_memp);                                         \
 417      while (__builtin_expect                                                  \
 418             (atomic_compare_and_exchange_bool_acq (__atg14_memp,              \
 419                                                    __atg14_old | __atg14_mask,\
 420                                                    __atg14_old), 0));         \
 421                                                                               \
 422      __atg14_old & __atg14_mask; })
 423 #endif
 424
 425 /* Atomically *mem &= mask.  */
 426 #ifndef atomic_and
 427 # define atomic_and(mem, mask) \
 428   do {                                                                        \
 429     __typeof (*(mem)) __atg15_old;                                            \
 430     __typeof (mem) __atg15_memp = (mem);                                      \
 431     __typeof (*(mem)) __atg15_mask = (mask);                                  \
 432                                                                               \
 433     do                                                                        \
 434       __atg15_old = (*__atg15_memp);                                          \
 435     while (__builtin_expect                                                   \
 436            (atomic_compare_and_exchange_bool_acq (__atg15_memp,               \
 437                                                   __atg15_old & __atg15_mask, \
 438                                                   __atg15_old), 0));          \
 439   } while (0)
 440 #endif
 441
 442 #ifndef catomic_and
 443 # define catomic_and(mem, mask) \
 444   do {                                                                        \
 445     __typeof (*(mem)) __atg20_old;                                            \
 446     __typeof (mem) __atg20_memp = (mem);                                      \
 447     __typeof (*(mem)) __atg20_mask = (mask);                                  \
 448                                                                               \
 449     do                                                                        \
 450       __atg20_old = (*__atg20_memp);                                          \
 451     while (__builtin_expect                                                   \
 452            (catomic_compare_and_exchange_bool_acq (__atg20_memp,              \
 453                                                    __atg20_old & __atg20_mask,\
 454                                                    __atg20_old), 0));         \
 455   } while (0)
 456 #endif
 457
 458 /* Atomically *mem &= mask and return the old value of *mem.  */
 459 #ifndef atomic_and_val
 460 # define atomic_and_val(mem, mask) \
 461   ({ __typeof (*(mem)) __atg16_old;                                           \
 462      __typeof (mem) __atg16_memp = (mem);                                     \
 463      __typeof (*(mem)) __atg16_mask = (mask);                                 \
 464                                                                               \
 465      do                                                                       \
 466        __atg16_old = (*__atg16_memp);                                         \
 467      while (__builtin_expect                                                  \
 468             (atomic_compare_and_exchange_bool_acq (__atg16_memp,              \
 469                                                    __atg16_old & __atg16_mask,\
 470                                                    __atg16_old), 0));         \
 471                                                                               \
 472      __atg16_old; })
 473 #endif
 474
 475 /* Atomically *mem |= mask and return the old value of *mem.  */
 476 #ifndef atomic_or
 477 # define atomic_or(mem, mask) \
 478   do {                                                                        \
 479     __typeof (*(mem)) __atg17_old;                                            \
 480     __typeof (mem) __atg17_memp = (mem);                                      \
 481     __typeof (*(mem)) __atg17_mask = (mask);                                  \
 482                                                                               \
 483     do                                                                        \
 484       __atg17_old = (*__atg17_memp);                                          \
 485     while (__builtin_expect                                                   \
 486            (atomic_compare_and_exchange_bool_acq (__atg17_memp,               \
 487                                                   __atg17_old | __atg17_mask, \
 488                                                   __atg17_old), 0));          \
 489   } while (0)
 490 #endif
 491
 492 #ifndef catomic_or
 493 # define catomic_or(mem, mask) \
 494   do {                                                                        \
 495     __typeof (*(mem)) __atg18_old;                                            \
 496     __typeof (mem) __atg18_memp = (mem);                                      \
 497     __typeof (*(mem)) __atg18_mask = (mask);                                  \
 498                                                                               \
 499     do                                                                        \
 500       __atg18_old = (*__atg18_memp);                                          \
 501     while (__builtin_expect                                                   \
 502            (catomic_compare_and_exchange_bool_acq (__atg18_memp,              \
 503                                                    __atg18_old | __atg18_mask,\
 504                                                    __atg18_old), 0));         \
 505   } while (0)
 506 #endif
 507
 508 /* Atomically *mem |= mask and return the old value of *mem.  */
 509 #ifndef atomic_or_val
 510 # define atomic_or_val(mem, mask) \
 511   ({ __typeof (*(mem)) __atg19_old;                                           \
 512      __typeof (mem) __atg19_memp = (mem);                                     \
 513      __typeof (*(mem)) __atg19_mask = (mask);                                 \
 514                                                                               \
 515      do                                                                       \
 516        __atg19_old = (*__atg19_memp);                                         \
 517      while (__builtin_expect                                                  \
 518             (atomic_compare_and_exchange_bool_acq (__atg19_memp,              \
 519                                                    __atg19_old | __atg19_mask,\
 520                                                    __atg19_old), 0));         \
 521                                                                               \
 522      __atg19_old; })
 523 #endif
 524
 525 #ifndef atomic_full_barrier
 526 # define atomic_full_barrier() __asm ("" ::: "memory")
 527 #endif
 528
 529
 530 #ifndef atomic_read_barrier
 531 # define atomic_read_barrier() atomic_full_barrier ()
 532 #endif
 533
 534
 535 #ifndef atomic_write_barrier
 536 # define atomic_write_barrier() atomic_full_barrier ()
 537 #endif
 538
 539
 540 #ifndef atomic_forced_read
 541 # define atomic_forced_read(x) \
 542   ({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; })
 543 #endif
 544
 545
 546 #ifndef atomic_delay
 547 # define atomic_delay() do { /* nothing */ } while (0)
 548 #endif
 549
 550 #endif  /* atomic.h */