include/atomic.h

   1 /* Internal macros for atomic operations for GNU C Library.
   2    Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, write to the Free
  18    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  19    02111-1307 USA.  */
  20
  21 #ifndef _ATOMIC_H
  22 #define _ATOMIC_H       1
  23
  24 /* This header defines three types of macros:
  25
  26    - atomic arithmetic and logic operation on memory.  They all
  27      have the prefix "atomic_".
  28
  29    - conditionally atomic operations of the same kinds.  These
  30      always behave identical but can be faster when atomicity
  31      is not really needed since only one thread has access to
  32      the memory location.  In that case the code is slower in
  33      the multi-thread case.  The interfaces have the prefix
  34      "catomic_".
  35
  36    - support functions like barriers.  They also have the preifx
  37      "atomic_".
  38
  39    Architectures must provide a few lowlevel macros (the compare
  40    and exchange definitions).  All others are optional.  They
  41    should only be provided if the architecture has specific
  42    support for the operation.
  43
  44    As <atomic.h> macros are usually heavily nested and often use local
  45    variables to make sure side-effects are evaluated properly, use for
  46    macro local variables a per-macro unique prefix.  This file uses
  47    __atgN_ prefix where N is different in each macro.  */
  48
  49 #include <stdlib.h>
  50
  51 #include <bits/atomic.h>
  52
  53 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
  54    bit width of *MEM.  The calling macro puts parens around MEM
  55    and following args.  */
  56 #define __atomic_val_bysize(pre, post, mem, ...)                              \
  57   ({                                                                          \
  58     __typeof (*mem) __atg1_result;                                            \
  59     if (sizeof (*mem) == 1)                                                   \
  60       __atg1_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  61     else if (sizeof (*mem) == 2)                                              \
  62       __atg1_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  63     else if (sizeof (*mem) == 4)                                              \
  64       __atg1_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  65     else if (sizeof (*mem) == 8)                                              \
  66       __atg1_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  67     else                                                                      \
  68       abort ();                                                               \
  69     __atg1_result;                                                            \
  70   })
  71 #define __atomic_bool_bysize(pre, post, mem, ...)                             \
  72   ({                                                                          \
  73     int __atg2_result;                                                        \
  74     if (sizeof (*mem) == 1)                                                   \
  75       __atg2_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  76     else if (sizeof (*mem) == 2)                                              \
  77       __atg2_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  78     else if (sizeof (*mem) == 4)                                              \
  79       __atg2_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  80     else if (sizeof (*mem) == 8)                                              \
  81       __atg2_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  82     else                                                                      \
  83       abort ();                                                               \
  84     __atg2_result;                                                            \
  85   })
  86
  87
  88 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  89    Return the old *MEM value.  */
  90 #if !defined atomic_compare_and_exchange_val_acq \
  91     && defined __arch_compare_and_exchange_val_32_acq
  92 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  93   __atomic_val_bysize (__arch_compare_and_exchange_val,acq,                   \
  94                        mem, newval, oldval)
  95 #endif
  96
  97
  98 #if !defined catomic_compare_and_exchange_val_acq \
  99     && defined __arch_c_compare_and_exchange_val_32_acq
 100 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 101   __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq,                 \
 102                        mem, newval, oldval)
 103 #else
 104 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 105   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 106 #endif
 107
 108
 109 #ifndef atomic_compare_and_exchange_val_rel
 110 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval)             \
 111   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 112 #endif
 113
 114
 115 #ifndef catomic_compare_and_exchange_val_rel
 116 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval)            \
 117   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 118 #endif
 119
 120
 121 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
 122    Return zero if *MEM was changed or non-zero if no exchange happened.  */
 123 #ifndef atomic_compare_and_exchange_bool_acq
 124 # ifdef __arch_compare_and_exchange_bool_32_acq
 125 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 126   __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq,                 \
 127                         mem, newval, oldval)
 128 #  else
 129 #   define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 130   ({ /* Cannot use __oldval here, because macros later in this file might     \
 131         call this macro with __oldval argument.  */                           \
 132      __typeof (oldval) __atg3_old = (oldval);                                 \
 133      atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old)            \
 134        != __atg3_old;                                                         \
 135   })
 136 # endif
 137 #endif
 138
 139
 140 #ifndef catomic_compare_and_exchange_bool_acq
 141 # ifdef __arch_c_compare_and_exchange_bool_32_acq
 142 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 143   __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq,               \
 144                         mem, newval, oldval)
 145 #  else
 146 #   define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 147   ({ /* Cannot use __oldval here, because macros later in this file might     \
 148         call this macro with __oldval argument.  */                           \
 149      __typeof (oldval) __atg4_old = (oldval);                                 \
 150      catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old)           \
 151        != __atg4_old;                                                         \
 152   })
 153 # endif
 154 #endif
 155
 156
 157 #ifndef atomic_compare_and_exchange_bool_rel
 158 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
 159   atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 160 #endif
 161
 162
 163 #ifndef catomic_compare_and_exchange_bool_rel
 164 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
 165   catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 166 #endif
 167
 168
 169 /* Store NEWVALUE in *MEM and return the old value.  */
 170 #ifndef atomic_exchange_acq
 171 # define atomic_exchange_acq(mem, newvalue) \
 172   ({ __typeof (*(mem)) __atg5_oldval;                                         \
 173      __typeof (mem) __atg5_memp = (mem);                                      \
 174      __typeof (*(mem)) __atg5_value = (newvalue);                             \
 175                                                                               \
 176      do                                                                       \
 177        __atg5_oldval = *__atg5_memp;                                          \
 178      while (__builtin_expect                                                  \
 179             (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
 180                                                    __atg5_oldval), 0));       \
 181                                                                               \
 182      __atg5_oldval; })
 183 #endif
 184
 185 #ifndef atomic_exchange_rel
 186 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
 187 #endif
 188
 189
 190 /* Add VALUE to *MEM and return the old value of *MEM.  */
 191 #ifndef atomic_exchange_and_add
 192 # define atomic_exchange_and_add(mem, value) \
 193   ({ __typeof (*(mem)) __atg6_oldval;                                         \
 194      __typeof (mem) __atg6_memp = (mem);                                      \
 195      __typeof (*(mem)) __atg6_value = (value);                                \
 196                                                                               \
 197      do                                                                       \
 198        __atg6_oldval = *__atg6_memp;                                          \
 199      while (__builtin_expect                                                  \
 200             (atomic_compare_and_exchange_bool_acq (__atg6_memp,               \
 201                                                    __atg6_oldval              \
 202                                                    + __atg6_value,            \
 203                                                    __atg6_oldval), 0));       \
 204                                                                               \
 205      __atg6_oldval; })
 206 #endif
 207
 208
 209 #ifndef catomic_exchange_and_add
 210 # define catomic_exchange_and_add(mem, value) \
 211   ({ __typeof (*(mem)) __atg7_oldv;                                           \
 212      __typeof (mem) __atg7_memp = (mem);                                      \
 213      __typeof (*(mem)) __atg7_value = (value);                                \
 214                                                                               \
 215      do                                                                       \
 216        __atg7_oldv = *__atg7_memp;                                            \
 217      while (__builtin_expect                                                  \
 218             (catomic_compare_and_exchange_bool_acq (__atg7_memp,              \
 219                                                     __atg7_oldv               \
 220                                                     + __atg7_value,           \
 221                                                     __atg7_oldv), 0));        \
 222                                                                               \
 223      __atg7_oldv; })
 224 #endif
 225
 226
 227 #ifndef atomic_max
 228 # define atomic_max(mem, value) \
 229   do {                                                                        \
 230     __typeof (*(mem)) __atg8_oldval;                                          \
 231     __typeof (mem) __atg8_memp = (mem);                                       \
 232     __typeof (*(mem)) __atg8_value = (value);                                 \
 233     do {                                                                      \
 234       __atg8_oldval = *__atg8_memp;                                           \
 235       if (__atg8_oldval >= __atg8_value)                                      \
 236         break;                                                                \
 237     } while (__builtin_expect                                                 \
 238              (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
 239                                                     __atg8_oldval), 0));      \
 240   } while (0)
 241 #endif
 242
 243
 244 #ifndef catomic_max
 245 # define catomic_max(mem, value) \
 246   do {                                                                        \
 247     __typeof (*(mem)) __atg9_oldv;                                            \
 248     __typeof (mem) __atg9_memp = (mem);                                       \
 249     __typeof (*(mem)) __atg9_value = (value);                                 \
 250     do {                                                                      \
 251       __atg9_oldv = *__atg9_memp;                                             \
 252       if (__atg9_oldv >= __atg9_value)                                        \
 253         break;                                                                \
 254     } while (__builtin_expect                                                 \
 255              (catomic_compare_and_exchange_bool_acq (__atg9_memp,             \
 256                                                      __atg9_value,            \
 257                                                      __atg9_oldv), 0));       \
 258   } while (0)
 259 #endif
 260
 261
 262 #ifndef atomic_min
 263 # define atomic_min(mem, value) \
 264   do {                                                                        \
 265     __typeof (*(mem)) __atg10_oldval;                                         \
 266     __typeof (mem) __atg10_memp = (mem);                                      \
 267     __typeof (*(mem)) __atg10_value = (value);                                \
 268     do {                                                                      \
 269       __atg10_oldval = *__atg10_memp;                                         \
 270       if (__atg10_oldval <= __atg10_value)                                    \
 271         break;                                                                \
 272     } while (__builtin_expect                                                 \
 273              (atomic_compare_and_exchange_bool_acq (__atg10_memp,             \
 274                                                     __atg10_value,            \
 275                                                     __atg10_oldval), 0));     \
 276   } while (0)
 277 #endif
 278
 279
 280 #ifndef atomic_add
 281 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
 282 #endif
 283
 284
 285 #ifndef catomic_add
 286 # define catomic_add(mem, value) \
 287   (void) catomic_exchange_and_add ((mem), (value))
 288 #endif
 289
 290
 291 #ifndef atomic_increment
 292 # define atomic_increment(mem) atomic_add ((mem), 1)
 293 #endif
 294
 295
 296 #ifndef catomic_increment
 297 # define catomic_increment(mem) catomic_add ((mem), 1)
 298 #endif
 299
 300
 301 #ifndef atomic_increment_val
 302 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
 303 #endif
 304
 305
 306 #ifndef catomic_increment_val
 307 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
 308 #endif
 309
 310
 311 /* Add one to *MEM and return true iff it's now zero.  */
 312 #ifndef atomic_increment_and_test
 313 # define atomic_increment_and_test(mem) \
 314   (atomic_exchange_and_add ((mem), 1) + 1 == 0)
 315 #endif
 316
 317
 318 #ifndef atomic_decrement
 319 # define atomic_decrement(mem) atomic_add ((mem), -1)
 320 #endif
 321
 322
 323 #ifndef catomic_decrement
 324 # define catomic_decrement(mem) catomic_add ((mem), -1)
 325 #endif
 326
 327
 328 #ifndef atomic_decrement_val
 329 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
 330 #endif
 331
 332
 333 #ifndef catomic_decrement_val
 334 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
 335 #endif
 336
 337
 338 /* Subtract 1 from *MEM and return true iff it's now zero.  */
 339 #ifndef atomic_decrement_and_test
 340 # define atomic_decrement_and_test(mem) \
 341   (atomic_exchange_and_add ((mem), -1) == 1)
 342 #endif
 343
 344
 345 /* Decrement *MEM if it is > 0, and return the old value.  */
 346 #ifndef atomic_decrement_if_positive
 347 # define atomic_decrement_if_positive(mem) \
 348   ({ __typeof (*(mem)) __atg11_oldval;                                        \
 349      __typeof (mem) __atg11_memp = (mem);                                     \
 350                                                                               \
 351      do                                                                       \
 352        {                                                                      \
 353          __atg11_oldval = *__atg11_memp;                                      \
 354          if (__builtin_expect (__atg11_oldval <= 0, 0))                       \
 355            break;                                                             \
 356        }                                                                      \
 357      while (__builtin_expect                                                  \
 358             (atomic_compare_and_exchange_bool_acq (__atg11_memp,              \
 359                                                    __atg11_oldval - 1,        \
 360                                                    __atg11_oldval), 0));      \
 361      __atg11_oldval; })
 362 #endif
 363
 364
 365 #ifndef atomic_add_negative
 366 # define atomic_add_negative(mem, value)                                      \
 367   ({ __typeof (value) __atg12_value = (value);                                \
 368      atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
 369 #endif
 370
 371
 372 #ifndef atomic_add_zero
 373 # define atomic_add_zero(mem, value)                                          \
 374   ({ __typeof (value) __atg13_value = (value);                                \
 375      atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
 376 #endif
 377
 378
 379 #ifndef atomic_bit_set
 380 # define atomic_bit_set(mem, bit) \
 381   (void) atomic_bit_test_set(mem, bit)
 382 #endif
 383
 384
 385 #ifndef atomic_bit_test_set
 386 # define atomic_bit_test_set(mem, bit) \
 387   ({ __typeof (*(mem)) __atg14_old;                                           \
 388      __typeof (mem) __atg14_memp = (mem);                                     \
 389      __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit));       \
 390                                                                               \
 391      do                                                                       \
 392        __atg14_old = (*__atg14_memp);                                         \
 393      while (__builtin_expect                                                  \
 394             (atomic_compare_and_exchange_bool_acq (__atg14_memp,              \
 395                                                    __atg14_old | __atg14_mask,\
 396                                                    __atg14_old), 0));         \
 397                                                                               \
 398      __atg14_old & __atg14_mask; })
 399 #endif
 400
 401 /* Atomically *mem &= mask.  */
 402 #ifndef atomic_and
 403 # define atomic_and(mem, mask) \
 404   do {                                                                        \
 405     __typeof (*(mem)) __atg15_old;                                            \
 406     __typeof (mem) __atg15_memp = (mem);                                      \
 407     __typeof (*(mem)) __atg15_mask = (mask);                                  \
 408                                                                               \
 409     do                                                                        \
 410       __atg15_old = (*__atg15_memp);                                          \
 411     while (__builtin_expect                                                   \
 412            (atomic_compare_and_exchange_bool_acq (__atg15_memp,               \
 413                                                   __atg15_old & __atg15_mask, \
 414                                                   __atg15_old), 0));          \
 415   } while (0)
 416 #endif
 417
 418 /* Atomically *mem &= mask and return the old value of *mem.  */
 419 #ifndef atomic_and_val
 420 # define atomic_and_val(mem, mask) \
 421   ({ __typeof (*(mem)) __atg16_old;                                           \
 422      __typeof (mem) __atg16_memp = (mem);                                     \
 423      __typeof (*(mem)) __atg16_mask = (mask);                                 \
 424                                                                               \
 425      do                                                                       \
 426        __atg16_old = (*__atg16_memp);                                         \
 427      while (__builtin_expect                                                  \
 428             (atomic_compare_and_exchange_bool_acq (__atg16_memp,              \
 429                                                    __atg16_old & __atg16_mask,\
 430                                                    __atg16_old), 0));         \
 431                                                                               \
 432      __atg16_old; })
 433 #endif
 434
 435 /* Atomically *mem |= mask and return the old value of *mem.  */
 436 #ifndef atomic_or
 437 # define atomic_or(mem, mask) \
 438   do {                                                                        \
 439     __typeof (*(mem)) __atg17_old;                                            \
 440     __typeof (mem) __atg17_memp = (mem);                                      \
 441     __typeof (*(mem)) __atg17_mask = (mask);                                  \
 442                                                                               \
 443     do                                                                        \
 444       __atg17_old = (*__atg17_memp);                                          \
 445     while (__builtin_expect                                                   \
 446            (atomic_compare_and_exchange_bool_acq (__atg17_memp,               \
 447                                                   __atg17_old | __atg17_mask, \
 448                                                   __atg17_old), 0));          \
 449   } while (0)
 450 #endif
 451
 452 #ifndef catomic_or
 453 # define catomic_or(mem, mask) \
 454   do {                                                                        \
 455     __typeof (*(mem)) __atg18_old;                                            \
 456     __typeof (mem) __atg18_memp = (mem);                                      \
 457     __typeof (*(mem)) __atg18_mask = (mask);                                  \
 458                                                                               \
 459     do                                                                        \
 460       __atg18_old = (*__atg18_memp);                                          \
 461     while (__builtin_expect                                                   \
 462            (catomic_compare_and_exchange_bool_acq (__atg18_memp,              \
 463                                                    __atg18_old | __atg18_mask,\
 464                                                    __atg18_old), 0));         \
 465   } while (0)
 466 #endif
 467
 468 /* Atomically *mem |= mask and return the old value of *mem.  */
 469 #ifndef atomic_or_val
 470 # define atomic_or_val(mem, mask) \
 471   ({ __typeof (*(mem)) __atg19_old;                                           \
 472      __typeof (mem) __atg19_memp = (mem);                                     \
 473      __typeof (*(mem)) __atg19_mask = (mask);                                 \
 474                                                                               \
 475      do                                                                       \
 476        __atg19_old = (*__atg19_memp);                                         \
 477      while (__builtin_expect                                                  \
 478             (atomic_compare_and_exchange_bool_acq (__atg19_memp,              \
 479                                                    __atg19_old | __atg19_mask,\
 480                                                    __atg19_old), 0));         \
 481                                                                               \
 482      __atg19_old; })
 483 #endif
 484
 485 #ifndef atomic_full_barrier
 486 # define atomic_full_barrier() __asm ("" ::: "memory")
 487 #endif
 488
 489
 490 #ifndef atomic_read_barrier
 491 # define atomic_read_barrier() atomic_full_barrier ()
 492 #endif
 493
 494
 495 #ifndef atomic_write_barrier
 496 # define atomic_write_barrier() atomic_full_barrier ()
 497 #endif
 498
 499
 500 #ifndef atomic_delay
 501 # define atomic_delay() do { /* nothing */ } while (0)
 502 #endif
 503
 504 #endif  /* atomic.h */