include/atomic.h

   1 /* Internal macros for atomic operations for GNU C Library.
   2    Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, write to the Free
  18    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  19    02111-1307 USA.  */
  20
  21 #ifndef _ATOMIC_H
  22 #define _ATOMIC_H       1
  23
  24 /* This header defines three types of macros:
  25
  26    - atomic arithmetic and logic operation on memory.  They all
  27      have the prefix "atomic_".
  28
  29    - conditionally atomic operations of the same kinds.  These
  30      always behave identical but can be faster when atomicity
  31      is not really needed since only one thread has access to
  32      the memory location.  In that case the code is slower in
  33      the multi-thread case.  The interfaces have the prefix
  34      "catomic_".
  35
  36    - support functions like barriers.  They also have the preifx
  37      "atomic_".
  38
  39    Architectures must provide a few lowlevel macros (the compare
  40    and exchange definitions).  All others are optional.  They
  41    should only be provided if the architecture has specific
  42    support for the operation.  */
  43
  44 #include <stdlib.h>
  45
  46 #include <bits/atomic.h>
  47
  48 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
  49    bit width of *MEM.  The calling macro puts parens around MEM
  50    and following args.  */
  51 #define __atomic_val_bysize(pre, post, mem, ...)                              \
  52   ({                                                                          \
  53     __typeof (*mem) __result;                                                 \
  54     if (sizeof (*mem) == 1)                                                   \
  55       __result = pre##_8_##post (mem, __VA_ARGS__);                           \
  56     else if (sizeof (*mem) == 2)                                              \
  57       __result = pre##_16_##post (mem, __VA_ARGS__);                          \
  58     else if (sizeof (*mem) == 4)                                              \
  59       __result = pre##_32_##post (mem, __VA_ARGS__);                          \
  60     else if (sizeof (*mem) == 8)                                              \
  61       __result = pre##_64_##post (mem, __VA_ARGS__);                          \
  62     else                                                                      \
  63       abort ();                                                               \
  64     __result;                                                                 \
  65   })
  66 #define __atomic_bool_bysize(pre, post, mem, ...)                             \
  67   ({                                                                          \
  68     int __result;                                                             \
  69     if (sizeof (*mem) == 1)                                                   \
  70       __result = pre##_8_##post (mem, __VA_ARGS__);                           \
  71     else if (sizeof (*mem) == 2)                                              \
  72       __result = pre##_16_##post (mem, __VA_ARGS__);                          \
  73     else if (sizeof (*mem) == 4)                                              \
  74       __result = pre##_32_##post (mem, __VA_ARGS__);                          \
  75     else if (sizeof (*mem) == 8)                                              \
  76       __result = pre##_64_##post (mem, __VA_ARGS__);                          \
  77     else                                                                      \
  78       abort ();                                                               \
  79     __result;                                                                 \
  80   })
  81
  82
  83 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  84    Return the old *MEM value.  */
  85 #if !defined atomic_compare_and_exchange_val_acq \
  86     && defined __arch_compare_and_exchange_val_32_acq
  87 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  88   __atomic_val_bysize (__arch_compare_and_exchange_val,acq,                   \
  89                        mem, newval, oldval)
  90 #endif
  91
  92
  93 #if !defined catomic_compare_and_exchange_val_acq \
  94     && defined __arch_c_compare_and_exchange_val_32_acq
  95 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  96   __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq,                 \
  97                        mem, newval, oldval)
  98 #else
  99 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 100   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 101 #endif
 102
 103
 104 #ifndef atomic_compare_and_exchange_val_rel
 105 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval)             \
 106   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 107 #endif
 108
 109
 110 #ifndef catomic_compare_and_exchange_val_rel
 111 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval)            \
 112   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 113 #endif
 114
 115
 116 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
 117    Return zero if *MEM was changed or non-zero if no exchange happened.  */
 118 #ifndef atomic_compare_and_exchange_bool_acq
 119 # ifdef __arch_compare_and_exchange_bool_32_acq
 120 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 121   __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq,                 \
 122                         mem, newval, oldval)
 123 #  else
 124 #   define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 125   ({ /* Cannot use __oldval here, because macros later in this file might     \
 126         call this macro with __oldval argument.  */                           \
 127      __typeof (oldval) __old = (oldval);                                      \
 128      atomic_compare_and_exchange_val_acq (mem, newval, __old) != __old;       \
 129   })
 130 # endif
 131 #endif
 132
 133
 134 #ifndef catomic_compare_and_exchange_bool_acq
 135 # ifdef __arch_c_compare_and_exchange_bool_32_acq
 136 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 137   __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq,               \
 138                         mem, newval, oldval)
 139 #  else
 140 #   define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 141   ({ /* Cannot use __oldval here, because macros later in this file might     \
 142         call this macro with __oldval argument.  */                           \
 143      __typeof (oldval) __old = (oldval);                                      \
 144      catomic_compare_and_exchange_val_acq (mem, newval, __old) != __old;      \
 145   })
 146 # endif
 147 #endif
 148
 149
 150 #ifndef atomic_compare_and_exchange_bool_rel
 151 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
 152   atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 153 #endif
 154
 155
 156 #ifndef catomic_compare_and_exchange_bool_rel
 157 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
 158   catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 159 #endif
 160
 161
 162 /* Store NEWVALUE in *MEM and return the old value.  */
 163 #ifndef atomic_exchange_acq
 164 # define atomic_exchange_acq(mem, newvalue) \
 165   ({ __typeof (*(mem)) __oldval;                                              \
 166      __typeof (mem) __memp = (mem);                                           \
 167      __typeof (*(mem)) __value = (newvalue);                                  \
 168                                                                               \
 169      do                                                                       \
 170        __oldval = *__memp;                                                    \
 171      while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,   \
 172                                                                     __value,  \
 173                                                                     __oldval),\
 174                               0));                                            \
 175                                                                               \
 176      __oldval; })
 177 #endif
 178
 179 #ifndef atomic_exchange_rel
 180 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
 181 #endif
 182
 183
 184 /* Add VALUE to *MEM and return the old value of *MEM.  */
 185 #ifndef atomic_exchange_and_add
 186 # define atomic_exchange_and_add(mem, value) \
 187   ({ __typeof (*(mem)) __oldval;                                              \
 188      __typeof (mem) __memp = (mem);                                           \
 189      __typeof (*(mem)) __value = (value);                                     \
 190                                                                               \
 191      do                                                                       \
 192        __oldval = *__memp;                                                    \
 193      while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,   \
 194                                                                     __oldval  \
 195                                                                     + __value,\
 196                                                                     __oldval),\
 197                               0));                                            \
 198                                                                               \
 199      __oldval; })
 200 #endif
 201
 202
 203 #ifndef catomic_exchange_and_add
 204 # define catomic_exchange_and_add(mem, value) \
 205   ({ __typeof (*(mem)) __oldv;                                                \
 206      __typeof (mem) __memp = (mem);                                           \
 207      __typeof (*(mem)) __value = (value);                                     \
 208                                                                               \
 209      do                                                                       \
 210        __oldv = *__memp;                                                      \
 211      while (__builtin_expect (catomic_compare_and_exchange_bool_acq (__memp,  \
 212                                                                      __oldv   \
 213                                                                     + __value,\
 214                                                                      __oldv), \
 215                               0));                                            \
 216                                                                               \
 217      __oldv; })
 218 #endif
 219
 220
 221 #ifndef atomic_max
 222 # define atomic_max(mem, value) \
 223   do {                                                                        \
 224     __typeof (*(mem)) __oldval;                                               \
 225     __typeof (mem) __memp = (mem);                                            \
 226     __typeof (*(mem)) __value = (value);                                      \
 227     do {                                                                      \
 228       __oldval = *__memp;                                                     \
 229       if (__oldval >= __value)                                                \
 230         break;                                                                \
 231     } while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,  \
 232                                                                      __value, \
 233                                                                      __oldval),\
 234                                0));                                           \
 235   } while (0)
 236 #endif
 237
 238
 239 #ifndef catomic_max
 240 # define catomic_max(mem, value) \
 241   do {                                                                        \
 242     __typeof (*(mem)) __oldv;                                                 \
 243     __typeof (mem) __memp = (mem);                                            \
 244     __typeof (*(mem)) __value = (value);                                      \
 245     do {                                                                      \
 246       __oldv = *__memp;                                                       \
 247       if (__oldv >= __value)                                                  \
 248         break;                                                                \
 249     } while (__builtin_expect (catomic_compare_and_exchange_bool_acq (__memp, \
 250                                                                       __value,\
 251                                                                       __oldv),\
 252                                0));                                           \
 253   } while (0)
 254 #endif
 255
 256
 257 #ifndef atomic_min
 258 # define atomic_min(mem, value) \
 259   do {                                                                        \
 260     __typeof (*(mem)) __oldval;                                               \
 261     __typeof (mem) __memp = (mem);                                            \
 262     __typeof (*(mem)) __value = (value);                                      \
 263     do {                                                                      \
 264       __oldval = *__memp;                                                     \
 265       if (__oldval <= __value)                                                \
 266         break;                                                                \
 267     } while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,  \
 268                                                                      __value, \
 269                                                                      __oldval),\
 270                                0));                                           \
 271   } while (0)
 272 #endif
 273
 274
 275 #ifndef atomic_add
 276 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
 277 #endif
 278
 279
 280 #ifndef catomic_add
 281 # define catomic_add(mem, value) \
 282   (void) catomic_exchange_and_add ((mem), (value))
 283 #endif
 284
 285
 286 #ifndef atomic_increment
 287 # define atomic_increment(mem) atomic_add ((mem), 1)
 288 #endif
 289
 290
 291 #ifndef catomic_increment
 292 # define catomic_increment(mem) catomic_add ((mem), 1)
 293 #endif
 294
 295
 296 #ifndef atomic_increment_val
 297 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
 298 #endif
 299
 300
 301 #ifndef catomic_increment_val
 302 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
 303 #endif
 304
 305
 306 /* Add one to *MEM and return true iff it's now zero.  */
 307 #ifndef atomic_increment_and_test
 308 # define atomic_increment_and_test(mem) \
 309   (atomic_exchange_and_add ((mem), 1) + 1 == 0)
 310 #endif
 311
 312
 313 #ifndef atomic_decrement
 314 # define atomic_decrement(mem) atomic_add ((mem), -1)
 315 #endif
 316
 317
 318 #ifndef catomic_decrement
 319 # define catomic_decrement(mem) catomic_add ((mem), -1)
 320 #endif
 321
 322
 323 #ifndef atomic_decrement_val
 324 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
 325 #endif
 326
 327
 328 #ifndef catomic_decrement_val
 329 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
 330 #endif
 331
 332
 333 /* Subtract 1 from *MEM and return true iff it's now zero.  */
 334 #ifndef atomic_decrement_and_test
 335 # define atomic_decrement_and_test(mem) \
 336   (atomic_exchange_and_add ((mem), -1) == 1)
 337 #endif
 338
 339
 340 /* Decrement *MEM if it is > 0, and return the old value.  */
 341 #ifndef atomic_decrement_if_positive
 342 # define atomic_decrement_if_positive(mem) \
 343   ({ __typeof (*(mem)) __oldval;                                              \
 344      __typeof (mem) __memp = (mem);                                           \
 345                                                                               \
 346      do                                                                       \
 347        {                                                                      \
 348          __oldval = *__memp;                                                  \
 349          if (__builtin_expect (__oldval <= 0, 0))                             \
 350            break;                                                             \
 351        }                                                                      \
 352      while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,   \
 353                                                                     __oldval  \
 354                                                                     - 1,      \
 355                                                                     __oldval),\
 356                               0));\
 357      __oldval; })
 358 #endif
 359
 360
 361 #ifndef atomic_add_negative
 362 # define atomic_add_negative(mem, value)                                      \
 363   ({ __typeof (value) __aan_value = (value);                                  \
 364      atomic_exchange_and_add (mem, __aan_value) < -__aan_value; })
 365 #endif
 366
 367
 368 #ifndef atomic_add_zero
 369 # define atomic_add_zero(mem, value)                                          \
 370   ({ __typeof (value) __aaz_value = (value);                                  \
 371      atomic_exchange_and_add (mem, __aaz_value) == -__aaz_value; })
 372 #endif
 373
 374
 375 #ifndef atomic_bit_set
 376 # define atomic_bit_set(mem, bit) \
 377   (void) atomic_bit_test_set(mem, bit)
 378 #endif
 379
 380
 381 #ifndef atomic_bit_test_set
 382 # define atomic_bit_test_set(mem, bit) \
 383   ({ __typeof (*(mem)) __oldval;                                              \
 384      __typeof (mem) __memp = (mem);                                           \
 385      __typeof (*(mem)) __mask = ((__typeof (*(mem))) 1 << (bit));             \
 386                                                                               \
 387      do                                                                       \
 388        __oldval = (*__memp);                                                  \
 389      while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,   \
 390                                                                     __oldval  \
 391                                                                     | __mask, \
 392                                                                     __oldval),\
 393                               0));                                            \
 394                                                                               \
 395      __oldval & __mask; })
 396 #endif
 397
 398 /* Atomically *mem &= mask.  */
 399 #ifndef atomic_and
 400 # define atomic_and(mem, mask) \
 401   do {                                                                        \
 402     __typeof (*(mem)) __oldval;                                               \
 403     __typeof (mem) __memp = (mem);                                            \
 404     __typeof (*(mem)) __mask = (mask);                                        \
 405                                                                               \
 406     do                                                                        \
 407       __oldval = (*__memp);                                                   \
 408     while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,    \
 409                                                                    __oldval   \
 410                                                                    & __mask,  \
 411                                                                    __oldval), \
 412                              0));                                             \
 413   } while (0)
 414 #endif
 415
 416 /* Atomically *mem &= mask and return the old value of *mem.  */
 417 #ifndef atomic_and_val
 418 # define atomic_and_val(mem, mask) \
 419   ({ __typeof (*(mem)) __oldval;                                              \
 420      __typeof (mem) __memp = (mem);                                           \
 421      __typeof (*(mem)) __mask = (mask);                                       \
 422                                                                               \
 423      do                                                                       \
 424        __oldval = (*__memp);                                                  \
 425      while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,   \
 426                                                                     __oldval  \
 427                                                                     & __mask, \
 428                                                                     __oldval),\
 429                               0));                                            \
 430                                                                               \
 431      __oldval; })
 432 #endif
 433
 434 /* Atomically *mem |= mask and return the old value of *mem.  */
 435 #ifndef atomic_or
 436 # define atomic_or(mem, mask) \
 437   do {                                                                        \
 438     __typeof (*(mem)) __oldval;                                               \
 439     __typeof (mem) __memp = (mem);                                            \
 440     __typeof (*(mem)) __mask = (mask);                                        \
 441                                                                               \
 442     do                                                                        \
 443       __oldval = (*__memp);                                                   \
 444     while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,    \
 445                                                                    __oldval   \
 446                                                                    | __mask,  \
 447                                                                    __oldval), \
 448                               0));                                            \
 449   } while (0)
 450 #endif
 451
 452 #ifndef catomic_or
 453 # define catomic_or(mem, mask) \
 454   do {                                                                        \
 455     __typeof (*(mem)) __oldval;                                               \
 456     __typeof (mem) __memp = (mem);                                            \
 457     __typeof (*(mem)) __mask = (mask);                                        \
 458                                                                               \
 459     do                                                                        \
 460       __oldval = (*__memp);                                                   \
 461     while (__builtin_expect (catomic_compare_and_exchange_bool_acq (__memp,   \
 462                                                                     __oldval  \
 463                                                                     | __mask, \
 464                                                                     __oldval),\
 465                               0));                                            \
 466   } while (0)
 467 #endif
 468
 469 /* Atomically *mem |= mask and return the old value of *mem.  */
 470 #ifndef atomic_or_val
 471 # define atomic_or_val(mem, mask) \
 472   ({ __typeof (*(mem)) __oldval;                                              \
 473      __typeof (mem) __memp = (mem);                                           \
 474      __typeof (*(mem)) __mask = (mask);                                       \
 475                                                                               \
 476      do                                                                       \
 477        __oldval = (*__memp);                                                  \
 478      while (__builtin_expect (atomic_compare_and_exchange_bool_acq (__memp,   \
 479                                                                     __oldval  \
 480                                                                     | __mask, \
 481                                                                     __oldval),\
 482                               0));                                            \
 483                                                                               \
 484      __oldval; })
 485 #endif
 486
 487 #ifndef atomic_full_barrier
 488 # define atomic_full_barrier() __asm ("" ::: "memory")
 489 #endif
 490
 491
 492 #ifndef atomic_read_barrier
 493 # define atomic_read_barrier() atomic_full_barrier ()
 494 #endif
 495
 496
 497 #ifndef atomic_write_barrier
 498 # define atomic_write_barrier() atomic_full_barrier ()
 499 #endif
 500
 501
 502 #ifndef atomic_delay
 503 # define atomic_delay() do { /* nothing */ } while (0)
 504 #endif
 505
 506 #endif  /* atomic.h */