src/atomic_ops/sysdeps/gcc/generic.h

   1 /*
   2  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   3  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   4  * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
   5  * Copyright (c) 2013-2017 Ivan Maidanski
   6  *
   7  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
   8  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
   9  *
  10  * Permission is hereby granted to use or copy this program
  11  * for any purpose, provided the above notices are retained on all copies.
  12  * Permission to modify the code and to distribute modified code is granted,
  13  * provided the above notices are retained, and a notice that the code was
  14  * modified is included with the above copyright notice.
  15  *
  16  */
  17
  18 /* The following implementation assumes GCC 4.7 or later.               */
  19 /* For the details, see GNU Manual, chapter 6.52 (Built-in functions    */
  20 /* for memory model aware atomic operations).                           */
  21
  22 #define AO_GCC_ATOMIC_TEST_AND_SET
  23 #include "../test_and_set_t_is_char.h"
  24
  25 #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) \
  26     || defined(AO_GCC_FORCE_HAVE_CAS)
  27 # define AO_GCC_HAVE_char_SYNC_CAS
  28 #endif
  29
  30 #if (__SIZEOF_SHORT__ == 2 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2)) \
  31     || defined(AO_GCC_FORCE_HAVE_CAS)
  32 # define AO_GCC_HAVE_short_SYNC_CAS
  33 #endif
  34
  35 #if (__SIZEOF_INT__ == 4 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)) \
  36     || (__SIZEOF_INT__ == 8 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)) \
  37     || defined(AO_GCC_FORCE_HAVE_CAS)
  38 # define AO_GCC_HAVE_int_SYNC_CAS
  39 #endif
  40
  41 #if (__SIZEOF_SIZE_T__ == 4 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)) \
  42     || (__SIZEOF_SIZE_T__ == 8 \
  43         && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)) \
  44     || defined(AO_GCC_FORCE_HAVE_CAS)
  45 # define AO_GCC_HAVE_SYNC_CAS
  46 #endif
  47
  48 #undef AO_compiler_barrier
  49 #define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)
  50
  51 #ifdef AO_UNIPROCESSOR
  52   /* If only a single processor (core) is used, AO_UNIPROCESSOR could   */
  53   /* be defined by the client to avoid unnecessary memory barrier.      */
  54   AO_INLINE void
  55   AO_nop_full(void)
  56   {
  57     AO_compiler_barrier();
  58   }
  59 # define AO_HAVE_nop_full
  60
  61 #else
  62   AO_INLINE void
  63   AO_nop_read(void)
  64   {
  65     __atomic_thread_fence(__ATOMIC_ACQUIRE);
  66   }
  67 # define AO_HAVE_nop_read
  68
  69 # ifndef AO_HAVE_nop_write
  70     AO_INLINE void
  71     AO_nop_write(void)
  72     {
  73       __atomic_thread_fence(__ATOMIC_RELEASE);
  74     }
  75 #   define AO_HAVE_nop_write
  76 # endif
  77
  78   AO_INLINE void
  79   AO_nop_full(void)
  80   {
  81     /* __sync_synchronize() could be used instead.      */
  82     __atomic_thread_fence(__ATOMIC_SEQ_CST);
  83   }
  84 # define AO_HAVE_nop_full
  85 #endif /* !AO_UNIPROCESSOR */
  86
  87 #include "generic-small.h"
  88
  89 #ifndef AO_PREFER_GENERALIZED
  90 # include "generic-arithm.h"
  91
  92 # define AO_CLEAR(addr) __atomic_clear(addr, __ATOMIC_RELEASE)
  93 # define AO_HAVE_CLEAR
  94
  95   AO_INLINE AO_TS_VAL_t
  96   AO_test_and_set(volatile AO_TS_t *addr)
  97   {
  98     return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELAXED);
  99   }
 100 # define AO_HAVE_test_and_set
 101
 102   AO_INLINE AO_TS_VAL_t
 103   AO_test_and_set_acquire(volatile AO_TS_t *addr)
 104   {
 105     return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_ACQUIRE);
 106   }
 107 # define AO_HAVE_test_and_set_acquire
 108
 109   AO_INLINE AO_TS_VAL_t
 110   AO_test_and_set_release(volatile AO_TS_t *addr)
 111   {
 112     return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_RELEASE);
 113   }
 114 # define AO_HAVE_test_and_set_release
 115
 116   AO_INLINE AO_TS_VAL_t
 117   AO_test_and_set_full(volatile AO_TS_t *addr)
 118   {
 119     return (AO_TS_VAL_t)__atomic_test_and_set(addr, __ATOMIC_SEQ_CST);
 120   }
 121 # define AO_HAVE_test_and_set_full
 122 #endif /* !AO_PREFER_GENERALIZED */
 123
 124 #ifdef AO_HAVE_DOUBLE_PTR_STORAGE
 125
 126 # if ((__SIZEOF_SIZE_T__ == 4 \
 127        && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)) \
 128       || (__SIZEOF_SIZE_T__ == 8 /* half of AO_double_t */ \
 129           && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16))) \
 130      && !defined(AO_SKIPATOMIC_double_compare_and_swap_ANY)
 131 #   define AO_GCC_HAVE_double_SYNC_CAS
 132 # endif
 133
 134 # if !defined(AO_GCC_HAVE_double_SYNC_CAS) || !defined(AO_PREFER_GENERALIZED)
 135
 136 # if !defined(AO_HAVE_double_load) && !defined(AO_SKIPATOMIC_double_load)
 137     AO_INLINE AO_double_t
 138     AO_double_load(const volatile AO_double_t *addr)
 139     {
 140       AO_double_t result;
 141
 142       result.AO_whole = __atomic_load_n(&addr->AO_whole, __ATOMIC_RELAXED);
 143       return result;
 144     }
 145 #   define AO_HAVE_double_load
 146 # endif
 147
 148 # if !defined(AO_HAVE_double_load_acquire) \
 149      && !defined(AO_SKIPATOMIC_double_load_acquire)
 150     AO_INLINE AO_double_t
 151     AO_double_load_acquire(const volatile AO_double_t *addr)
 152     {
 153       AO_double_t result;
 154
 155       result.AO_whole = __atomic_load_n(&addr->AO_whole, __ATOMIC_ACQUIRE);
 156       return result;
 157     }
 158 #   define AO_HAVE_double_load_acquire
 159 # endif
 160
 161 # if !defined(AO_HAVE_double_store) && !defined(AO_SKIPATOMIC_double_store)
 162     AO_INLINE void
 163     AO_double_store(volatile AO_double_t *addr, AO_double_t value)
 164     {
 165       __atomic_store_n(&addr->AO_whole, value.AO_whole, __ATOMIC_RELAXED);
 166     }
 167 #   define AO_HAVE_double_store
 168 # endif
 169
 170 # if !defined(AO_HAVE_double_store_release) \
 171      && !defined(AO_SKIPATOMIC_double_store_release)
 172     AO_INLINE void
 173     AO_double_store_release(volatile AO_double_t *addr, AO_double_t value)
 174     {
 175       __atomic_store_n(&addr->AO_whole, value.AO_whole, __ATOMIC_RELEASE);
 176     }
 177 #   define AO_HAVE_double_store_release
 178 # endif
 179
 180 #endif /* !AO_GCC_HAVE_double_SYNC_CAS || !AO_PREFER_GENERALIZED */
 181
 182 #endif /* AO_HAVE_DOUBLE_PTR_STORAGE */
 183
 184 #ifdef AO_GCC_HAVE_double_SYNC_CAS
 185 # ifndef AO_HAVE_double_compare_and_swap
 186     AO_INLINE int
 187     AO_double_compare_and_swap(volatile AO_double_t *addr,
 188                                AO_double_t old_val, AO_double_t new_val)
 189     {
 190       return (int)__atomic_compare_exchange_n(&addr->AO_whole,
 191                                 &old_val.AO_whole /* p_expected */,
 192                                 new_val.AO_whole /* desired */,
 193                                 0 /* is_weak: false */,
 194                                 __ATOMIC_RELAXED /* success */,
 195                                 __ATOMIC_RELAXED /* failure */);
 196     }
 197 #   define AO_HAVE_double_compare_and_swap
 198 # endif
 199
 200 # ifndef AO_HAVE_double_compare_and_swap_acquire
 201     AO_INLINE int
 202     AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
 203                                        AO_double_t old_val,
 204                                        AO_double_t new_val)
 205     {
 206       return (int)__atomic_compare_exchange_n(&addr->AO_whole,
 207                                 &old_val.AO_whole, new_val.AO_whole, 0,
 208                                 __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
 209     }
 210 #   define AO_HAVE_double_compare_and_swap_acquire
 211 # endif
 212
 213 # ifndef AO_HAVE_double_compare_and_swap_release
 214     AO_INLINE int
 215     AO_double_compare_and_swap_release(volatile AO_double_t *addr,
 216                                        AO_double_t old_val,
 217                                        AO_double_t new_val)
 218     {
 219       return (int)__atomic_compare_exchange_n(&addr->AO_whole,
 220                                 &old_val.AO_whole, new_val.AO_whole, 0,
 221                                 __ATOMIC_RELEASE,
 222                                 __ATOMIC_RELAXED /* failure */);
 223     }
 224 #   define AO_HAVE_double_compare_and_swap_release
 225 # endif
 226
 227 # ifndef AO_HAVE_double_compare_and_swap_full
 228     AO_INLINE int
 229     AO_double_compare_and_swap_full(volatile AO_double_t *addr,
 230                                     AO_double_t old_val, AO_double_t new_val)
 231     {
 232       return (int)__atomic_compare_exchange_n(&addr->AO_whole,
 233                                 &old_val.AO_whole, new_val.AO_whole, 0,
 234                                 __ATOMIC_ACQ_REL,
 235                                 __ATOMIC_ACQUIRE /* failure */);
 236     }
 237 #   define AO_HAVE_double_compare_and_swap_full
 238 # endif
 239 #endif /* AO_GCC_HAVE_double_SYNC_CAS */