src/base/atomicops_internals_tsan.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 // This file is an internal atomic implementation for compiler-based
   6 // ThreadSanitizer. Use base/atomicops.h instead.
   7
   8 #ifndef BASE_ATOMICOPS_INTERNALS_TSAN_H_
   9 #define BASE_ATOMICOPS_INTERNALS_TSAN_H_
  10
  11 #include "base/base_export.h"
  12
  13 // This struct is not part of the public API of this module; clients may not
  14 // use it.  (However, it's exported via BASE_EXPORT because clients implicitly
  15 // do use it at link time by inlining these functions.)
  16 // Features of this x86.  Values may not be correct before main() is run,
  17 // but are set conservatively.
  18 struct AtomicOps_x86CPUFeatureStruct {
  19   bool has_amd_lock_mb_bug;  // Processor has AMD memory-barrier bug; do lfence
  20                              // after acquire compare-and-swap.
  21   bool has_sse2;             // Processor has SSE2.
  22 };
  23 BASE_EXPORT extern struct AtomicOps_x86CPUFeatureStruct
  24     AtomicOps_Internalx86CPUFeatures;
  25
  26 #define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory")
  27
  28 namespace base {
  29 namespace subtle {
  30
  31 #ifndef TSAN_INTERFACE_ATOMIC_H
  32 #define TSAN_INTERFACE_ATOMIC_H
  33
  34 #ifdef __cplusplus
  35 extern "C" {
  36 #endif
  37
  38 typedef char  __tsan_atomic8;
  39 typedef short __tsan_atomic16;  // NOLINT
  40 typedef int   __tsan_atomic32;
  41 typedef long  __tsan_atomic64;  // NOLINT
  42
  43 #if defined(__SIZEOF_INT128__) \
  44     || (__clang_major__ * 100 + __clang_minor__ >= 302)
  45 typedef __int128 __tsan_atomic128;
  46 #define __TSAN_HAS_INT128 1
  47 #else
  48 typedef char     __tsan_atomic128;
  49 #define __TSAN_HAS_INT128 0
  50 #endif
  51
  52 typedef enum {
  53   __tsan_memory_order_relaxed,
  54   __tsan_memory_order_consume,
  55   __tsan_memory_order_acquire,
  56   __tsan_memory_order_release,
  57   __tsan_memory_order_acq_rel,
  58   __tsan_memory_order_seq_cst,
  59 } __tsan_memory_order;
  60
  61 __tsan_atomic8 __tsan_atomic8_load(const volatile __tsan_atomic8 *a,
  62     __tsan_memory_order mo);
  63 __tsan_atomic16 __tsan_atomic16_load(const volatile __tsan_atomic16 *a,
  64     __tsan_memory_order mo);
  65 __tsan_atomic32 __tsan_atomic32_load(const volatile __tsan_atomic32 *a,
  66     __tsan_memory_order mo);
  67 __tsan_atomic64 __tsan_atomic64_load(const volatile __tsan_atomic64 *a,
  68     __tsan_memory_order mo);
  69 __tsan_atomic128 __tsan_atomic128_load(const volatile __tsan_atomic128 *a,
  70     __tsan_memory_order mo);
  71
  72 void __tsan_atomic8_store(volatile __tsan_atomic8 *a, __tsan_atomic8 v,
  73     __tsan_memory_order mo);
  74 void __tsan_atomic16_store(volatile __tsan_atomic16 *a, __tsan_atomic16 v,
  75     __tsan_memory_order mo);
  76 void __tsan_atomic32_store(volatile __tsan_atomic32 *a, __tsan_atomic32 v,
  77     __tsan_memory_order mo);
  78 void __tsan_atomic64_store(volatile __tsan_atomic64 *a, __tsan_atomic64 v,
  79     __tsan_memory_order mo);
  80 void __tsan_atomic128_store(volatile __tsan_atomic128 *a, __tsan_atomic128 v,
  81     __tsan_memory_order mo);
  82
  83 __tsan_atomic8 __tsan_atomic8_exchange(volatile __tsan_atomic8 *a,
  84     __tsan_atomic8 v, __tsan_memory_order mo);
  85 __tsan_atomic16 __tsan_atomic16_exchange(volatile __tsan_atomic16 *a,
  86     __tsan_atomic16 v, __tsan_memory_order mo);
  87 __tsan_atomic32 __tsan_atomic32_exchange(volatile __tsan_atomic32 *a,
  88     __tsan_atomic32 v, __tsan_memory_order mo);
  89 __tsan_atomic64 __tsan_atomic64_exchange(volatile __tsan_atomic64 *a,
  90     __tsan_atomic64 v, __tsan_memory_order mo);
  91 __tsan_atomic128 __tsan_atomic128_exchange(volatile __tsan_atomic128 *a,
  92     __tsan_atomic128 v, __tsan_memory_order mo);
  93
  94 __tsan_atomic8 __tsan_atomic8_fetch_add(volatile __tsan_atomic8 *a,
  95     __tsan_atomic8 v, __tsan_memory_order mo);
  96 __tsan_atomic16 __tsan_atomic16_fetch_add(volatile __tsan_atomic16 *a,
  97     __tsan_atomic16 v, __tsan_memory_order mo);
  98 __tsan_atomic32 __tsan_atomic32_fetch_add(volatile __tsan_atomic32 *a,
  99     __tsan_atomic32 v, __tsan_memory_order mo);
 100 __tsan_atomic64 __tsan_atomic64_fetch_add(volatile __tsan_atomic64 *a,
 101     __tsan_atomic64 v, __tsan_memory_order mo);
 102 __tsan_atomic128 __tsan_atomic128_fetch_add(volatile __tsan_atomic128 *a,
 103     __tsan_atomic128 v, __tsan_memory_order mo);
 104
 105 __tsan_atomic8 __tsan_atomic8_fetch_and(volatile __tsan_atomic8 *a,
 106     __tsan_atomic8 v, __tsan_memory_order mo);
 107 __tsan_atomic16 __tsan_atomic16_fetch_and(volatile __tsan_atomic16 *a,
 108     __tsan_atomic16 v, __tsan_memory_order mo);
 109 __tsan_atomic32 __tsan_atomic32_fetch_and(volatile __tsan_atomic32 *a,
 110     __tsan_atomic32 v, __tsan_memory_order mo);
 111 __tsan_atomic64 __tsan_atomic64_fetch_and(volatile __tsan_atomic64 *a,
 112     __tsan_atomic64 v, __tsan_memory_order mo);
 113 __tsan_atomic128 __tsan_atomic128_fetch_and(volatile __tsan_atomic128 *a,
 114     __tsan_atomic128 v, __tsan_memory_order mo);
 115
 116 __tsan_atomic8 __tsan_atomic8_fetch_or(volatile __tsan_atomic8 *a,
 117     __tsan_atomic8 v, __tsan_memory_order mo);
 118 __tsan_atomic16 __tsan_atomic16_fetch_or(volatile __tsan_atomic16 *a,
 119     __tsan_atomic16 v, __tsan_memory_order mo);
 120 __tsan_atomic32 __tsan_atomic32_fetch_or(volatile __tsan_atomic32 *a,
 121     __tsan_atomic32 v, __tsan_memory_order mo);
 122 __tsan_atomic64 __tsan_atomic64_fetch_or(volatile __tsan_atomic64 *a,
 123     __tsan_atomic64 v, __tsan_memory_order mo);
 124 __tsan_atomic128 __tsan_atomic128_fetch_or(volatile __tsan_atomic128 *a,
 125     __tsan_atomic128 v, __tsan_memory_order mo);
 126
 127 __tsan_atomic8 __tsan_atomic8_fetch_xor(volatile __tsan_atomic8 *a,
 128     __tsan_atomic8 v, __tsan_memory_order mo);
 129 __tsan_atomic16 __tsan_atomic16_fetch_xor(volatile __tsan_atomic16 *a,
 130     __tsan_atomic16 v, __tsan_memory_order mo);
 131 __tsan_atomic32 __tsan_atomic32_fetch_xor(volatile __tsan_atomic32 *a,
 132     __tsan_atomic32 v, __tsan_memory_order mo);
 133 __tsan_atomic64 __tsan_atomic64_fetch_xor(volatile __tsan_atomic64 *a,
 134     __tsan_atomic64 v, __tsan_memory_order mo);
 135 __tsan_atomic128 __tsan_atomic128_fetch_xor(volatile __tsan_atomic128 *a,
 136     __tsan_atomic128 v, __tsan_memory_order mo);
 137
 138 __tsan_atomic8 __tsan_atomic8_fetch_nand(volatile __tsan_atomic8 *a,
 139     __tsan_atomic8 v, __tsan_memory_order mo);
 140 __tsan_atomic16 __tsan_atomic16_fetch_nand(volatile __tsan_atomic16 *a,
 141     __tsan_atomic16 v, __tsan_memory_order mo);
 142 __tsan_atomic32 __tsan_atomic32_fetch_nand(volatile __tsan_atomic32 *a,
 143     __tsan_atomic32 v, __tsan_memory_order mo);
 144 __tsan_atomic64 __tsan_atomic64_fetch_nand(volatile __tsan_atomic64 *a,
 145     __tsan_atomic64 v, __tsan_memory_order mo);
 146 __tsan_atomic128 __tsan_atomic128_fetch_nand(volatile __tsan_atomic128 *a,
 147     __tsan_atomic128 v, __tsan_memory_order mo);
 148
 149 int __tsan_atomic8_compare_exchange_weak(volatile __tsan_atomic8 *a,
 150     __tsan_atomic8 *c, __tsan_atomic8 v, __tsan_memory_order mo,
 151     __tsan_memory_order fail_mo);
 152 int __tsan_atomic16_compare_exchange_weak(volatile __tsan_atomic16 *a,
 153     __tsan_atomic16 *c, __tsan_atomic16 v, __tsan_memory_order mo,
 154     __tsan_memory_order fail_mo);
 155 int __tsan_atomic32_compare_exchange_weak(volatile __tsan_atomic32 *a,
 156     __tsan_atomic32 *c, __tsan_atomic32 v, __tsan_memory_order mo,
 157     __tsan_memory_order fail_mo);
 158 int __tsan_atomic64_compare_exchange_weak(volatile __tsan_atomic64 *a,
 159     __tsan_atomic64 *c, __tsan_atomic64 v, __tsan_memory_order mo,
 160     __tsan_memory_order fail_mo);
 161 int __tsan_atomic128_compare_exchange_weak(volatile __tsan_atomic128 *a,
 162     __tsan_atomic128 *c, __tsan_atomic128 v, __tsan_memory_order mo,
 163     __tsan_memory_order fail_mo);
 164
 165 int __tsan_atomic8_compare_exchange_strong(volatile __tsan_atomic8 *a,
 166     __tsan_atomic8 *c, __tsan_atomic8 v, __tsan_memory_order mo,
 167     __tsan_memory_order fail_mo);
 168 int __tsan_atomic16_compare_exchange_strong(volatile __tsan_atomic16 *a,
 169     __tsan_atomic16 *c, __tsan_atomic16 v, __tsan_memory_order mo,
 170     __tsan_memory_order fail_mo);
 171 int __tsan_atomic32_compare_exchange_strong(volatile __tsan_atomic32 *a,
 172     __tsan_atomic32 *c, __tsan_atomic32 v, __tsan_memory_order mo,
 173     __tsan_memory_order fail_mo);
 174 int __tsan_atomic64_compare_exchange_strong(volatile __tsan_atomic64 *a,
 175     __tsan_atomic64 *c, __tsan_atomic64 v, __tsan_memory_order mo,
 176     __tsan_memory_order fail_mo);
 177 int __tsan_atomic128_compare_exchange_strong(volatile __tsan_atomic128 *a,
 178     __tsan_atomic128 *c, __tsan_atomic128 v, __tsan_memory_order mo,
 179     __tsan_memory_order fail_mo);
 180
 181 __tsan_atomic8 __tsan_atomic8_compare_exchange_val(
 182     volatile __tsan_atomic8 *a, __tsan_atomic8 c, __tsan_atomic8 v,
 183     __tsan_memory_order mo, __tsan_memory_order fail_mo);
 184 __tsan_atomic16 __tsan_atomic16_compare_exchange_val(
 185     volatile __tsan_atomic16 *a, __tsan_atomic16 c, __tsan_atomic16 v,
 186     __tsan_memory_order mo, __tsan_memory_order fail_mo);
 187 __tsan_atomic32 __tsan_atomic32_compare_exchange_val(
 188     volatile __tsan_atomic32 *a, __tsan_atomic32 c, __tsan_atomic32 v,
 189     __tsan_memory_order mo, __tsan_memory_order fail_mo);
 190 __tsan_atomic64 __tsan_atomic64_compare_exchange_val(
 191     volatile __tsan_atomic64 *a, __tsan_atomic64 c, __tsan_atomic64 v,
 192     __tsan_memory_order mo, __tsan_memory_order fail_mo);
 193 __tsan_atomic128 __tsan_atomic128_compare_exchange_val(
 194     volatile __tsan_atomic128 *a, __tsan_atomic128 c, __tsan_atomic128 v,
 195     __tsan_memory_order mo, __tsan_memory_order fail_mo);
 196
 197 void __tsan_atomic_thread_fence(__tsan_memory_order mo);
 198 void __tsan_atomic_signal_fence(__tsan_memory_order mo);
 199
 200 #ifdef __cplusplus
 201 }  // extern "C"
 202 #endif
 203
 204 #endif  // #ifndef TSAN_INTERFACE_ATOMIC_H
 205
 206 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr,
 207                                   Atomic32 old_value,
 208                                   Atomic32 new_value) {
 209   Atomic32 cmp = old_value;
 210   __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value,
 211       __tsan_memory_order_relaxed, __tsan_memory_order_relaxed);
 212   return cmp;
 213 }
 214
 215 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr,
 216                                   Atomic32 new_value) {
 217   return __tsan_atomic32_exchange(ptr, new_value,
 218       __tsan_memory_order_relaxed);
 219 }
 220
 221 inline Atomic32 Acquire_AtomicExchange(volatile Atomic32 *ptr,
 222                                 Atomic32 new_value) {
 223   return __tsan_atomic32_exchange(ptr, new_value,
 224       __tsan_memory_order_acquire);
 225 }
 226
 227 inline Atomic32 Release_AtomicExchange(volatile Atomic32 *ptr,
 228                                 Atomic32 new_value) {
 229   return __tsan_atomic32_exchange(ptr, new_value,
 230       __tsan_memory_order_release);
 231 }
 232
 233 inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32 *ptr,
 234                                    Atomic32 increment) {
 235   return increment + __tsan_atomic32_fetch_add(ptr, increment,
 236       __tsan_memory_order_relaxed);
 237 }
 238
 239 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32 *ptr,
 240                                  Atomic32 increment) {
 241   return increment + __tsan_atomic32_fetch_add(ptr, increment,
 242       __tsan_memory_order_acq_rel);
 243 }
 244
 245 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr,
 246                                 Atomic32 old_value,
 247                                 Atomic32 new_value) {
 248   Atomic32 cmp = old_value;
 249   __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value,
 250       __tsan_memory_order_acquire, __tsan_memory_order_acquire);
 251   return cmp;
 252 }
 253
 254 inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr,
 255                                 Atomic32 old_value,
 256                                 Atomic32 new_value) {
 257   Atomic32 cmp = old_value;
 258   __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value,
 259       __tsan_memory_order_release, __tsan_memory_order_relaxed);
 260   return cmp;
 261 }
 262
 263 inline void NoBarrier_Store(volatile Atomic32 *ptr, Atomic32 value) {
 264   __tsan_atomic32_store(ptr, value, __tsan_memory_order_relaxed);
 265 }
 266
 267 inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) {
 268   __tsan_atomic32_store(ptr, value, __tsan_memory_order_relaxed);
 269   __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst);
 270 }
 271
 272 inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) {
 273   __tsan_atomic32_store(ptr, value, __tsan_memory_order_release);
 274 }
 275
 276 inline Atomic32 NoBarrier_Load(volatile const Atomic32 *ptr) {
 277   return __tsan_atomic32_load(ptr, __tsan_memory_order_relaxed);
 278 }
 279
 280 inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) {
 281   return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire);
 282 }
 283
 284 inline Atomic32 Release_Load(volatile const Atomic32 *ptr) {
 285   __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst);
 286   return __tsan_atomic32_load(ptr, __tsan_memory_order_relaxed);
 287 }
 288
 289 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr,
 290                                       Atomic64 old_value,
 291                                       Atomic64 new_value) {
 292   Atomic64 cmp = old_value;
 293   __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value,
 294       __tsan_memory_order_relaxed, __tsan_memory_order_relaxed);
 295   return cmp;
 296 }
 297
 298 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr,
 299                                       Atomic64 new_value) {
 300   return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_relaxed);
 301 }
 302
 303 inline Atomic64 Acquire_AtomicExchange(volatile Atomic64 *ptr,
 304                                     Atomic64 new_value) {
 305   return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_acquire);
 306 }
 307
 308 inline Atomic64 Release_AtomicExchange(volatile Atomic64 *ptr,
 309                                     Atomic64 new_value) {
 310   return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_release);
 311 }
 312
 313 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64 *ptr,
 314                                        Atomic64 increment) {
 315   return increment + __tsan_atomic64_fetch_add(ptr, increment,
 316       __tsan_memory_order_relaxed);
 317 }
 318
 319 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64 *ptr,
 320                                      Atomic64 increment) {
 321   return increment + __tsan_atomic64_fetch_add(ptr, increment,
 322       __tsan_memory_order_acq_rel);
 323 }
 324
 325 inline void NoBarrier_Store(volatile Atomic64 *ptr, Atomic64 value) {
 326   __tsan_atomic64_store(ptr, value, __tsan_memory_order_relaxed);
 327 }
 328
 329 inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) {
 330   __tsan_atomic64_store(ptr, value, __tsan_memory_order_relaxed);
 331   __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst);
 332 }
 333
 334 inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) {
 335   __tsan_atomic64_store(ptr, value, __tsan_memory_order_release);
 336 }
 337
 338 inline Atomic64 NoBarrier_Load(volatile const Atomic64 *ptr) {
 339   return __tsan_atomic64_load(ptr, __tsan_memory_order_relaxed);
 340 }
 341
 342 inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) {
 343   return __tsan_atomic64_load(ptr, __tsan_memory_order_acquire);
 344 }
 345
 346 inline Atomic64 Release_Load(volatile const Atomic64 *ptr) {
 347   __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst);
 348   return __tsan_atomic64_load(ptr, __tsan_memory_order_relaxed);
 349 }
 350
 351 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr,
 352                                     Atomic64 old_value,
 353                                     Atomic64 new_value) {
 354   Atomic64 cmp = old_value;
 355   __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value,
 356       __tsan_memory_order_acquire, __tsan_memory_order_acquire);
 357   return cmp;
 358 }
 359
 360 inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr,
 361                                     Atomic64 old_value,
 362                                     Atomic64 new_value) {
 363   Atomic64 cmp = old_value;
 364   __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value,
 365       __tsan_memory_order_release, __tsan_memory_order_relaxed);
 366   return cmp;
 367 }
 368
 369 inline void MemoryBarrier() {
 370   __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst);
 371 }
 372
 373 }  // namespace base::subtle
 374 }  // namespace base
 375
 376 #undef ATOMICOPS_COMPILER_BARRIER
 377
 378 #endif  // BASE_ATOMICOPS_INTERNALS_TSAN_H_