include/linux/atomic/atomic-arch-fallback.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-fallback.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 #ifndef _LINUX_ATOMIC_FALLBACK_H
   7 #define _LINUX_ATOMIC_FALLBACK_H
   8
   9 #include <linux/compiler.h>
  10
  11 #if defined(arch_xchg)
  12 #define raw_xchg arch_xchg
  13 #elif defined(arch_xchg_relaxed)
  14 #define raw_xchg(...) \
  15         __atomic_op_fence(arch_xchg, __VA_ARGS__)
  16 #else
  17 extern void raw_xchg_not_implemented(void);
  18 #define raw_xchg(...) raw_xchg_not_implemented()
  19 #endif
  20
  21 #if defined(arch_xchg_acquire)
  22 #define raw_xchg_acquire arch_xchg_acquire
  23 #elif defined(arch_xchg_relaxed)
  24 #define raw_xchg_acquire(...) \
  25         __atomic_op_acquire(arch_xchg, __VA_ARGS__)
  26 #elif defined(arch_xchg)
  27 #define raw_xchg_acquire arch_xchg
  28 #else
  29 extern void raw_xchg_acquire_not_implemented(void);
  30 #define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
  31 #endif
  32
  33 #if defined(arch_xchg_release)
  34 #define raw_xchg_release arch_xchg_release
  35 #elif defined(arch_xchg_relaxed)
  36 #define raw_xchg_release(...) \
  37         __atomic_op_release(arch_xchg, __VA_ARGS__)
  38 #elif defined(arch_xchg)
  39 #define raw_xchg_release arch_xchg
  40 #else
  41 extern void raw_xchg_release_not_implemented(void);
  42 #define raw_xchg_release(...) raw_xchg_release_not_implemented()
  43 #endif
  44
  45 #if defined(arch_xchg_relaxed)
  46 #define raw_xchg_relaxed arch_xchg_relaxed
  47 #elif defined(arch_xchg)
  48 #define raw_xchg_relaxed arch_xchg
  49 #else
  50 extern void raw_xchg_relaxed_not_implemented(void);
  51 #define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
  52 #endif
  53
  54 #if defined(arch_cmpxchg)
  55 #define raw_cmpxchg arch_cmpxchg
  56 #elif defined(arch_cmpxchg_relaxed)
  57 #define raw_cmpxchg(...) \
  58         __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
  59 #else
  60 extern void raw_cmpxchg_not_implemented(void);
  61 #define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
  62 #endif
  63
  64 #if defined(arch_cmpxchg_acquire)
  65 #define raw_cmpxchg_acquire arch_cmpxchg_acquire
  66 #elif defined(arch_cmpxchg_relaxed)
  67 #define raw_cmpxchg_acquire(...) \
  68         __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
  69 #elif defined(arch_cmpxchg)
  70 #define raw_cmpxchg_acquire arch_cmpxchg
  71 #else
  72 extern void raw_cmpxchg_acquire_not_implemented(void);
  73 #define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
  74 #endif
  75
  76 #if defined(arch_cmpxchg_release)
  77 #define raw_cmpxchg_release arch_cmpxchg_release
  78 #elif defined(arch_cmpxchg_relaxed)
  79 #define raw_cmpxchg_release(...) \
  80         __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
  81 #elif defined(arch_cmpxchg)
  82 #define raw_cmpxchg_release arch_cmpxchg
  83 #else
  84 extern void raw_cmpxchg_release_not_implemented(void);
  85 #define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
  86 #endif
  87
  88 #if defined(arch_cmpxchg_relaxed)
  89 #define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
  90 #elif defined(arch_cmpxchg)
  91 #define raw_cmpxchg_relaxed arch_cmpxchg
  92 #else
  93 extern void raw_cmpxchg_relaxed_not_implemented(void);
  94 #define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
  95 #endif
  96
  97 #if defined(arch_cmpxchg64)
  98 #define raw_cmpxchg64 arch_cmpxchg64
  99 #elif defined(arch_cmpxchg64_relaxed)
 100 #define raw_cmpxchg64(...) \
 101         __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
 102 #else
 103 extern void raw_cmpxchg64_not_implemented(void);
 104 #define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
 105 #endif
 106
 107 #if defined(arch_cmpxchg64_acquire)
 108 #define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
 109 #elif defined(arch_cmpxchg64_relaxed)
 110 #define raw_cmpxchg64_acquire(...) \
 111         __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
 112 #elif defined(arch_cmpxchg64)
 113 #define raw_cmpxchg64_acquire arch_cmpxchg64
 114 #else
 115 extern void raw_cmpxchg64_acquire_not_implemented(void);
 116 #define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
 117 #endif
 118
 119 #if defined(arch_cmpxchg64_release)
 120 #define raw_cmpxchg64_release arch_cmpxchg64_release
 121 #elif defined(arch_cmpxchg64_relaxed)
 122 #define raw_cmpxchg64_release(...) \
 123         __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
 124 #elif defined(arch_cmpxchg64)
 125 #define raw_cmpxchg64_release arch_cmpxchg64
 126 #else
 127 extern void raw_cmpxchg64_release_not_implemented(void);
 128 #define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
 129 #endif
 130
 131 #if defined(arch_cmpxchg64_relaxed)
 132 #define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
 133 #elif defined(arch_cmpxchg64)
 134 #define raw_cmpxchg64_relaxed arch_cmpxchg64
 135 #else
 136 extern void raw_cmpxchg64_relaxed_not_implemented(void);
 137 #define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
 138 #endif
 139
 140 #if defined(arch_cmpxchg128)
 141 #define raw_cmpxchg128 arch_cmpxchg128
 142 #elif defined(arch_cmpxchg128_relaxed)
 143 #define raw_cmpxchg128(...) \
 144         __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
 145 #else
 146 extern void raw_cmpxchg128_not_implemented(void);
 147 #define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
 148 #endif
 149
 150 #if defined(arch_cmpxchg128_acquire)
 151 #define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
 152 #elif defined(arch_cmpxchg128_relaxed)
 153 #define raw_cmpxchg128_acquire(...) \
 154         __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
 155 #elif defined(arch_cmpxchg128)
 156 #define raw_cmpxchg128_acquire arch_cmpxchg128
 157 #else
 158 extern void raw_cmpxchg128_acquire_not_implemented(void);
 159 #define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
 160 #endif
 161
 162 #if defined(arch_cmpxchg128_release)
 163 #define raw_cmpxchg128_release arch_cmpxchg128_release
 164 #elif defined(arch_cmpxchg128_relaxed)
 165 #define raw_cmpxchg128_release(...) \
 166         __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
 167 #elif defined(arch_cmpxchg128)
 168 #define raw_cmpxchg128_release arch_cmpxchg128
 169 #else
 170 extern void raw_cmpxchg128_release_not_implemented(void);
 171 #define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
 172 #endif
 173
 174 #if defined(arch_cmpxchg128_relaxed)
 175 #define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
 176 #elif defined(arch_cmpxchg128)
 177 #define raw_cmpxchg128_relaxed arch_cmpxchg128
 178 #else
 179 extern void raw_cmpxchg128_relaxed_not_implemented(void);
 180 #define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
 181 #endif
 182
 183 #if defined(arch_try_cmpxchg)
 184 #define raw_try_cmpxchg arch_try_cmpxchg
 185 #elif defined(arch_try_cmpxchg_relaxed)
 186 #define raw_try_cmpxchg(...) \
 187         __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
 188 #else
 189 #define raw_try_cmpxchg(_ptr, _oldp, _new) \
 190 ({ \
 191         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 192         ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
 193         if (unlikely(___r != ___o)) \
 194                 *___op = ___r; \
 195         likely(___r == ___o); \
 196 })
 197 #endif
 198
 199 #if defined(arch_try_cmpxchg_acquire)
 200 #define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
 201 #elif defined(arch_try_cmpxchg_relaxed)
 202 #define raw_try_cmpxchg_acquire(...) \
 203         __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
 204 #elif defined(arch_try_cmpxchg)
 205 #define raw_try_cmpxchg_acquire arch_try_cmpxchg
 206 #else
 207 #define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
 208 ({ \
 209         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 210         ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
 211         if (unlikely(___r != ___o)) \
 212                 *___op = ___r; \
 213         likely(___r == ___o); \
 214 })
 215 #endif
 216
 217 #if defined(arch_try_cmpxchg_release)
 218 #define raw_try_cmpxchg_release arch_try_cmpxchg_release
 219 #elif defined(arch_try_cmpxchg_relaxed)
 220 #define raw_try_cmpxchg_release(...) \
 221         __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
 222 #elif defined(arch_try_cmpxchg)
 223 #define raw_try_cmpxchg_release arch_try_cmpxchg
 224 #else
 225 #define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
 226 ({ \
 227         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 228         ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
 229         if (unlikely(___r != ___o)) \
 230                 *___op = ___r; \
 231         likely(___r == ___o); \
 232 })
 233 #endif
 234
 235 #if defined(arch_try_cmpxchg_relaxed)
 236 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
 237 #elif defined(arch_try_cmpxchg)
 238 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg
 239 #else
 240 #define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
 241 ({ \
 242         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 243         ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
 244         if (unlikely(___r != ___o)) \
 245                 *___op = ___r; \
 246         likely(___r == ___o); \
 247 })
 248 #endif
 249
 250 #if defined(arch_try_cmpxchg64)
 251 #define raw_try_cmpxchg64 arch_try_cmpxchg64
 252 #elif defined(arch_try_cmpxchg64_relaxed)
 253 #define raw_try_cmpxchg64(...) \
 254         __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
 255 #else
 256 #define raw_try_cmpxchg64(_ptr, _oldp, _new) \
 257 ({ \
 258         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 259         ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
 260         if (unlikely(___r != ___o)) \
 261                 *___op = ___r; \
 262         likely(___r == ___o); \
 263 })
 264 #endif
 265
 266 #if defined(arch_try_cmpxchg64_acquire)
 267 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
 268 #elif defined(arch_try_cmpxchg64_relaxed)
 269 #define raw_try_cmpxchg64_acquire(...) \
 270         __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
 271 #elif defined(arch_try_cmpxchg64)
 272 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
 273 #else
 274 #define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
 275 ({ \
 276         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 277         ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
 278         if (unlikely(___r != ___o)) \
 279                 *___op = ___r; \
 280         likely(___r == ___o); \
 281 })
 282 #endif
 283
 284 #if defined(arch_try_cmpxchg64_release)
 285 #define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
 286 #elif defined(arch_try_cmpxchg64_relaxed)
 287 #define raw_try_cmpxchg64_release(...) \
 288         __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
 289 #elif defined(arch_try_cmpxchg64)
 290 #define raw_try_cmpxchg64_release arch_try_cmpxchg64
 291 #else
 292 #define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
 293 ({ \
 294         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 295         ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
 296         if (unlikely(___r != ___o)) \
 297                 *___op = ___r; \
 298         likely(___r == ___o); \
 299 })
 300 #endif
 301
 302 #if defined(arch_try_cmpxchg64_relaxed)
 303 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
 304 #elif defined(arch_try_cmpxchg64)
 305 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
 306 #else
 307 #define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
 308 ({ \
 309         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 310         ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
 311         if (unlikely(___r != ___o)) \
 312                 *___op = ___r; \
 313         likely(___r == ___o); \
 314 })
 315 #endif
 316
 317 #if defined(arch_try_cmpxchg128)
 318 #define raw_try_cmpxchg128 arch_try_cmpxchg128
 319 #elif defined(arch_try_cmpxchg128_relaxed)
 320 #define raw_try_cmpxchg128(...) \
 321         __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
 322 #else
 323 #define raw_try_cmpxchg128(_ptr, _oldp, _new) \
 324 ({ \
 325         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 326         ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
 327         if (unlikely(___r != ___o)) \
 328                 *___op = ___r; \
 329         likely(___r == ___o); \
 330 })
 331 #endif
 332
 333 #if defined(arch_try_cmpxchg128_acquire)
 334 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
 335 #elif defined(arch_try_cmpxchg128_relaxed)
 336 #define raw_try_cmpxchg128_acquire(...) \
 337         __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
 338 #elif defined(arch_try_cmpxchg128)
 339 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
 340 #else
 341 #define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
 342 ({ \
 343         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 344         ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
 345         if (unlikely(___r != ___o)) \
 346                 *___op = ___r; \
 347         likely(___r == ___o); \
 348 })
 349 #endif
 350
 351 #if defined(arch_try_cmpxchg128_release)
 352 #define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
 353 #elif defined(arch_try_cmpxchg128_relaxed)
 354 #define raw_try_cmpxchg128_release(...) \
 355         __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
 356 #elif defined(arch_try_cmpxchg128)
 357 #define raw_try_cmpxchg128_release arch_try_cmpxchg128
 358 #else
 359 #define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
 360 ({ \
 361         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 362         ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
 363         if (unlikely(___r != ___o)) \
 364                 *___op = ___r; \
 365         likely(___r == ___o); \
 366 })
 367 #endif
 368
 369 #if defined(arch_try_cmpxchg128_relaxed)
 370 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
 371 #elif defined(arch_try_cmpxchg128)
 372 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
 373 #else
 374 #define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
 375 ({ \
 376         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 377         ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
 378         if (unlikely(___r != ___o)) \
 379                 *___op = ___r; \
 380         likely(___r == ___o); \
 381 })
 382 #endif
 383
 384 #define raw_cmpxchg_local arch_cmpxchg_local
 385
 386 #ifdef arch_try_cmpxchg_local
 387 #define raw_try_cmpxchg_local arch_try_cmpxchg_local
 388 #else
 389 #define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
 390 ({ \
 391         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 392         ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
 393         if (unlikely(___r != ___o)) \
 394                 *___op = ___r; \
 395         likely(___r == ___o); \
 396 })
 397 #endif
 398
 399 #define raw_cmpxchg64_local arch_cmpxchg64_local
 400
 401 #ifdef arch_try_cmpxchg64_local
 402 #define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
 403 #else
 404 #define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
 405 ({ \
 406         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 407         ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
 408         if (unlikely(___r != ___o)) \
 409                 *___op = ___r; \
 410         likely(___r == ___o); \
 411 })
 412 #endif
 413
 414 #define raw_cmpxchg128_local arch_cmpxchg128_local
 415
 416 #ifdef arch_try_cmpxchg128_local
 417 #define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
 418 #else
 419 #define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
 420 ({ \
 421         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 422         ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
 423         if (unlikely(___r != ___o)) \
 424                 *___op = ___r; \
 425         likely(___r == ___o); \
 426 })
 427 #endif
 428
 429 #define raw_sync_cmpxchg arch_sync_cmpxchg
 430
 431 /**
 432  * raw_atomic_read() - atomic load with relaxed ordering
 433  * @v: pointer to atomic_t
 434  *
 435  * Atomically loads the value of @v with relaxed ordering.
 436  *
 437  * Safe to use in noinstr code; prefer atomic_read() elsewhere.
 438  *
 439  * Return: The value loaded from @v.
 440  */
 441 static __always_inline int
 442 raw_atomic_read(const atomic_t *v)
 443 {
 444         return arch_atomic_read(v);
 445 }
 446
 447 /**
 448  * raw_atomic_read_acquire() - atomic load with acquire ordering
 449  * @v: pointer to atomic_t
 450  *
 451  * Atomically loads the value of @v with acquire ordering.
 452  *
 453  * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
 454  *
 455  * Return: The value loaded from @v.
 456  */
 457 static __always_inline int
 458 raw_atomic_read_acquire(const atomic_t *v)
 459 {
 460 #if defined(arch_atomic_read_acquire)
 461         return arch_atomic_read_acquire(v);
 462 #else
 463         int ret;
 464
 465         if (__native_word(atomic_t)) {
 466                 ret = smp_load_acquire(&(v)->counter);
 467         } else {
 468                 ret = raw_atomic_read(v);
 469                 __atomic_acquire_fence();
 470         }
 471
 472         return ret;
 473 #endif
 474 }
 475
 476 /**
 477  * raw_atomic_set() - atomic set with relaxed ordering
 478  * @v: pointer to atomic_t
 479  * @i: int value to assign
 480  *
 481  * Atomically sets @v to @i with relaxed ordering.
 482  *
 483  * Safe to use in noinstr code; prefer atomic_set() elsewhere.
 484  *
 485  * Return: Nothing.
 486  */
 487 static __always_inline void
 488 raw_atomic_set(atomic_t *v, int i)
 489 {
 490         arch_atomic_set(v, i);
 491 }
 492
 493 /**
 494  * raw_atomic_set_release() - atomic set with release ordering
 495  * @v: pointer to atomic_t
 496  * @i: int value to assign
 497  *
 498  * Atomically sets @v to @i with release ordering.
 499  *
 500  * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
 501  *
 502  * Return: Nothing.
 503  */
 504 static __always_inline void
 505 raw_atomic_set_release(atomic_t *v, int i)
 506 {
 507 #if defined(arch_atomic_set_release)
 508         arch_atomic_set_release(v, i);
 509 #else
 510         if (__native_word(atomic_t)) {
 511                 smp_store_release(&(v)->counter, i);
 512         } else {
 513                 __atomic_release_fence();
 514                 raw_atomic_set(v, i);
 515         }
 516 #endif
 517 }
 518
 519 /**
 520  * raw_atomic_add() - atomic add with relaxed ordering
 521  * @i: int value to add
 522  * @v: pointer to atomic_t
 523  *
 524  * Atomically updates @v to (@v + @i) with relaxed ordering.
 525  *
 526  * Safe to use in noinstr code; prefer atomic_add() elsewhere.
 527  *
 528  * Return: Nothing.
 529  */
 530 static __always_inline void
 531 raw_atomic_add(int i, atomic_t *v)
 532 {
 533         arch_atomic_add(i, v);
 534 }
 535
 536 /**
 537  * raw_atomic_add_return() - atomic add with full ordering
 538  * @i: int value to add
 539  * @v: pointer to atomic_t
 540  *
 541  * Atomically updates @v to (@v + @i) with full ordering.
 542  *
 543  * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
 544  *
 545  * Return: The updated value of @v.
 546  */
 547 static __always_inline int
 548 raw_atomic_add_return(int i, atomic_t *v)
 549 {
 550 #if defined(arch_atomic_add_return)
 551         return arch_atomic_add_return(i, v);
 552 #elif defined(arch_atomic_add_return_relaxed)
 553         int ret;
 554         __atomic_pre_full_fence();
 555         ret = arch_atomic_add_return_relaxed(i, v);
 556         __atomic_post_full_fence();
 557         return ret;
 558 #else
 559 #error "Unable to define raw_atomic_add_return"
 560 #endif
 561 }
 562
 563 /**
 564  * raw_atomic_add_return_acquire() - atomic add with acquire ordering
 565  * @i: int value to add
 566  * @v: pointer to atomic_t
 567  *
 568  * Atomically updates @v to (@v + @i) with acquire ordering.
 569  *
 570  * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
 571  *
 572  * Return: The updated value of @v.
 573  */
 574 static __always_inline int
 575 raw_atomic_add_return_acquire(int i, atomic_t *v)
 576 {
 577 #if defined(arch_atomic_add_return_acquire)
 578         return arch_atomic_add_return_acquire(i, v);
 579 #elif defined(arch_atomic_add_return_relaxed)
 580         int ret = arch_atomic_add_return_relaxed(i, v);
 581         __atomic_acquire_fence();
 582         return ret;
 583 #elif defined(arch_atomic_add_return)
 584         return arch_atomic_add_return(i, v);
 585 #else
 586 #error "Unable to define raw_atomic_add_return_acquire"
 587 #endif
 588 }
 589
 590 /**
 591  * raw_atomic_add_return_release() - atomic add with release ordering
 592  * @i: int value to add
 593  * @v: pointer to atomic_t
 594  *
 595  * Atomically updates @v to (@v + @i) with release ordering.
 596  *
 597  * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
 598  *
 599  * Return: The updated value of @v.
 600  */
 601 static __always_inline int
 602 raw_atomic_add_return_release(int i, atomic_t *v)
 603 {
 604 #if defined(arch_atomic_add_return_release)
 605         return arch_atomic_add_return_release(i, v);
 606 #elif defined(arch_atomic_add_return_relaxed)
 607         __atomic_release_fence();
 608         return arch_atomic_add_return_relaxed(i, v);
 609 #elif defined(arch_atomic_add_return)
 610         return arch_atomic_add_return(i, v);
 611 #else
 612 #error "Unable to define raw_atomic_add_return_release"
 613 #endif
 614 }
 615
 616 /**
 617  * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
 618  * @i: int value to add
 619  * @v: pointer to atomic_t
 620  *
 621  * Atomically updates @v to (@v + @i) with relaxed ordering.
 622  *
 623  * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
 624  *
 625  * Return: The updated value of @v.
 626  */
 627 static __always_inline int
 628 raw_atomic_add_return_relaxed(int i, atomic_t *v)
 629 {
 630 #if defined(arch_atomic_add_return_relaxed)
 631         return arch_atomic_add_return_relaxed(i, v);
 632 #elif defined(arch_atomic_add_return)
 633         return arch_atomic_add_return(i, v);
 634 #else
 635 #error "Unable to define raw_atomic_add_return_relaxed"
 636 #endif
 637 }
 638
 639 /**
 640  * raw_atomic_fetch_add() - atomic add with full ordering
 641  * @i: int value to add
 642  * @v: pointer to atomic_t
 643  *
 644  * Atomically updates @v to (@v + @i) with full ordering.
 645  *
 646  * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
 647  *
 648  * Return: The original value of @v.
 649  */
 650 static __always_inline int
 651 raw_atomic_fetch_add(int i, atomic_t *v)
 652 {
 653 #if defined(arch_atomic_fetch_add)
 654         return arch_atomic_fetch_add(i, v);
 655 #elif defined(arch_atomic_fetch_add_relaxed)
 656         int ret;
 657         __atomic_pre_full_fence();
 658         ret = arch_atomic_fetch_add_relaxed(i, v);
 659         __atomic_post_full_fence();
 660         return ret;
 661 #else
 662 #error "Unable to define raw_atomic_fetch_add"
 663 #endif
 664 }
 665
 666 /**
 667  * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
 668  * @i: int value to add
 669  * @v: pointer to atomic_t
 670  *
 671  * Atomically updates @v to (@v + @i) with acquire ordering.
 672  *
 673  * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
 674  *
 675  * Return: The original value of @v.
 676  */
 677 static __always_inline int
 678 raw_atomic_fetch_add_acquire(int i, atomic_t *v)
 679 {
 680 #if defined(arch_atomic_fetch_add_acquire)
 681         return arch_atomic_fetch_add_acquire(i, v);
 682 #elif defined(arch_atomic_fetch_add_relaxed)
 683         int ret = arch_atomic_fetch_add_relaxed(i, v);
 684         __atomic_acquire_fence();
 685         return ret;
 686 #elif defined(arch_atomic_fetch_add)
 687         return arch_atomic_fetch_add(i, v);
 688 #else
 689 #error "Unable to define raw_atomic_fetch_add_acquire"
 690 #endif
 691 }
 692
 693 /**
 694  * raw_atomic_fetch_add_release() - atomic add with release ordering
 695  * @i: int value to add
 696  * @v: pointer to atomic_t
 697  *
 698  * Atomically updates @v to (@v + @i) with release ordering.
 699  *
 700  * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
 701  *
 702  * Return: The original value of @v.
 703  */
 704 static __always_inline int
 705 raw_atomic_fetch_add_release(int i, atomic_t *v)
 706 {
 707 #if defined(arch_atomic_fetch_add_release)
 708         return arch_atomic_fetch_add_release(i, v);
 709 #elif defined(arch_atomic_fetch_add_relaxed)
 710         __atomic_release_fence();
 711         return arch_atomic_fetch_add_relaxed(i, v);
 712 #elif defined(arch_atomic_fetch_add)
 713         return arch_atomic_fetch_add(i, v);
 714 #else
 715 #error "Unable to define raw_atomic_fetch_add_release"
 716 #endif
 717 }
 718
 719 /**
 720  * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
 721  * @i: int value to add
 722  * @v: pointer to atomic_t
 723  *
 724  * Atomically updates @v to (@v + @i) with relaxed ordering.
 725  *
 726  * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
 727  *
 728  * Return: The original value of @v.
 729  */
 730 static __always_inline int
 731 raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
 732 {
 733 #if defined(arch_atomic_fetch_add_relaxed)
 734         return arch_atomic_fetch_add_relaxed(i, v);
 735 #elif defined(arch_atomic_fetch_add)
 736         return arch_atomic_fetch_add(i, v);
 737 #else
 738 #error "Unable to define raw_atomic_fetch_add_relaxed"
 739 #endif
 740 }
 741
 742 /**
 743  * raw_atomic_sub() - atomic subtract with relaxed ordering
 744  * @i: int value to subtract
 745  * @v: pointer to atomic_t
 746  *
 747  * Atomically updates @v to (@v - @i) with relaxed ordering.
 748  *
 749  * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
 750  *
 751  * Return: Nothing.
 752  */
 753 static __always_inline void
 754 raw_atomic_sub(int i, atomic_t *v)
 755 {
 756         arch_atomic_sub(i, v);
 757 }
 758
 759 /**
 760  * raw_atomic_sub_return() - atomic subtract with full ordering
 761  * @i: int value to subtract
 762  * @v: pointer to atomic_t
 763  *
 764  * Atomically updates @v to (@v - @i) with full ordering.
 765  *
 766  * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
 767  *
 768  * Return: The updated value of @v.
 769  */
 770 static __always_inline int
 771 raw_atomic_sub_return(int i, atomic_t *v)
 772 {
 773 #if defined(arch_atomic_sub_return)
 774         return arch_atomic_sub_return(i, v);
 775 #elif defined(arch_atomic_sub_return_relaxed)
 776         int ret;
 777         __atomic_pre_full_fence();
 778         ret = arch_atomic_sub_return_relaxed(i, v);
 779         __atomic_post_full_fence();
 780         return ret;
 781 #else
 782 #error "Unable to define raw_atomic_sub_return"
 783 #endif
 784 }
 785
 786 /**
 787  * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
 788  * @i: int value to subtract
 789  * @v: pointer to atomic_t
 790  *
 791  * Atomically updates @v to (@v - @i) with acquire ordering.
 792  *
 793  * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
 794  *
 795  * Return: The updated value of @v.
 796  */
 797 static __always_inline int
 798 raw_atomic_sub_return_acquire(int i, atomic_t *v)
 799 {
 800 #if defined(arch_atomic_sub_return_acquire)
 801         return arch_atomic_sub_return_acquire(i, v);
 802 #elif defined(arch_atomic_sub_return_relaxed)
 803         int ret = arch_atomic_sub_return_relaxed(i, v);
 804         __atomic_acquire_fence();
 805         return ret;
 806 #elif defined(arch_atomic_sub_return)
 807         return arch_atomic_sub_return(i, v);
 808 #else
 809 #error "Unable to define raw_atomic_sub_return_acquire"
 810 #endif
 811 }
 812
 813 /**
 814  * raw_atomic_sub_return_release() - atomic subtract with release ordering
 815  * @i: int value to subtract
 816  * @v: pointer to atomic_t
 817  *
 818  * Atomically updates @v to (@v - @i) with release ordering.
 819  *
 820  * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
 821  *
 822  * Return: The updated value of @v.
 823  */
 824 static __always_inline int
 825 raw_atomic_sub_return_release(int i, atomic_t *v)
 826 {
 827 #if defined(arch_atomic_sub_return_release)
 828         return arch_atomic_sub_return_release(i, v);
 829 #elif defined(arch_atomic_sub_return_relaxed)
 830         __atomic_release_fence();
 831         return arch_atomic_sub_return_relaxed(i, v);
 832 #elif defined(arch_atomic_sub_return)
 833         return arch_atomic_sub_return(i, v);
 834 #else
 835 #error "Unable to define raw_atomic_sub_return_release"
 836 #endif
 837 }
 838
 839 /**
 840  * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
 841  * @i: int value to subtract
 842  * @v: pointer to atomic_t
 843  *
 844  * Atomically updates @v to (@v - @i) with relaxed ordering.
 845  *
 846  * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
 847  *
 848  * Return: The updated value of @v.
 849  */
 850 static __always_inline int
 851 raw_atomic_sub_return_relaxed(int i, atomic_t *v)
 852 {
 853 #if defined(arch_atomic_sub_return_relaxed)
 854         return arch_atomic_sub_return_relaxed(i, v);
 855 #elif defined(arch_atomic_sub_return)
 856         return arch_atomic_sub_return(i, v);
 857 #else
 858 #error "Unable to define raw_atomic_sub_return_relaxed"
 859 #endif
 860 }
 861
 862 /**
 863  * raw_atomic_fetch_sub() - atomic subtract with full ordering
 864  * @i: int value to subtract
 865  * @v: pointer to atomic_t
 866  *
 867  * Atomically updates @v to (@v - @i) with full ordering.
 868  *
 869  * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
 870  *
 871  * Return: The original value of @v.
 872  */
 873 static __always_inline int
 874 raw_atomic_fetch_sub(int i, atomic_t *v)
 875 {
 876 #if defined(arch_atomic_fetch_sub)
 877         return arch_atomic_fetch_sub(i, v);
 878 #elif defined(arch_atomic_fetch_sub_relaxed)
 879         int ret;
 880         __atomic_pre_full_fence();
 881         ret = arch_atomic_fetch_sub_relaxed(i, v);
 882         __atomic_post_full_fence();
 883         return ret;
 884 #else
 885 #error "Unable to define raw_atomic_fetch_sub"
 886 #endif
 887 }
 888
 889 /**
 890  * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
 891  * @i: int value to subtract
 892  * @v: pointer to atomic_t
 893  *
 894  * Atomically updates @v to (@v - @i) with acquire ordering.
 895  *
 896  * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
 897  *
 898  * Return: The original value of @v.
 899  */
 900 static __always_inline int
 901 raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
 902 {
 903 #if defined(arch_atomic_fetch_sub_acquire)
 904         return arch_atomic_fetch_sub_acquire(i, v);
 905 #elif defined(arch_atomic_fetch_sub_relaxed)
 906         int ret = arch_atomic_fetch_sub_relaxed(i, v);
 907         __atomic_acquire_fence();
 908         return ret;
 909 #elif defined(arch_atomic_fetch_sub)
 910         return arch_atomic_fetch_sub(i, v);
 911 #else
 912 #error "Unable to define raw_atomic_fetch_sub_acquire"
 913 #endif
 914 }
 915
 916 /**
 917  * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
 918  * @i: int value to subtract
 919  * @v: pointer to atomic_t
 920  *
 921  * Atomically updates @v to (@v - @i) with release ordering.
 922  *
 923  * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
 924  *
 925  * Return: The original value of @v.
 926  */
 927 static __always_inline int
 928 raw_atomic_fetch_sub_release(int i, atomic_t *v)
 929 {
 930 #if defined(arch_atomic_fetch_sub_release)
 931         return arch_atomic_fetch_sub_release(i, v);
 932 #elif defined(arch_atomic_fetch_sub_relaxed)
 933         __atomic_release_fence();
 934         return arch_atomic_fetch_sub_relaxed(i, v);
 935 #elif defined(arch_atomic_fetch_sub)
 936         return arch_atomic_fetch_sub(i, v);
 937 #else
 938 #error "Unable to define raw_atomic_fetch_sub_release"
 939 #endif
 940 }
 941
 942 /**
 943  * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 944  * @i: int value to subtract
 945  * @v: pointer to atomic_t
 946  *
 947  * Atomically updates @v to (@v - @i) with relaxed ordering.
 948  *
 949  * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
 950  *
 951  * Return: The original value of @v.
 952  */
 953 static __always_inline int
 954 raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
 955 {
 956 #if defined(arch_atomic_fetch_sub_relaxed)
 957         return arch_atomic_fetch_sub_relaxed(i, v);
 958 #elif defined(arch_atomic_fetch_sub)
 959         return arch_atomic_fetch_sub(i, v);
 960 #else
 961 #error "Unable to define raw_atomic_fetch_sub_relaxed"
 962 #endif
 963 }
 964
 965 /**
 966  * raw_atomic_inc() - atomic increment with relaxed ordering
 967  * @v: pointer to atomic_t
 968  *
 969  * Atomically updates @v to (@v + 1) with relaxed ordering.
 970  *
 971  * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
 972  *
 973  * Return: Nothing.
 974  */
 975 static __always_inline void
 976 raw_atomic_inc(atomic_t *v)
 977 {
 978 #if defined(arch_atomic_inc)
 979         arch_atomic_inc(v);
 980 #else
 981         raw_atomic_add(1, v);
 982 #endif
 983 }
 984
 985 /**
 986  * raw_atomic_inc_return() - atomic increment with full ordering
 987  * @v: pointer to atomic_t
 988  *
 989  * Atomically updates @v to (@v + 1) with full ordering.
 990  *
 991  * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
 992  *
 993  * Return: The updated value of @v.
 994  */
 995 static __always_inline int
 996 raw_atomic_inc_return(atomic_t *v)
 997 {
 998 #if defined(arch_atomic_inc_return)
 999         return arch_atomic_inc_return(v);
1000 #elif defined(arch_atomic_inc_return_relaxed)
1001         int ret;
1002         __atomic_pre_full_fence();
1003         ret = arch_atomic_inc_return_relaxed(v);
1004         __atomic_post_full_fence();
1005         return ret;
1006 #else
1007         return raw_atomic_add_return(1, v);
1008 #endif
1009 }
1010
1011 /**
1012  * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
1013  * @v: pointer to atomic_t
1014  *
1015  * Atomically updates @v to (@v + 1) with acquire ordering.
1016  *
1017  * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1018  *
1019  * Return: The updated value of @v.
1020  */
1021 static __always_inline int
1022 raw_atomic_inc_return_acquire(atomic_t *v)
1023 {
1024 #if defined(arch_atomic_inc_return_acquire)
1025         return arch_atomic_inc_return_acquire(v);
1026 #elif defined(arch_atomic_inc_return_relaxed)
1027         int ret = arch_atomic_inc_return_relaxed(v);
1028         __atomic_acquire_fence();
1029         return ret;
1030 #elif defined(arch_atomic_inc_return)
1031         return arch_atomic_inc_return(v);
1032 #else
1033         return raw_atomic_add_return_acquire(1, v);
1034 #endif
1035 }
1036
1037 /**
1038  * raw_atomic_inc_return_release() - atomic increment with release ordering
1039  * @v: pointer to atomic_t
1040  *
1041  * Atomically updates @v to (@v + 1) with release ordering.
1042  *
1043  * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1044  *
1045  * Return: The updated value of @v.
1046  */
1047 static __always_inline int
1048 raw_atomic_inc_return_release(atomic_t *v)
1049 {
1050 #if defined(arch_atomic_inc_return_release)
1051         return arch_atomic_inc_return_release(v);
1052 #elif defined(arch_atomic_inc_return_relaxed)
1053         __atomic_release_fence();
1054         return arch_atomic_inc_return_relaxed(v);
1055 #elif defined(arch_atomic_inc_return)
1056         return arch_atomic_inc_return(v);
1057 #else
1058         return raw_atomic_add_return_release(1, v);
1059 #endif
1060 }
1061
1062 /**
1063  * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
1064  * @v: pointer to atomic_t
1065  *
1066  * Atomically updates @v to (@v + 1) with relaxed ordering.
1067  *
1068  * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
1069  *
1070  * Return: The updated value of @v.
1071  */
1072 static __always_inline int
1073 raw_atomic_inc_return_relaxed(atomic_t *v)
1074 {
1075 #if defined(arch_atomic_inc_return_relaxed)
1076         return arch_atomic_inc_return_relaxed(v);
1077 #elif defined(arch_atomic_inc_return)
1078         return arch_atomic_inc_return(v);
1079 #else
1080         return raw_atomic_add_return_relaxed(1, v);
1081 #endif
1082 }
1083
1084 /**
1085  * raw_atomic_fetch_inc() - atomic increment with full ordering
1086  * @v: pointer to atomic_t
1087  *
1088  * Atomically updates @v to (@v + 1) with full ordering.
1089  *
1090  * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1091  *
1092  * Return: The original value of @v.
1093  */
1094 static __always_inline int
1095 raw_atomic_fetch_inc(atomic_t *v)
1096 {
1097 #if defined(arch_atomic_fetch_inc)
1098         return arch_atomic_fetch_inc(v);
1099 #elif defined(arch_atomic_fetch_inc_relaxed)
1100         int ret;
1101         __atomic_pre_full_fence();
1102         ret = arch_atomic_fetch_inc_relaxed(v);
1103         __atomic_post_full_fence();
1104         return ret;
1105 #else
1106         return raw_atomic_fetch_add(1, v);
1107 #endif
1108 }
1109
1110 /**
1111  * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
1112  * @v: pointer to atomic_t
1113  *
1114  * Atomically updates @v to (@v + 1) with acquire ordering.
1115  *
1116  * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1117  *
1118  * Return: The original value of @v.
1119  */
1120 static __always_inline int
1121 raw_atomic_fetch_inc_acquire(atomic_t *v)
1122 {
1123 #if defined(arch_atomic_fetch_inc_acquire)
1124         return arch_atomic_fetch_inc_acquire(v);
1125 #elif defined(arch_atomic_fetch_inc_relaxed)
1126         int ret = arch_atomic_fetch_inc_relaxed(v);
1127         __atomic_acquire_fence();
1128         return ret;
1129 #elif defined(arch_atomic_fetch_inc)
1130         return arch_atomic_fetch_inc(v);
1131 #else
1132         return raw_atomic_fetch_add_acquire(1, v);
1133 #endif
1134 }
1135
1136 /**
1137  * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1138  * @v: pointer to atomic_t
1139  *
1140  * Atomically updates @v to (@v + 1) with release ordering.
1141  *
1142  * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1143  *
1144  * Return: The original value of @v.
1145  */
1146 static __always_inline int
1147 raw_atomic_fetch_inc_release(atomic_t *v)
1148 {
1149 #if defined(arch_atomic_fetch_inc_release)
1150         return arch_atomic_fetch_inc_release(v);
1151 #elif defined(arch_atomic_fetch_inc_relaxed)
1152         __atomic_release_fence();
1153         return arch_atomic_fetch_inc_relaxed(v);
1154 #elif defined(arch_atomic_fetch_inc)
1155         return arch_atomic_fetch_inc(v);
1156 #else
1157         return raw_atomic_fetch_add_release(1, v);
1158 #endif
1159 }
1160
1161 /**
1162  * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1163  * @v: pointer to atomic_t
1164  *
1165  * Atomically updates @v to (@v + 1) with relaxed ordering.
1166  *
1167  * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1168  *
1169  * Return: The original value of @v.
1170  */
1171 static __always_inline int
1172 raw_atomic_fetch_inc_relaxed(atomic_t *v)
1173 {
1174 #if defined(arch_atomic_fetch_inc_relaxed)
1175         return arch_atomic_fetch_inc_relaxed(v);
1176 #elif defined(arch_atomic_fetch_inc)
1177         return arch_atomic_fetch_inc(v);
1178 #else
1179         return raw_atomic_fetch_add_relaxed(1, v);
1180 #endif
1181 }
1182
1183 /**
1184  * raw_atomic_dec() - atomic decrement with relaxed ordering
1185  * @v: pointer to atomic_t
1186  *
1187  * Atomically updates @v to (@v - 1) with relaxed ordering.
1188  *
1189  * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1190  *
1191  * Return: Nothing.
1192  */
1193 static __always_inline void
1194 raw_atomic_dec(atomic_t *v)
1195 {
1196 #if defined(arch_atomic_dec)
1197         arch_atomic_dec(v);
1198 #else
1199         raw_atomic_sub(1, v);
1200 #endif
1201 }
1202
1203 /**
1204  * raw_atomic_dec_return() - atomic decrement with full ordering
1205  * @v: pointer to atomic_t
1206  *
1207  * Atomically updates @v to (@v - 1) with full ordering.
1208  *
1209  * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1210  *
1211  * Return: The updated value of @v.
1212  */
1213 static __always_inline int
1214 raw_atomic_dec_return(atomic_t *v)
1215 {
1216 #if defined(arch_atomic_dec_return)
1217         return arch_atomic_dec_return(v);
1218 #elif defined(arch_atomic_dec_return_relaxed)
1219         int ret;
1220         __atomic_pre_full_fence();
1221         ret = arch_atomic_dec_return_relaxed(v);
1222         __atomic_post_full_fence();
1223         return ret;
1224 #else
1225         return raw_atomic_sub_return(1, v);
1226 #endif
1227 }
1228
1229 /**
1230  * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
1231  * @v: pointer to atomic_t
1232  *
1233  * Atomically updates @v to (@v - 1) with acquire ordering.
1234  *
1235  * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1236  *
1237  * Return: The updated value of @v.
1238  */
1239 static __always_inline int
1240 raw_atomic_dec_return_acquire(atomic_t *v)
1241 {
1242 #if defined(arch_atomic_dec_return_acquire)
1243         return arch_atomic_dec_return_acquire(v);
1244 #elif defined(arch_atomic_dec_return_relaxed)
1245         int ret = arch_atomic_dec_return_relaxed(v);
1246         __atomic_acquire_fence();
1247         return ret;
1248 #elif defined(arch_atomic_dec_return)
1249         return arch_atomic_dec_return(v);
1250 #else
1251         return raw_atomic_sub_return_acquire(1, v);
1252 #endif
1253 }
1254
1255 /**
1256  * raw_atomic_dec_return_release() - atomic decrement with release ordering
1257  * @v: pointer to atomic_t
1258  *
1259  * Atomically updates @v to (@v - 1) with release ordering.
1260  *
1261  * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1262  *
1263  * Return: The updated value of @v.
1264  */
1265 static __always_inline int
1266 raw_atomic_dec_return_release(atomic_t *v)
1267 {
1268 #if defined(arch_atomic_dec_return_release)
1269         return arch_atomic_dec_return_release(v);
1270 #elif defined(arch_atomic_dec_return_relaxed)
1271         __atomic_release_fence();
1272         return arch_atomic_dec_return_relaxed(v);
1273 #elif defined(arch_atomic_dec_return)
1274         return arch_atomic_dec_return(v);
1275 #else
1276         return raw_atomic_sub_return_release(1, v);
1277 #endif
1278 }
1279
1280 /**
1281  * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1282  * @v: pointer to atomic_t
1283  *
1284  * Atomically updates @v to (@v - 1) with relaxed ordering.
1285  *
1286  * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1287  *
1288  * Return: The updated value of @v.
1289  */
1290 static __always_inline int
1291 raw_atomic_dec_return_relaxed(atomic_t *v)
1292 {
1293 #if defined(arch_atomic_dec_return_relaxed)
1294         return arch_atomic_dec_return_relaxed(v);
1295 #elif defined(arch_atomic_dec_return)
1296         return arch_atomic_dec_return(v);
1297 #else
1298         return raw_atomic_sub_return_relaxed(1, v);
1299 #endif
1300 }
1301
1302 /**
1303  * raw_atomic_fetch_dec() - atomic decrement with full ordering
1304  * @v: pointer to atomic_t
1305  *
1306  * Atomically updates @v to (@v - 1) with full ordering.
1307  *
1308  * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1309  *
1310  * Return: The original value of @v.
1311  */
1312 static __always_inline int
1313 raw_atomic_fetch_dec(atomic_t *v)
1314 {
1315 #if defined(arch_atomic_fetch_dec)
1316         return arch_atomic_fetch_dec(v);
1317 #elif defined(arch_atomic_fetch_dec_relaxed)
1318         int ret;
1319         __atomic_pre_full_fence();
1320         ret = arch_atomic_fetch_dec_relaxed(v);
1321         __atomic_post_full_fence();
1322         return ret;
1323 #else
1324         return raw_atomic_fetch_sub(1, v);
1325 #endif
1326 }
1327
1328 /**
1329  * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1330  * @v: pointer to atomic_t
1331  *
1332  * Atomically updates @v to (@v - 1) with acquire ordering.
1333  *
1334  * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1335  *
1336  * Return: The original value of @v.
1337  */
1338 static __always_inline int
1339 raw_atomic_fetch_dec_acquire(atomic_t *v)
1340 {
1341 #if defined(arch_atomic_fetch_dec_acquire)
1342         return arch_atomic_fetch_dec_acquire(v);
1343 #elif defined(arch_atomic_fetch_dec_relaxed)
1344         int ret = arch_atomic_fetch_dec_relaxed(v);
1345         __atomic_acquire_fence();
1346         return ret;
1347 #elif defined(arch_atomic_fetch_dec)
1348         return arch_atomic_fetch_dec(v);
1349 #else
1350         return raw_atomic_fetch_sub_acquire(1, v);
1351 #endif
1352 }
1353
1354 /**
1355  * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1356  * @v: pointer to atomic_t
1357  *
1358  * Atomically updates @v to (@v - 1) with release ordering.
1359  *
1360  * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1361  *
1362  * Return: The original value of @v.
1363  */
1364 static __always_inline int
1365 raw_atomic_fetch_dec_release(atomic_t *v)
1366 {
1367 #if defined(arch_atomic_fetch_dec_release)
1368         return arch_atomic_fetch_dec_release(v);
1369 #elif defined(arch_atomic_fetch_dec_relaxed)
1370         __atomic_release_fence();
1371         return arch_atomic_fetch_dec_relaxed(v);
1372 #elif defined(arch_atomic_fetch_dec)
1373         return arch_atomic_fetch_dec(v);
1374 #else
1375         return raw_atomic_fetch_sub_release(1, v);
1376 #endif
1377 }
1378
1379 /**
1380  * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1381  * @v: pointer to atomic_t
1382  *
1383  * Atomically updates @v to (@v - 1) with relaxed ordering.
1384  *
1385  * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1386  *
1387  * Return: The original value of @v.
1388  */
1389 static __always_inline int
1390 raw_atomic_fetch_dec_relaxed(atomic_t *v)
1391 {
1392 #if defined(arch_atomic_fetch_dec_relaxed)
1393         return arch_atomic_fetch_dec_relaxed(v);
1394 #elif defined(arch_atomic_fetch_dec)
1395         return arch_atomic_fetch_dec(v);
1396 #else
1397         return raw_atomic_fetch_sub_relaxed(1, v);
1398 #endif
1399 }
1400
1401 /**
1402  * raw_atomic_and() - atomic bitwise AND with relaxed ordering
1403  * @i: int value
1404  * @v: pointer to atomic_t
1405  *
1406  * Atomically updates @v to (@v & @i) with relaxed ordering.
1407  *
1408  * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1409  *
1410  * Return: Nothing.
1411  */
1412 static __always_inline void
1413 raw_atomic_and(int i, atomic_t *v)
1414 {
1415         arch_atomic_and(i, v);
1416 }
1417
1418 /**
1419  * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1420  * @i: int value
1421  * @v: pointer to atomic_t
1422  *
1423  * Atomically updates @v to (@v & @i) with full ordering.
1424  *
1425  * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1426  *
1427  * Return: The original value of @v.
1428  */
1429 static __always_inline int
1430 raw_atomic_fetch_and(int i, atomic_t *v)
1431 {
1432 #if defined(arch_atomic_fetch_and)
1433         return arch_atomic_fetch_and(i, v);
1434 #elif defined(arch_atomic_fetch_and_relaxed)
1435         int ret;
1436         __atomic_pre_full_fence();
1437         ret = arch_atomic_fetch_and_relaxed(i, v);
1438         __atomic_post_full_fence();
1439         return ret;
1440 #else
1441 #error "Unable to define raw_atomic_fetch_and"
1442 #endif
1443 }
1444
1445 /**
1446  * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
1447  * @i: int value
1448  * @v: pointer to atomic_t
1449  *
1450  * Atomically updates @v to (@v & @i) with acquire ordering.
1451  *
1452  * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
1453  *
1454  * Return: The original value of @v.
1455  */
1456 static __always_inline int
1457 raw_atomic_fetch_and_acquire(int i, atomic_t *v)
1458 {
1459 #if defined(arch_atomic_fetch_and_acquire)
1460         return arch_atomic_fetch_and_acquire(i, v);
1461 #elif defined(arch_atomic_fetch_and_relaxed)
1462         int ret = arch_atomic_fetch_and_relaxed(i, v);
1463         __atomic_acquire_fence();
1464         return ret;
1465 #elif defined(arch_atomic_fetch_and)
1466         return arch_atomic_fetch_and(i, v);
1467 #else
1468 #error "Unable to define raw_atomic_fetch_and_acquire"
1469 #endif
1470 }
1471
1472 /**
1473  * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1474  * @i: int value
1475  * @v: pointer to atomic_t
1476  *
1477  * Atomically updates @v to (@v & @i) with release ordering.
1478  *
1479  * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1480  *
1481  * Return: The original value of @v.
1482  */
1483 static __always_inline int
1484 raw_atomic_fetch_and_release(int i, atomic_t *v)
1485 {
1486 #if defined(arch_atomic_fetch_and_release)
1487         return arch_atomic_fetch_and_release(i, v);
1488 #elif defined(arch_atomic_fetch_and_relaxed)
1489         __atomic_release_fence();
1490         return arch_atomic_fetch_and_relaxed(i, v);
1491 #elif defined(arch_atomic_fetch_and)
1492         return arch_atomic_fetch_and(i, v);
1493 #else
1494 #error "Unable to define raw_atomic_fetch_and_release"
1495 #endif
1496 }
1497
1498 /**
1499  * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1500  * @i: int value
1501  * @v: pointer to atomic_t
1502  *
1503  * Atomically updates @v to (@v & @i) with relaxed ordering.
1504  *
1505  * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1506  *
1507  * Return: The original value of @v.
1508  */
1509 static __always_inline int
1510 raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
1511 {
1512 #if defined(arch_atomic_fetch_and_relaxed)
1513         return arch_atomic_fetch_and_relaxed(i, v);
1514 #elif defined(arch_atomic_fetch_and)
1515         return arch_atomic_fetch_and(i, v);
1516 #else
1517 #error "Unable to define raw_atomic_fetch_and_relaxed"
1518 #endif
1519 }
1520
1521 /**
1522  * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
1523  * @i: int value
1524  * @v: pointer to atomic_t
1525  *
1526  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1527  *
1528  * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1529  *
1530  * Return: Nothing.
1531  */
1532 static __always_inline void
1533 raw_atomic_andnot(int i, atomic_t *v)
1534 {
1535 #if defined(arch_atomic_andnot)
1536         arch_atomic_andnot(i, v);
1537 #else
1538         raw_atomic_and(~i, v);
1539 #endif
1540 }
1541
1542 /**
1543  * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
1544  * @i: int value
1545  * @v: pointer to atomic_t
1546  *
1547  * Atomically updates @v to (@v & ~@i) with full ordering.
1548  *
1549  * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
1550  *
1551  * Return: The original value of @v.
1552  */
1553 static __always_inline int
1554 raw_atomic_fetch_andnot(int i, atomic_t *v)
1555 {
1556 #if defined(arch_atomic_fetch_andnot)
1557         return arch_atomic_fetch_andnot(i, v);
1558 #elif defined(arch_atomic_fetch_andnot_relaxed)
1559         int ret;
1560         __atomic_pre_full_fence();
1561         ret = arch_atomic_fetch_andnot_relaxed(i, v);
1562         __atomic_post_full_fence();
1563         return ret;
1564 #else
1565         return raw_atomic_fetch_and(~i, v);
1566 #endif
1567 }
1568
1569 /**
1570  * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1571  * @i: int value
1572  * @v: pointer to atomic_t
1573  *
1574  * Atomically updates @v to (@v & ~@i) with acquire ordering.
1575  *
1576  * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1577  *
1578  * Return: The original value of @v.
1579  */
1580 static __always_inline int
1581 raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
1582 {
1583 #if defined(arch_atomic_fetch_andnot_acquire)
1584         return arch_atomic_fetch_andnot_acquire(i, v);
1585 #elif defined(arch_atomic_fetch_andnot_relaxed)
1586         int ret = arch_atomic_fetch_andnot_relaxed(i, v);
1587         __atomic_acquire_fence();
1588         return ret;
1589 #elif defined(arch_atomic_fetch_andnot)
1590         return arch_atomic_fetch_andnot(i, v);
1591 #else
1592         return raw_atomic_fetch_and_acquire(~i, v);
1593 #endif
1594 }
1595
1596 /**
1597  * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1598  * @i: int value
1599  * @v: pointer to atomic_t
1600  *
1601  * Atomically updates @v to (@v & ~@i) with release ordering.
1602  *
1603  * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1604  *
1605  * Return: The original value of @v.
1606  */
1607 static __always_inline int
1608 raw_atomic_fetch_andnot_release(int i, atomic_t *v)
1609 {
1610 #if defined(arch_atomic_fetch_andnot_release)
1611         return arch_atomic_fetch_andnot_release(i, v);
1612 #elif defined(arch_atomic_fetch_andnot_relaxed)
1613         __atomic_release_fence();
1614         return arch_atomic_fetch_andnot_relaxed(i, v);
1615 #elif defined(arch_atomic_fetch_andnot)
1616         return arch_atomic_fetch_andnot(i, v);
1617 #else
1618         return raw_atomic_fetch_and_release(~i, v);
1619 #endif
1620 }
1621
1622 /**
1623  * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1624  * @i: int value
1625  * @v: pointer to atomic_t
1626  *
1627  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1628  *
1629  * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1630  *
1631  * Return: The original value of @v.
1632  */
1633 static __always_inline int
1634 raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1635 {
1636 #if defined(arch_atomic_fetch_andnot_relaxed)
1637         return arch_atomic_fetch_andnot_relaxed(i, v);
1638 #elif defined(arch_atomic_fetch_andnot)
1639         return arch_atomic_fetch_andnot(i, v);
1640 #else
1641         return raw_atomic_fetch_and_relaxed(~i, v);
1642 #endif
1643 }
1644
1645 /**
1646  * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1647  * @i: int value
1648  * @v: pointer to atomic_t
1649  *
1650  * Atomically updates @v to (@v | @i) with relaxed ordering.
1651  *
1652  * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1653  *
1654  * Return: Nothing.
1655  */
1656 static __always_inline void
1657 raw_atomic_or(int i, atomic_t *v)
1658 {
1659         arch_atomic_or(i, v);
1660 }
1661
1662 /**
1663  * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1664  * @i: int value
1665  * @v: pointer to atomic_t
1666  *
1667  * Atomically updates @v to (@v | @i) with full ordering.
1668  *
1669  * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1670  *
1671  * Return: The original value of @v.
1672  */
1673 static __always_inline int
1674 raw_atomic_fetch_or(int i, atomic_t *v)
1675 {
1676 #if defined(arch_atomic_fetch_or)
1677         return arch_atomic_fetch_or(i, v);
1678 #elif defined(arch_atomic_fetch_or_relaxed)
1679         int ret;
1680         __atomic_pre_full_fence();
1681         ret = arch_atomic_fetch_or_relaxed(i, v);
1682         __atomic_post_full_fence();
1683         return ret;
1684 #else
1685 #error "Unable to define raw_atomic_fetch_or"
1686 #endif
1687 }
1688
1689 /**
1690  * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1691  * @i: int value
1692  * @v: pointer to atomic_t
1693  *
1694  * Atomically updates @v to (@v | @i) with acquire ordering.
1695  *
1696  * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
1697  *
1698  * Return: The original value of @v.
1699  */
1700 static __always_inline int
1701 raw_atomic_fetch_or_acquire(int i, atomic_t *v)
1702 {
1703 #if defined(arch_atomic_fetch_or_acquire)
1704         return arch_atomic_fetch_or_acquire(i, v);
1705 #elif defined(arch_atomic_fetch_or_relaxed)
1706         int ret = arch_atomic_fetch_or_relaxed(i, v);
1707         __atomic_acquire_fence();
1708         return ret;
1709 #elif defined(arch_atomic_fetch_or)
1710         return arch_atomic_fetch_or(i, v);
1711 #else
1712 #error "Unable to define raw_atomic_fetch_or_acquire"
1713 #endif
1714 }
1715
1716 /**
1717  * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1718  * @i: int value
1719  * @v: pointer to atomic_t
1720  *
1721  * Atomically updates @v to (@v | @i) with release ordering.
1722  *
1723  * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1724  *
1725  * Return: The original value of @v.
1726  */
1727 static __always_inline int
1728 raw_atomic_fetch_or_release(int i, atomic_t *v)
1729 {
1730 #if defined(arch_atomic_fetch_or_release)
1731         return arch_atomic_fetch_or_release(i, v);
1732 #elif defined(arch_atomic_fetch_or_relaxed)
1733         __atomic_release_fence();
1734         return arch_atomic_fetch_or_relaxed(i, v);
1735 #elif defined(arch_atomic_fetch_or)
1736         return arch_atomic_fetch_or(i, v);
1737 #else
1738 #error "Unable to define raw_atomic_fetch_or_release"
1739 #endif
1740 }
1741
1742 /**
1743  * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1744  * @i: int value
1745  * @v: pointer to atomic_t
1746  *
1747  * Atomically updates @v to (@v | @i) with relaxed ordering.
1748  *
1749  * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1750  *
1751  * Return: The original value of @v.
1752  */
1753 static __always_inline int
1754 raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1755 {
1756 #if defined(arch_atomic_fetch_or_relaxed)
1757         return arch_atomic_fetch_or_relaxed(i, v);
1758 #elif defined(arch_atomic_fetch_or)
1759         return arch_atomic_fetch_or(i, v);
1760 #else
1761 #error "Unable to define raw_atomic_fetch_or_relaxed"
1762 #endif
1763 }
1764
1765 /**
1766  * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1767  * @i: int value
1768  * @v: pointer to atomic_t
1769  *
1770  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1771  *
1772  * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1773  *
1774  * Return: Nothing.
1775  */
1776 static __always_inline void
1777 raw_atomic_xor(int i, atomic_t *v)
1778 {
1779         arch_atomic_xor(i, v);
1780 }
1781
1782 /**
1783  * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1784  * @i: int value
1785  * @v: pointer to atomic_t
1786  *
1787  * Atomically updates @v to (@v ^ @i) with full ordering.
1788  *
1789  * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1790  *
1791  * Return: The original value of @v.
1792  */
1793 static __always_inline int
1794 raw_atomic_fetch_xor(int i, atomic_t *v)
1795 {
1796 #if defined(arch_atomic_fetch_xor)
1797         return arch_atomic_fetch_xor(i, v);
1798 #elif defined(arch_atomic_fetch_xor_relaxed)
1799         int ret;
1800         __atomic_pre_full_fence();
1801         ret = arch_atomic_fetch_xor_relaxed(i, v);
1802         __atomic_post_full_fence();
1803         return ret;
1804 #else
1805 #error "Unable to define raw_atomic_fetch_xor"
1806 #endif
1807 }
1808
1809 /**
1810  * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1811  * @i: int value
1812  * @v: pointer to atomic_t
1813  *
1814  * Atomically updates @v to (@v ^ @i) with acquire ordering.
1815  *
1816  * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
1817  *
1818  * Return: The original value of @v.
1819  */
1820 static __always_inline int
1821 raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
1822 {
1823 #if defined(arch_atomic_fetch_xor_acquire)
1824         return arch_atomic_fetch_xor_acquire(i, v);
1825 #elif defined(arch_atomic_fetch_xor_relaxed)
1826         int ret = arch_atomic_fetch_xor_relaxed(i, v);
1827         __atomic_acquire_fence();
1828         return ret;
1829 #elif defined(arch_atomic_fetch_xor)
1830         return arch_atomic_fetch_xor(i, v);
1831 #else
1832 #error "Unable to define raw_atomic_fetch_xor_acquire"
1833 #endif
1834 }
1835
1836 /**
1837  * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1838  * @i: int value
1839  * @v: pointer to atomic_t
1840  *
1841  * Atomically updates @v to (@v ^ @i) with release ordering.
1842  *
1843  * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1844  *
1845  * Return: The original value of @v.
1846  */
1847 static __always_inline int
1848 raw_atomic_fetch_xor_release(int i, atomic_t *v)
1849 {
1850 #if defined(arch_atomic_fetch_xor_release)
1851         return arch_atomic_fetch_xor_release(i, v);
1852 #elif defined(arch_atomic_fetch_xor_relaxed)
1853         __atomic_release_fence();
1854         return arch_atomic_fetch_xor_relaxed(i, v);
1855 #elif defined(arch_atomic_fetch_xor)
1856         return arch_atomic_fetch_xor(i, v);
1857 #else
1858 #error "Unable to define raw_atomic_fetch_xor_release"
1859 #endif
1860 }
1861
1862 /**
1863  * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1864  * @i: int value
1865  * @v: pointer to atomic_t
1866  *
1867  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1868  *
1869  * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1870  *
1871  * Return: The original value of @v.
1872  */
1873 static __always_inline int
1874 raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1875 {
1876 #if defined(arch_atomic_fetch_xor_relaxed)
1877         return arch_atomic_fetch_xor_relaxed(i, v);
1878 #elif defined(arch_atomic_fetch_xor)
1879         return arch_atomic_fetch_xor(i, v);
1880 #else
1881 #error "Unable to define raw_atomic_fetch_xor_relaxed"
1882 #endif
1883 }
1884
1885 /**
1886  * raw_atomic_xchg() - atomic exchange with full ordering
1887  * @v: pointer to atomic_t
1888  * @new: int value to assign
1889  *
1890  * Atomically updates @v to @new with full ordering.
1891  *
1892  * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1893  *
1894  * Return: The original value of @v.
1895  */
1896 static __always_inline int
1897 raw_atomic_xchg(atomic_t *v, int new)
1898 {
1899 #if defined(arch_atomic_xchg)
1900         return arch_atomic_xchg(v, new);
1901 #elif defined(arch_atomic_xchg_relaxed)
1902         int ret;
1903         __atomic_pre_full_fence();
1904         ret = arch_atomic_xchg_relaxed(v, new);
1905         __atomic_post_full_fence();
1906         return ret;
1907 #else
1908         return raw_xchg(&v->counter, new);
1909 #endif
1910 }
1911
1912 /**
1913  * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1914  * @v: pointer to atomic_t
1915  * @new: int value to assign
1916  *
1917  * Atomically updates @v to @new with acquire ordering.
1918  *
1919  * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1920  *
1921  * Return: The original value of @v.
1922  */
1923 static __always_inline int
1924 raw_atomic_xchg_acquire(atomic_t *v, int new)
1925 {
1926 #if defined(arch_atomic_xchg_acquire)
1927         return arch_atomic_xchg_acquire(v, new);
1928 #elif defined(arch_atomic_xchg_relaxed)
1929         int ret = arch_atomic_xchg_relaxed(v, new);
1930         __atomic_acquire_fence();
1931         return ret;
1932 #elif defined(arch_atomic_xchg)
1933         return arch_atomic_xchg(v, new);
1934 #else
1935         return raw_xchg_acquire(&v->counter, new);
1936 #endif
1937 }
1938
1939 /**
1940  * raw_atomic_xchg_release() - atomic exchange with release ordering
1941  * @v: pointer to atomic_t
1942  * @new: int value to assign
1943  *
1944  * Atomically updates @v to @new with release ordering.
1945  *
1946  * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1947  *
1948  * Return: The original value of @v.
1949  */
1950 static __always_inline int
1951 raw_atomic_xchg_release(atomic_t *v, int new)
1952 {
1953 #if defined(arch_atomic_xchg_release)
1954         return arch_atomic_xchg_release(v, new);
1955 #elif defined(arch_atomic_xchg_relaxed)
1956         __atomic_release_fence();
1957         return arch_atomic_xchg_relaxed(v, new);
1958 #elif defined(arch_atomic_xchg)
1959         return arch_atomic_xchg(v, new);
1960 #else
1961         return raw_xchg_release(&v->counter, new);
1962 #endif
1963 }
1964
1965 /**
1966  * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1967  * @v: pointer to atomic_t
1968  * @new: int value to assign
1969  *
1970  * Atomically updates @v to @new with relaxed ordering.
1971  *
1972  * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1973  *
1974  * Return: The original value of @v.
1975  */
1976 static __always_inline int
1977 raw_atomic_xchg_relaxed(atomic_t *v, int new)
1978 {
1979 #if defined(arch_atomic_xchg_relaxed)
1980         return arch_atomic_xchg_relaxed(v, new);
1981 #elif defined(arch_atomic_xchg)
1982         return arch_atomic_xchg(v, new);
1983 #else
1984         return raw_xchg_relaxed(&v->counter, new);
1985 #endif
1986 }
1987
1988 /**
1989  * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
1990  * @v: pointer to atomic_t
1991  * @old: int value to compare with
1992  * @new: int value to assign
1993  *
1994  * If (@v == @old), atomically updates @v to @new with full ordering.
1995  *
1996  * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
1997  *
1998  * Return: The original value of @v.
1999  */
2000 static __always_inline int
2001 raw_atomic_cmpxchg(atomic_t *v, int old, int new)
2002 {
2003 #if defined(arch_atomic_cmpxchg)
2004         return arch_atomic_cmpxchg(v, old, new);
2005 #elif defined(arch_atomic_cmpxchg_relaxed)
2006         int ret;
2007         __atomic_pre_full_fence();
2008         ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2009         __atomic_post_full_fence();
2010         return ret;
2011 #else
2012         return raw_cmpxchg(&v->counter, old, new);
2013 #endif
2014 }
2015
2016 /**
2017  * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2018  * @v: pointer to atomic_t
2019  * @old: int value to compare with
2020  * @new: int value to assign
2021  *
2022  * If (@v == @old), atomically updates @v to @new with acquire ordering.
2023  *
2024  * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2025  *
2026  * Return: The original value of @v.
2027  */
2028 static __always_inline int
2029 raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
2030 {
2031 #if defined(arch_atomic_cmpxchg_acquire)
2032         return arch_atomic_cmpxchg_acquire(v, old, new);
2033 #elif defined(arch_atomic_cmpxchg_relaxed)
2034         int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2035         __atomic_acquire_fence();
2036         return ret;
2037 #elif defined(arch_atomic_cmpxchg)
2038         return arch_atomic_cmpxchg(v, old, new);
2039 #else
2040         return raw_cmpxchg_acquire(&v->counter, old, new);
2041 #endif
2042 }
2043
2044 /**
2045  * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2046  * @v: pointer to atomic_t
2047  * @old: int value to compare with
2048  * @new: int value to assign
2049  *
2050  * If (@v == @old), atomically updates @v to @new with release ordering.
2051  *
2052  * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2053  *
2054  * Return: The original value of @v.
2055  */
2056 static __always_inline int
2057 raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
2058 {
2059 #if defined(arch_atomic_cmpxchg_release)
2060         return arch_atomic_cmpxchg_release(v, old, new);
2061 #elif defined(arch_atomic_cmpxchg_relaxed)
2062         __atomic_release_fence();
2063         return arch_atomic_cmpxchg_relaxed(v, old, new);
2064 #elif defined(arch_atomic_cmpxchg)
2065         return arch_atomic_cmpxchg(v, old, new);
2066 #else
2067         return raw_cmpxchg_release(&v->counter, old, new);
2068 #endif
2069 }
2070
2071 /**
2072  * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2073  * @v: pointer to atomic_t
2074  * @old: int value to compare with
2075  * @new: int value to assign
2076  *
2077  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2078  *
2079  * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2080  *
2081  * Return: The original value of @v.
2082  */
2083 static __always_inline int
2084 raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
2085 {
2086 #if defined(arch_atomic_cmpxchg_relaxed)
2087         return arch_atomic_cmpxchg_relaxed(v, old, new);
2088 #elif defined(arch_atomic_cmpxchg)
2089         return arch_atomic_cmpxchg(v, old, new);
2090 #else
2091         return raw_cmpxchg_relaxed(&v->counter, old, new);
2092 #endif
2093 }
2094
2095 /**
2096  * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
2097  * @v: pointer to atomic_t
2098  * @old: pointer to int value to compare with
2099  * @new: int value to assign
2100  *
2101  * If (@v == @old), atomically updates @v to @new with full ordering.
2102  * Otherwise, updates @old to the current value of @v.
2103  *
2104  * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2105  *
2106  * Return: @true if the exchange occured, @false otherwise.
2107  */
2108 static __always_inline bool
2109 raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
2110 {
2111 #if defined(arch_atomic_try_cmpxchg)
2112         return arch_atomic_try_cmpxchg(v, old, new);
2113 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2114         bool ret;
2115         __atomic_pre_full_fence();
2116         ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2117         __atomic_post_full_fence();
2118         return ret;
2119 #else
2120         int r, o = *old;
2121         r = raw_atomic_cmpxchg(v, o, new);
2122         if (unlikely(r != o))
2123                 *old = r;
2124         return likely(r == o);
2125 #endif
2126 }
2127
2128 /**
2129  * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2130  * @v: pointer to atomic_t
2131  * @old: pointer to int value to compare with
2132  * @new: int value to assign
2133  *
2134  * If (@v == @old), atomically updates @v to @new with acquire ordering.
2135  * Otherwise, updates @old to the current value of @v.
2136  *
2137  * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2138  *
2139  * Return: @true if the exchange occured, @false otherwise.
2140  */
2141 static __always_inline bool
2142 raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
2143 {
2144 #if defined(arch_atomic_try_cmpxchg_acquire)
2145         return arch_atomic_try_cmpxchg_acquire(v, old, new);
2146 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2147         bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2148         __atomic_acquire_fence();
2149         return ret;
2150 #elif defined(arch_atomic_try_cmpxchg)
2151         return arch_atomic_try_cmpxchg(v, old, new);
2152 #else
2153         int r, o = *old;
2154         r = raw_atomic_cmpxchg_acquire(v, o, new);
2155         if (unlikely(r != o))
2156                 *old = r;
2157         return likely(r == o);
2158 #endif
2159 }
2160
2161 /**
2162  * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2163  * @v: pointer to atomic_t
2164  * @old: pointer to int value to compare with
2165  * @new: int value to assign
2166  *
2167  * If (@v == @old), atomically updates @v to @new with release ordering.
2168  * Otherwise, updates @old to the current value of @v.
2169  *
2170  * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2171  *
2172  * Return: @true if the exchange occured, @false otherwise.
2173  */
2174 static __always_inline bool
2175 raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
2176 {
2177 #if defined(arch_atomic_try_cmpxchg_release)
2178         return arch_atomic_try_cmpxchg_release(v, old, new);
2179 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2180         __atomic_release_fence();
2181         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2182 #elif defined(arch_atomic_try_cmpxchg)
2183         return arch_atomic_try_cmpxchg(v, old, new);
2184 #else
2185         int r, o = *old;
2186         r = raw_atomic_cmpxchg_release(v, o, new);
2187         if (unlikely(r != o))
2188                 *old = r;
2189         return likely(r == o);
2190 #endif
2191 }
2192
2193 /**
2194  * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2195  * @v: pointer to atomic_t
2196  * @old: pointer to int value to compare with
2197  * @new: int value to assign
2198  *
2199  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2200  * Otherwise, updates @old to the current value of @v.
2201  *
2202  * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2203  *
2204  * Return: @true if the exchange occured, @false otherwise.
2205  */
2206 static __always_inline bool
2207 raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
2208 {
2209 #if defined(arch_atomic_try_cmpxchg_relaxed)
2210         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2211 #elif defined(arch_atomic_try_cmpxchg)
2212         return arch_atomic_try_cmpxchg(v, old, new);
2213 #else
2214         int r, o = *old;
2215         r = raw_atomic_cmpxchg_relaxed(v, o, new);
2216         if (unlikely(r != o))
2217                 *old = r;
2218         return likely(r == o);
2219 #endif
2220 }
2221
2222 /**
2223  * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
2224  * @i: int value to add
2225  * @v: pointer to atomic_t
2226  *
2227  * Atomically updates @v to (@v - @i) with full ordering.
2228  *
2229  * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
2230  *
2231  * Return: @true if the resulting value of @v is zero, @false otherwise.
2232  */
2233 static __always_inline bool
2234 raw_atomic_sub_and_test(int i, atomic_t *v)
2235 {
2236 #if defined(arch_atomic_sub_and_test)
2237         return arch_atomic_sub_and_test(i, v);
2238 #else
2239         return raw_atomic_sub_return(i, v) == 0;
2240 #endif
2241 }
2242
2243 /**
2244  * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
2245  * @v: pointer to atomic_t
2246  *
2247  * Atomically updates @v to (@v - 1) with full ordering.
2248  *
2249  * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
2250  *
2251  * Return: @true if the resulting value of @v is zero, @false otherwise.
2252  */
2253 static __always_inline bool
2254 raw_atomic_dec_and_test(atomic_t *v)
2255 {
2256 #if defined(arch_atomic_dec_and_test)
2257         return arch_atomic_dec_and_test(v);
2258 #else
2259         return raw_atomic_dec_return(v) == 0;
2260 #endif
2261 }
2262
2263 /**
2264  * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
2265  * @v: pointer to atomic_t
2266  *
2267  * Atomically updates @v to (@v + 1) with full ordering.
2268  *
2269  * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
2270  *
2271  * Return: @true if the resulting value of @v is zero, @false otherwise.
2272  */
2273 static __always_inline bool
2274 raw_atomic_inc_and_test(atomic_t *v)
2275 {
2276 #if defined(arch_atomic_inc_and_test)
2277         return arch_atomic_inc_and_test(v);
2278 #else
2279         return raw_atomic_inc_return(v) == 0;
2280 #endif
2281 }
2282
2283 /**
2284  * raw_atomic_add_negative() - atomic add and test if negative with full ordering
2285  * @i: int value to add
2286  * @v: pointer to atomic_t
2287  *
2288  * Atomically updates @v to (@v + @i) with full ordering.
2289  *
2290  * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2291  *
2292  * Return: @true if the resulting value of @v is negative, @false otherwise.
2293  */
2294 static __always_inline bool
2295 raw_atomic_add_negative(int i, atomic_t *v)
2296 {
2297 #if defined(arch_atomic_add_negative)
2298         return arch_atomic_add_negative(i, v);
2299 #elif defined(arch_atomic_add_negative_relaxed)
2300         bool ret;
2301         __atomic_pre_full_fence();
2302         ret = arch_atomic_add_negative_relaxed(i, v);
2303         __atomic_post_full_fence();
2304         return ret;
2305 #else
2306         return raw_atomic_add_return(i, v) < 0;
2307 #endif
2308 }
2309
2310 /**
2311  * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
2312  * @i: int value to add
2313  * @v: pointer to atomic_t
2314  *
2315  * Atomically updates @v to (@v + @i) with acquire ordering.
2316  *
2317  * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
2318  *
2319  * Return: @true if the resulting value of @v is negative, @false otherwise.
2320  */
2321 static __always_inline bool
2322 raw_atomic_add_negative_acquire(int i, atomic_t *v)
2323 {
2324 #if defined(arch_atomic_add_negative_acquire)
2325         return arch_atomic_add_negative_acquire(i, v);
2326 #elif defined(arch_atomic_add_negative_relaxed)
2327         bool ret = arch_atomic_add_negative_relaxed(i, v);
2328         __atomic_acquire_fence();
2329         return ret;
2330 #elif defined(arch_atomic_add_negative)
2331         return arch_atomic_add_negative(i, v);
2332 #else
2333         return raw_atomic_add_return_acquire(i, v) < 0;
2334 #endif
2335 }
2336
2337 /**
2338  * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
2339  * @i: int value to add
2340  * @v: pointer to atomic_t
2341  *
2342  * Atomically updates @v to (@v + @i) with release ordering.
2343  *
2344  * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2345  *
2346  * Return: @true if the resulting value of @v is negative, @false otherwise.
2347  */
2348 static __always_inline bool
2349 raw_atomic_add_negative_release(int i, atomic_t *v)
2350 {
2351 #if defined(arch_atomic_add_negative_release)
2352         return arch_atomic_add_negative_release(i, v);
2353 #elif defined(arch_atomic_add_negative_relaxed)
2354         __atomic_release_fence();
2355         return arch_atomic_add_negative_relaxed(i, v);
2356 #elif defined(arch_atomic_add_negative)
2357         return arch_atomic_add_negative(i, v);
2358 #else
2359         return raw_atomic_add_return_release(i, v) < 0;
2360 #endif
2361 }
2362
2363 /**
2364  * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
2365  * @i: int value to add
2366  * @v: pointer to atomic_t
2367  *
2368  * Atomically updates @v to (@v + @i) with relaxed ordering.
2369  *
2370  * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
2371  *
2372  * Return: @true if the resulting value of @v is negative, @false otherwise.
2373  */
2374 static __always_inline bool
2375 raw_atomic_add_negative_relaxed(int i, atomic_t *v)
2376 {
2377 #if defined(arch_atomic_add_negative_relaxed)
2378         return arch_atomic_add_negative_relaxed(i, v);
2379 #elif defined(arch_atomic_add_negative)
2380         return arch_atomic_add_negative(i, v);
2381 #else
2382         return raw_atomic_add_return_relaxed(i, v) < 0;
2383 #endif
2384 }
2385
2386 /**
2387  * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
2388  * @v: pointer to atomic_t
2389  * @a: int value to add
2390  * @u: int value to compare with
2391  *
2392  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2393  *
2394  * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2395  *
2396  * Return: The original value of @v.
2397  */
2398 static __always_inline int
2399 raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
2400 {
2401 #if defined(arch_atomic_fetch_add_unless)
2402         return arch_atomic_fetch_add_unless(v, a, u);
2403 #else
2404         int c = raw_atomic_read(v);
2405
2406         do {
2407                 if (unlikely(c == u))
2408                         break;
2409         } while (!raw_atomic_try_cmpxchg(v, &c, c + a));
2410
2411         return c;
2412 #endif
2413 }
2414
2415 /**
2416  * raw_atomic_add_unless() - atomic add unless value with full ordering
2417  * @v: pointer to atomic_t
2418  * @a: int value to add
2419  * @u: int value to compare with
2420  *
2421  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2422  *
2423  * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2424  *
2425  * Return: @true if @v was updated, @false otherwise.
2426  */
2427 static __always_inline bool
2428 raw_atomic_add_unless(atomic_t *v, int a, int u)
2429 {
2430 #if defined(arch_atomic_add_unless)
2431         return arch_atomic_add_unless(v, a, u);
2432 #else
2433         return raw_atomic_fetch_add_unless(v, a, u) != u;
2434 #endif
2435 }
2436
2437 /**
2438  * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
2439  * @v: pointer to atomic_t
2440  *
2441  * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2442  *
2443  * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2444  *
2445  * Return: @true if @v was updated, @false otherwise.
2446  */
2447 static __always_inline bool
2448 raw_atomic_inc_not_zero(atomic_t *v)
2449 {
2450 #if defined(arch_atomic_inc_not_zero)
2451         return arch_atomic_inc_not_zero(v);
2452 #else
2453         return raw_atomic_add_unless(v, 1, 0);
2454 #endif
2455 }
2456
2457 /**
2458  * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2459  * @v: pointer to atomic_t
2460  *
2461  * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2462  *
2463  * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2464  *
2465  * Return: @true if @v was updated, @false otherwise.
2466  */
2467 static __always_inline bool
2468 raw_atomic_inc_unless_negative(atomic_t *v)
2469 {
2470 #if defined(arch_atomic_inc_unless_negative)
2471         return arch_atomic_inc_unless_negative(v);
2472 #else
2473         int c = raw_atomic_read(v);
2474
2475         do {
2476                 if (unlikely(c < 0))
2477                         return false;
2478         } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
2479
2480         return true;
2481 #endif
2482 }
2483
2484 /**
2485  * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2486  * @v: pointer to atomic_t
2487  *
2488  * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2489  *
2490  * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2491  *
2492  * Return: @true if @v was updated, @false otherwise.
2493  */
2494 static __always_inline bool
2495 raw_atomic_dec_unless_positive(atomic_t *v)
2496 {
2497 #if defined(arch_atomic_dec_unless_positive)
2498         return arch_atomic_dec_unless_positive(v);
2499 #else
2500         int c = raw_atomic_read(v);
2501
2502         do {
2503                 if (unlikely(c > 0))
2504                         return false;
2505         } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
2506
2507         return true;
2508 #endif
2509 }
2510
2511 /**
2512  * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2513  * @v: pointer to atomic_t
2514  *
2515  * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2516  *
2517  * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2518  *
2519  * Return: The old value of (@v - 1), regardless of whether @v was updated.
2520  */
2521 static __always_inline int
2522 raw_atomic_dec_if_positive(atomic_t *v)
2523 {
2524 #if defined(arch_atomic_dec_if_positive)
2525         return arch_atomic_dec_if_positive(v);
2526 #else
2527         int dec, c = raw_atomic_read(v);
2528
2529         do {
2530                 dec = c - 1;
2531                 if (unlikely(dec < 0))
2532                         break;
2533         } while (!raw_atomic_try_cmpxchg(v, &c, dec));
2534
2535         return dec;
2536 #endif
2537 }
2538
2539 #ifdef CONFIG_GENERIC_ATOMIC64
2540 #include <asm-generic/atomic64.h>
2541 #endif
2542
2543 /**
2544  * raw_atomic64_read() - atomic load with relaxed ordering
2545  * @v: pointer to atomic64_t
2546  *
2547  * Atomically loads the value of @v with relaxed ordering.
2548  *
2549  * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2550  *
2551  * Return: The value loaded from @v.
2552  */
2553 static __always_inline s64
2554 raw_atomic64_read(const atomic64_t *v)
2555 {
2556         return arch_atomic64_read(v);
2557 }
2558
2559 /**
2560  * raw_atomic64_read_acquire() - atomic load with acquire ordering
2561  * @v: pointer to atomic64_t
2562  *
2563  * Atomically loads the value of @v with acquire ordering.
2564  *
2565  * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2566  *
2567  * Return: The value loaded from @v.
2568  */
2569 static __always_inline s64
2570 raw_atomic64_read_acquire(const atomic64_t *v)
2571 {
2572 #if defined(arch_atomic64_read_acquire)
2573         return arch_atomic64_read_acquire(v);
2574 #else
2575         s64 ret;
2576
2577         if (__native_word(atomic64_t)) {
2578                 ret = smp_load_acquire(&(v)->counter);
2579         } else {
2580                 ret = raw_atomic64_read(v);
2581                 __atomic_acquire_fence();
2582         }
2583
2584         return ret;
2585 #endif
2586 }
2587
2588 /**
2589  * raw_atomic64_set() - atomic set with relaxed ordering
2590  * @v: pointer to atomic64_t
2591  * @i: s64 value to assign
2592  *
2593  * Atomically sets @v to @i with relaxed ordering.
2594  *
2595  * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2596  *
2597  * Return: Nothing.
2598  */
2599 static __always_inline void
2600 raw_atomic64_set(atomic64_t *v, s64 i)
2601 {
2602         arch_atomic64_set(v, i);
2603 }
2604
2605 /**
2606  * raw_atomic64_set_release() - atomic set with release ordering
2607  * @v: pointer to atomic64_t
2608  * @i: s64 value to assign
2609  *
2610  * Atomically sets @v to @i with release ordering.
2611  *
2612  * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2613  *
2614  * Return: Nothing.
2615  */
2616 static __always_inline void
2617 raw_atomic64_set_release(atomic64_t *v, s64 i)
2618 {
2619 #if defined(arch_atomic64_set_release)
2620         arch_atomic64_set_release(v, i);
2621 #else
2622         if (__native_word(atomic64_t)) {
2623                 smp_store_release(&(v)->counter, i);
2624         } else {
2625                 __atomic_release_fence();
2626                 raw_atomic64_set(v, i);
2627         }
2628 #endif
2629 }
2630
2631 /**
2632  * raw_atomic64_add() - atomic add with relaxed ordering
2633  * @i: s64 value to add
2634  * @v: pointer to atomic64_t
2635  *
2636  * Atomically updates @v to (@v + @i) with relaxed ordering.
2637  *
2638  * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2639  *
2640  * Return: Nothing.
2641  */
2642 static __always_inline void
2643 raw_atomic64_add(s64 i, atomic64_t *v)
2644 {
2645         arch_atomic64_add(i, v);
2646 }
2647
2648 /**
2649  * raw_atomic64_add_return() - atomic add with full ordering
2650  * @i: s64 value to add
2651  * @v: pointer to atomic64_t
2652  *
2653  * Atomically updates @v to (@v + @i) with full ordering.
2654  *
2655  * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2656  *
2657  * Return: The updated value of @v.
2658  */
2659 static __always_inline s64
2660 raw_atomic64_add_return(s64 i, atomic64_t *v)
2661 {
2662 #if defined(arch_atomic64_add_return)
2663         return arch_atomic64_add_return(i, v);
2664 #elif defined(arch_atomic64_add_return_relaxed)
2665         s64 ret;
2666         __atomic_pre_full_fence();
2667         ret = arch_atomic64_add_return_relaxed(i, v);
2668         __atomic_post_full_fence();
2669         return ret;
2670 #else
2671 #error "Unable to define raw_atomic64_add_return"
2672 #endif
2673 }
2674
2675 /**
2676  * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2677  * @i: s64 value to add
2678  * @v: pointer to atomic64_t
2679  *
2680  * Atomically updates @v to (@v + @i) with acquire ordering.
2681  *
2682  * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2683  *
2684  * Return: The updated value of @v.
2685  */
2686 static __always_inline s64
2687 raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
2688 {
2689 #if defined(arch_atomic64_add_return_acquire)
2690         return arch_atomic64_add_return_acquire(i, v);
2691 #elif defined(arch_atomic64_add_return_relaxed)
2692         s64 ret = arch_atomic64_add_return_relaxed(i, v);
2693         __atomic_acquire_fence();
2694         return ret;
2695 #elif defined(arch_atomic64_add_return)
2696         return arch_atomic64_add_return(i, v);
2697 #else
2698 #error "Unable to define raw_atomic64_add_return_acquire"
2699 #endif
2700 }
2701
2702 /**
2703  * raw_atomic64_add_return_release() - atomic add with release ordering
2704  * @i: s64 value to add
2705  * @v: pointer to atomic64_t
2706  *
2707  * Atomically updates @v to (@v + @i) with release ordering.
2708  *
2709  * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2710  *
2711  * Return: The updated value of @v.
2712  */
2713 static __always_inline s64
2714 raw_atomic64_add_return_release(s64 i, atomic64_t *v)
2715 {
2716 #if defined(arch_atomic64_add_return_release)
2717         return arch_atomic64_add_return_release(i, v);
2718 #elif defined(arch_atomic64_add_return_relaxed)
2719         __atomic_release_fence();
2720         return arch_atomic64_add_return_relaxed(i, v);
2721 #elif defined(arch_atomic64_add_return)
2722         return arch_atomic64_add_return(i, v);
2723 #else
2724 #error "Unable to define raw_atomic64_add_return_release"
2725 #endif
2726 }
2727
2728 /**
2729  * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2730  * @i: s64 value to add
2731  * @v: pointer to atomic64_t
2732  *
2733  * Atomically updates @v to (@v + @i) with relaxed ordering.
2734  *
2735  * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2736  *
2737  * Return: The updated value of @v.
2738  */
2739 static __always_inline s64
2740 raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2741 {
2742 #if defined(arch_atomic64_add_return_relaxed)
2743         return arch_atomic64_add_return_relaxed(i, v);
2744 #elif defined(arch_atomic64_add_return)
2745         return arch_atomic64_add_return(i, v);
2746 #else
2747 #error "Unable to define raw_atomic64_add_return_relaxed"
2748 #endif
2749 }
2750
2751 /**
2752  * raw_atomic64_fetch_add() - atomic add with full ordering
2753  * @i: s64 value to add
2754  * @v: pointer to atomic64_t
2755  *
2756  * Atomically updates @v to (@v + @i) with full ordering.
2757  *
2758  * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2759  *
2760  * Return: The original value of @v.
2761  */
2762 static __always_inline s64
2763 raw_atomic64_fetch_add(s64 i, atomic64_t *v)
2764 {
2765 #if defined(arch_atomic64_fetch_add)
2766         return arch_atomic64_fetch_add(i, v);
2767 #elif defined(arch_atomic64_fetch_add_relaxed)
2768         s64 ret;
2769         __atomic_pre_full_fence();
2770         ret = arch_atomic64_fetch_add_relaxed(i, v);
2771         __atomic_post_full_fence();
2772         return ret;
2773 #else
2774 #error "Unable to define raw_atomic64_fetch_add"
2775 #endif
2776 }
2777
2778 /**
2779  * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
2780  * @i: s64 value to add
2781  * @v: pointer to atomic64_t
2782  *
2783  * Atomically updates @v to (@v + @i) with acquire ordering.
2784  *
2785  * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
2786  *
2787  * Return: The original value of @v.
2788  */
2789 static __always_inline s64
2790 raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
2791 {
2792 #if defined(arch_atomic64_fetch_add_acquire)
2793         return arch_atomic64_fetch_add_acquire(i, v);
2794 #elif defined(arch_atomic64_fetch_add_relaxed)
2795         s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
2796         __atomic_acquire_fence();
2797         return ret;
2798 #elif defined(arch_atomic64_fetch_add)
2799         return arch_atomic64_fetch_add(i, v);
2800 #else
2801 #error "Unable to define raw_atomic64_fetch_add_acquire"
2802 #endif
2803 }
2804
2805 /**
2806  * raw_atomic64_fetch_add_release() - atomic add with release ordering
2807  * @i: s64 value to add
2808  * @v: pointer to atomic64_t
2809  *
2810  * Atomically updates @v to (@v + @i) with release ordering.
2811  *
2812  * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2813  *
2814  * Return: The original value of @v.
2815  */
2816 static __always_inline s64
2817 raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
2818 {
2819 #if defined(arch_atomic64_fetch_add_release)
2820         return arch_atomic64_fetch_add_release(i, v);
2821 #elif defined(arch_atomic64_fetch_add_relaxed)
2822         __atomic_release_fence();
2823         return arch_atomic64_fetch_add_relaxed(i, v);
2824 #elif defined(arch_atomic64_fetch_add)
2825         return arch_atomic64_fetch_add(i, v);
2826 #else
2827 #error "Unable to define raw_atomic64_fetch_add_release"
2828 #endif
2829 }
2830
2831 /**
2832  * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2833  * @i: s64 value to add
2834  * @v: pointer to atomic64_t
2835  *
2836  * Atomically updates @v to (@v + @i) with relaxed ordering.
2837  *
2838  * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2839  *
2840  * Return: The original value of @v.
2841  */
2842 static __always_inline s64
2843 raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2844 {
2845 #if defined(arch_atomic64_fetch_add_relaxed)
2846         return arch_atomic64_fetch_add_relaxed(i, v);
2847 #elif defined(arch_atomic64_fetch_add)
2848         return arch_atomic64_fetch_add(i, v);
2849 #else
2850 #error "Unable to define raw_atomic64_fetch_add_relaxed"
2851 #endif
2852 }
2853
2854 /**
2855  * raw_atomic64_sub() - atomic subtract with relaxed ordering
2856  * @i: s64 value to subtract
2857  * @v: pointer to atomic64_t
2858  *
2859  * Atomically updates @v to (@v - @i) with relaxed ordering.
2860  *
2861  * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2862  *
2863  * Return: Nothing.
2864  */
2865 static __always_inline void
2866 raw_atomic64_sub(s64 i, atomic64_t *v)
2867 {
2868         arch_atomic64_sub(i, v);
2869 }
2870
2871 /**
2872  * raw_atomic64_sub_return() - atomic subtract with full ordering
2873  * @i: s64 value to subtract
2874  * @v: pointer to atomic64_t
2875  *
2876  * Atomically updates @v to (@v - @i) with full ordering.
2877  *
2878  * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2879  *
2880  * Return: The updated value of @v.
2881  */
2882 static __always_inline s64
2883 raw_atomic64_sub_return(s64 i, atomic64_t *v)
2884 {
2885 #if defined(arch_atomic64_sub_return)
2886         return arch_atomic64_sub_return(i, v);
2887 #elif defined(arch_atomic64_sub_return_relaxed)
2888         s64 ret;
2889         __atomic_pre_full_fence();
2890         ret = arch_atomic64_sub_return_relaxed(i, v);
2891         __atomic_post_full_fence();
2892         return ret;
2893 #else
2894 #error "Unable to define raw_atomic64_sub_return"
2895 #endif
2896 }
2897
2898 /**
2899  * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
2900  * @i: s64 value to subtract
2901  * @v: pointer to atomic64_t
2902  *
2903  * Atomically updates @v to (@v - @i) with acquire ordering.
2904  *
2905  * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
2906  *
2907  * Return: The updated value of @v.
2908  */
2909 static __always_inline s64
2910 raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
2911 {
2912 #if defined(arch_atomic64_sub_return_acquire)
2913         return arch_atomic64_sub_return_acquire(i, v);
2914 #elif defined(arch_atomic64_sub_return_relaxed)
2915         s64 ret = arch_atomic64_sub_return_relaxed(i, v);
2916         __atomic_acquire_fence();
2917         return ret;
2918 #elif defined(arch_atomic64_sub_return)
2919         return arch_atomic64_sub_return(i, v);
2920 #else
2921 #error "Unable to define raw_atomic64_sub_return_acquire"
2922 #endif
2923 }
2924
2925 /**
2926  * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2927  * @i: s64 value to subtract
2928  * @v: pointer to atomic64_t
2929  *
2930  * Atomically updates @v to (@v - @i) with release ordering.
2931  *
2932  * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2933  *
2934  * Return: The updated value of @v.
2935  */
2936 static __always_inline s64
2937 raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
2938 {
2939 #if defined(arch_atomic64_sub_return_release)
2940         return arch_atomic64_sub_return_release(i, v);
2941 #elif defined(arch_atomic64_sub_return_relaxed)
2942         __atomic_release_fence();
2943         return arch_atomic64_sub_return_relaxed(i, v);
2944 #elif defined(arch_atomic64_sub_return)
2945         return arch_atomic64_sub_return(i, v);
2946 #else
2947 #error "Unable to define raw_atomic64_sub_return_release"
2948 #endif
2949 }
2950
2951 /**
2952  * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2953  * @i: s64 value to subtract
2954  * @v: pointer to atomic64_t
2955  *
2956  * Atomically updates @v to (@v - @i) with relaxed ordering.
2957  *
2958  * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2959  *
2960  * Return: The updated value of @v.
2961  */
2962 static __always_inline s64
2963 raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2964 {
2965 #if defined(arch_atomic64_sub_return_relaxed)
2966         return arch_atomic64_sub_return_relaxed(i, v);
2967 #elif defined(arch_atomic64_sub_return)
2968         return arch_atomic64_sub_return(i, v);
2969 #else
2970 #error "Unable to define raw_atomic64_sub_return_relaxed"
2971 #endif
2972 }
2973
2974 /**
2975  * raw_atomic64_fetch_sub() - atomic subtract with full ordering
2976  * @i: s64 value to subtract
2977  * @v: pointer to atomic64_t
2978  *
2979  * Atomically updates @v to (@v - @i) with full ordering.
2980  *
2981  * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
2982  *
2983  * Return: The original value of @v.
2984  */
2985 static __always_inline s64
2986 raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
2987 {
2988 #if defined(arch_atomic64_fetch_sub)
2989         return arch_atomic64_fetch_sub(i, v);
2990 #elif defined(arch_atomic64_fetch_sub_relaxed)
2991         s64 ret;
2992         __atomic_pre_full_fence();
2993         ret = arch_atomic64_fetch_sub_relaxed(i, v);
2994         __atomic_post_full_fence();
2995         return ret;
2996 #else
2997 #error "Unable to define raw_atomic64_fetch_sub"
2998 #endif
2999 }
3000
3001 /**
3002  * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
3003  * @i: s64 value to subtract
3004  * @v: pointer to atomic64_t
3005  *
3006  * Atomically updates @v to (@v - @i) with acquire ordering.
3007  *
3008  * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
3009  *
3010  * Return: The original value of @v.
3011  */
3012 static __always_inline s64
3013 raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
3014 {
3015 #if defined(arch_atomic64_fetch_sub_acquire)
3016         return arch_atomic64_fetch_sub_acquire(i, v);
3017 #elif defined(arch_atomic64_fetch_sub_relaxed)
3018         s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3019         __atomic_acquire_fence();
3020         return ret;
3021 #elif defined(arch_atomic64_fetch_sub)
3022         return arch_atomic64_fetch_sub(i, v);
3023 #else
3024 #error "Unable to define raw_atomic64_fetch_sub_acquire"
3025 #endif
3026 }
3027
3028 /**
3029  * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3030  * @i: s64 value to subtract
3031  * @v: pointer to atomic64_t
3032  *
3033  * Atomically updates @v to (@v - @i) with release ordering.
3034  *
3035  * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3036  *
3037  * Return: The original value of @v.
3038  */
3039 static __always_inline s64
3040 raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
3041 {
3042 #if defined(arch_atomic64_fetch_sub_release)
3043         return arch_atomic64_fetch_sub_release(i, v);
3044 #elif defined(arch_atomic64_fetch_sub_relaxed)
3045         __atomic_release_fence();
3046         return arch_atomic64_fetch_sub_relaxed(i, v);
3047 #elif defined(arch_atomic64_fetch_sub)
3048         return arch_atomic64_fetch_sub(i, v);
3049 #else
3050 #error "Unable to define raw_atomic64_fetch_sub_release"
3051 #endif
3052 }
3053
3054 /**
3055  * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3056  * @i: s64 value to subtract
3057  * @v: pointer to atomic64_t
3058  *
3059  * Atomically updates @v to (@v - @i) with relaxed ordering.
3060  *
3061  * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3062  *
3063  * Return: The original value of @v.
3064  */
3065 static __always_inline s64
3066 raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
3067 {
3068 #if defined(arch_atomic64_fetch_sub_relaxed)
3069         return arch_atomic64_fetch_sub_relaxed(i, v);
3070 #elif defined(arch_atomic64_fetch_sub)
3071         return arch_atomic64_fetch_sub(i, v);
3072 #else
3073 #error "Unable to define raw_atomic64_fetch_sub_relaxed"
3074 #endif
3075 }
3076
3077 /**
3078  * raw_atomic64_inc() - atomic increment with relaxed ordering
3079  * @v: pointer to atomic64_t
3080  *
3081  * Atomically updates @v to (@v + 1) with relaxed ordering.
3082  *
3083  * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
3084  *
3085  * Return: Nothing.
3086  */
3087 static __always_inline void
3088 raw_atomic64_inc(atomic64_t *v)
3089 {
3090 #if defined(arch_atomic64_inc)
3091         arch_atomic64_inc(v);
3092 #else
3093         raw_atomic64_add(1, v);
3094 #endif
3095 }
3096
3097 /**
3098  * raw_atomic64_inc_return() - atomic increment with full ordering
3099  * @v: pointer to atomic64_t
3100  *
3101  * Atomically updates @v to (@v + 1) with full ordering.
3102  *
3103  * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3104  *
3105  * Return: The updated value of @v.
3106  */
3107 static __always_inline s64
3108 raw_atomic64_inc_return(atomic64_t *v)
3109 {
3110 #if defined(arch_atomic64_inc_return)
3111         return arch_atomic64_inc_return(v);
3112 #elif defined(arch_atomic64_inc_return_relaxed)
3113         s64 ret;
3114         __atomic_pre_full_fence();
3115         ret = arch_atomic64_inc_return_relaxed(v);
3116         __atomic_post_full_fence();
3117         return ret;
3118 #else
3119         return raw_atomic64_add_return(1, v);
3120 #endif
3121 }
3122
3123 /**
3124  * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
3125  * @v: pointer to atomic64_t
3126  *
3127  * Atomically updates @v to (@v + 1) with acquire ordering.
3128  *
3129  * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3130  *
3131  * Return: The updated value of @v.
3132  */
3133 static __always_inline s64
3134 raw_atomic64_inc_return_acquire(atomic64_t *v)
3135 {
3136 #if defined(arch_atomic64_inc_return_acquire)
3137         return arch_atomic64_inc_return_acquire(v);
3138 #elif defined(arch_atomic64_inc_return_relaxed)
3139         s64 ret = arch_atomic64_inc_return_relaxed(v);
3140         __atomic_acquire_fence();
3141         return ret;
3142 #elif defined(arch_atomic64_inc_return)
3143         return arch_atomic64_inc_return(v);
3144 #else
3145         return raw_atomic64_add_return_acquire(1, v);
3146 #endif
3147 }
3148
3149 /**
3150  * raw_atomic64_inc_return_release() - atomic increment with release ordering
3151  * @v: pointer to atomic64_t
3152  *
3153  * Atomically updates @v to (@v + 1) with release ordering.
3154  *
3155  * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3156  *
3157  * Return: The updated value of @v.
3158  */
3159 static __always_inline s64
3160 raw_atomic64_inc_return_release(atomic64_t *v)
3161 {
3162 #if defined(arch_atomic64_inc_return_release)
3163         return arch_atomic64_inc_return_release(v);
3164 #elif defined(arch_atomic64_inc_return_relaxed)
3165         __atomic_release_fence();
3166         return arch_atomic64_inc_return_relaxed(v);
3167 #elif defined(arch_atomic64_inc_return)
3168         return arch_atomic64_inc_return(v);
3169 #else
3170         return raw_atomic64_add_return_release(1, v);
3171 #endif
3172 }
3173
3174 /**
3175  * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
3176  * @v: pointer to atomic64_t
3177  *
3178  * Atomically updates @v to (@v + 1) with relaxed ordering.
3179  *
3180  * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
3181  *
3182  * Return: The updated value of @v.
3183  */
3184 static __always_inline s64
3185 raw_atomic64_inc_return_relaxed(atomic64_t *v)
3186 {
3187 #if defined(arch_atomic64_inc_return_relaxed)
3188         return arch_atomic64_inc_return_relaxed(v);
3189 #elif defined(arch_atomic64_inc_return)
3190         return arch_atomic64_inc_return(v);
3191 #else
3192         return raw_atomic64_add_return_relaxed(1, v);
3193 #endif
3194 }
3195
3196 /**
3197  * raw_atomic64_fetch_inc() - atomic increment with full ordering
3198  * @v: pointer to atomic64_t
3199  *
3200  * Atomically updates @v to (@v + 1) with full ordering.
3201  *
3202  * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3203  *
3204  * Return: The original value of @v.
3205  */
3206 static __always_inline s64
3207 raw_atomic64_fetch_inc(atomic64_t *v)
3208 {
3209 #if defined(arch_atomic64_fetch_inc)
3210         return arch_atomic64_fetch_inc(v);
3211 #elif defined(arch_atomic64_fetch_inc_relaxed)
3212         s64 ret;
3213         __atomic_pre_full_fence();
3214         ret = arch_atomic64_fetch_inc_relaxed(v);
3215         __atomic_post_full_fence();
3216         return ret;
3217 #else
3218         return raw_atomic64_fetch_add(1, v);
3219 #endif
3220 }
3221
3222 /**
3223  * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3224  * @v: pointer to atomic64_t
3225  *
3226  * Atomically updates @v to (@v + 1) with acquire ordering.
3227  *
3228  * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3229  *
3230  * Return: The original value of @v.
3231  */
3232 static __always_inline s64
3233 raw_atomic64_fetch_inc_acquire(atomic64_t *v)
3234 {
3235 #if defined(arch_atomic64_fetch_inc_acquire)
3236         return arch_atomic64_fetch_inc_acquire(v);
3237 #elif defined(arch_atomic64_fetch_inc_relaxed)
3238         s64 ret = arch_atomic64_fetch_inc_relaxed(v);
3239         __atomic_acquire_fence();
3240         return ret;
3241 #elif defined(arch_atomic64_fetch_inc)
3242         return arch_atomic64_fetch_inc(v);
3243 #else
3244         return raw_atomic64_fetch_add_acquire(1, v);
3245 #endif
3246 }
3247
3248 /**
3249  * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3250  * @v: pointer to atomic64_t
3251  *
3252  * Atomically updates @v to (@v + 1) with release ordering.
3253  *
3254  * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3255  *
3256  * Return: The original value of @v.
3257  */
3258 static __always_inline s64
3259 raw_atomic64_fetch_inc_release(atomic64_t *v)
3260 {
3261 #if defined(arch_atomic64_fetch_inc_release)
3262         return arch_atomic64_fetch_inc_release(v);
3263 #elif defined(arch_atomic64_fetch_inc_relaxed)
3264         __atomic_release_fence();
3265         return arch_atomic64_fetch_inc_relaxed(v);
3266 #elif defined(arch_atomic64_fetch_inc)
3267         return arch_atomic64_fetch_inc(v);
3268 #else
3269         return raw_atomic64_fetch_add_release(1, v);
3270 #endif
3271 }
3272
3273 /**
3274  * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
3275  * @v: pointer to atomic64_t
3276  *
3277  * Atomically updates @v to (@v + 1) with relaxed ordering.
3278  *
3279  * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
3280  *
3281  * Return: The original value of @v.
3282  */
3283 static __always_inline s64
3284 raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
3285 {
3286 #if defined(arch_atomic64_fetch_inc_relaxed)
3287         return arch_atomic64_fetch_inc_relaxed(v);
3288 #elif defined(arch_atomic64_fetch_inc)
3289         return arch_atomic64_fetch_inc(v);
3290 #else
3291         return raw_atomic64_fetch_add_relaxed(1, v);
3292 #endif
3293 }
3294
3295 /**
3296  * raw_atomic64_dec() - atomic decrement with relaxed ordering
3297  * @v: pointer to atomic64_t
3298  *
3299  * Atomically updates @v to (@v - 1) with relaxed ordering.
3300  *
3301  * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3302  *
3303  * Return: Nothing.
3304  */
3305 static __always_inline void
3306 raw_atomic64_dec(atomic64_t *v)
3307 {
3308 #if defined(arch_atomic64_dec)
3309         arch_atomic64_dec(v);
3310 #else
3311         raw_atomic64_sub(1, v);
3312 #endif
3313 }
3314
3315 /**
3316  * raw_atomic64_dec_return() - atomic decrement with full ordering
3317  * @v: pointer to atomic64_t
3318  *
3319  * Atomically updates @v to (@v - 1) with full ordering.
3320  *
3321  * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3322  *
3323  * Return: The updated value of @v.
3324  */
3325 static __always_inline s64
3326 raw_atomic64_dec_return(atomic64_t *v)
3327 {
3328 #if defined(arch_atomic64_dec_return)
3329         return arch_atomic64_dec_return(v);
3330 #elif defined(arch_atomic64_dec_return_relaxed)
3331         s64 ret;
3332         __atomic_pre_full_fence();
3333         ret = arch_atomic64_dec_return_relaxed(v);
3334         __atomic_post_full_fence();
3335         return ret;
3336 #else
3337         return raw_atomic64_sub_return(1, v);
3338 #endif
3339 }
3340
3341 /**
3342  * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
3343  * @v: pointer to atomic64_t
3344  *
3345  * Atomically updates @v to (@v - 1) with acquire ordering.
3346  *
3347  * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3348  *
3349  * Return: The updated value of @v.
3350  */
3351 static __always_inline s64
3352 raw_atomic64_dec_return_acquire(atomic64_t *v)
3353 {
3354 #if defined(arch_atomic64_dec_return_acquire)
3355         return arch_atomic64_dec_return_acquire(v);
3356 #elif defined(arch_atomic64_dec_return_relaxed)
3357         s64 ret = arch_atomic64_dec_return_relaxed(v);
3358         __atomic_acquire_fence();
3359         return ret;
3360 #elif defined(arch_atomic64_dec_return)
3361         return arch_atomic64_dec_return(v);
3362 #else
3363         return raw_atomic64_sub_return_acquire(1, v);
3364 #endif
3365 }
3366
3367 /**
3368  * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3369  * @v: pointer to atomic64_t
3370  *
3371  * Atomically updates @v to (@v - 1) with release ordering.
3372  *
3373  * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3374  *
3375  * Return: The updated value of @v.
3376  */
3377 static __always_inline s64
3378 raw_atomic64_dec_return_release(atomic64_t *v)
3379 {
3380 #if defined(arch_atomic64_dec_return_release)
3381         return arch_atomic64_dec_return_release(v);
3382 #elif defined(arch_atomic64_dec_return_relaxed)
3383         __atomic_release_fence();
3384         return arch_atomic64_dec_return_relaxed(v);
3385 #elif defined(arch_atomic64_dec_return)
3386         return arch_atomic64_dec_return(v);
3387 #else
3388         return raw_atomic64_sub_return_release(1, v);
3389 #endif
3390 }
3391
3392 /**
3393  * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3394  * @v: pointer to atomic64_t
3395  *
3396  * Atomically updates @v to (@v - 1) with relaxed ordering.
3397  *
3398  * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3399  *
3400  * Return: The updated value of @v.
3401  */
3402 static __always_inline s64
3403 raw_atomic64_dec_return_relaxed(atomic64_t *v)
3404 {
3405 #if defined(arch_atomic64_dec_return_relaxed)
3406         return arch_atomic64_dec_return_relaxed(v);
3407 #elif defined(arch_atomic64_dec_return)
3408         return arch_atomic64_dec_return(v);
3409 #else
3410         return raw_atomic64_sub_return_relaxed(1, v);
3411 #endif
3412 }
3413
3414 /**
3415  * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3416  * @v: pointer to atomic64_t
3417  *
3418  * Atomically updates @v to (@v - 1) with full ordering.
3419  *
3420  * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3421  *
3422  * Return: The original value of @v.
3423  */
3424 static __always_inline s64
3425 raw_atomic64_fetch_dec(atomic64_t *v)
3426 {
3427 #if defined(arch_atomic64_fetch_dec)
3428         return arch_atomic64_fetch_dec(v);
3429 #elif defined(arch_atomic64_fetch_dec_relaxed)
3430         s64 ret;
3431         __atomic_pre_full_fence();
3432         ret = arch_atomic64_fetch_dec_relaxed(v);
3433         __atomic_post_full_fence();
3434         return ret;
3435 #else
3436         return raw_atomic64_fetch_sub(1, v);
3437 #endif
3438 }
3439
3440 /**
3441  * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3442  * @v: pointer to atomic64_t
3443  *
3444  * Atomically updates @v to (@v - 1) with acquire ordering.
3445  *
3446  * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3447  *
3448  * Return: The original value of @v.
3449  */
3450 static __always_inline s64
3451 raw_atomic64_fetch_dec_acquire(atomic64_t *v)
3452 {
3453 #if defined(arch_atomic64_fetch_dec_acquire)
3454         return arch_atomic64_fetch_dec_acquire(v);
3455 #elif defined(arch_atomic64_fetch_dec_relaxed)
3456         s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3457         __atomic_acquire_fence();
3458         return ret;
3459 #elif defined(arch_atomic64_fetch_dec)
3460         return arch_atomic64_fetch_dec(v);
3461 #else
3462         return raw_atomic64_fetch_sub_acquire(1, v);
3463 #endif
3464 }
3465
3466 /**
3467  * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3468  * @v: pointer to atomic64_t
3469  *
3470  * Atomically updates @v to (@v - 1) with release ordering.
3471  *
3472  * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3473  *
3474  * Return: The original value of @v.
3475  */
3476 static __always_inline s64
3477 raw_atomic64_fetch_dec_release(atomic64_t *v)
3478 {
3479 #if defined(arch_atomic64_fetch_dec_release)
3480         return arch_atomic64_fetch_dec_release(v);
3481 #elif defined(arch_atomic64_fetch_dec_relaxed)
3482         __atomic_release_fence();
3483         return arch_atomic64_fetch_dec_relaxed(v);
3484 #elif defined(arch_atomic64_fetch_dec)
3485         return arch_atomic64_fetch_dec(v);
3486 #else
3487         return raw_atomic64_fetch_sub_release(1, v);
3488 #endif
3489 }
3490
3491 /**
3492  * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3493  * @v: pointer to atomic64_t
3494  *
3495  * Atomically updates @v to (@v - 1) with relaxed ordering.
3496  *
3497  * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3498  *
3499  * Return: The original value of @v.
3500  */
3501 static __always_inline s64
3502 raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
3503 {
3504 #if defined(arch_atomic64_fetch_dec_relaxed)
3505         return arch_atomic64_fetch_dec_relaxed(v);
3506 #elif defined(arch_atomic64_fetch_dec)
3507         return arch_atomic64_fetch_dec(v);
3508 #else
3509         return raw_atomic64_fetch_sub_relaxed(1, v);
3510 #endif
3511 }
3512
3513 /**
3514  * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
3515  * @i: s64 value
3516  * @v: pointer to atomic64_t
3517  *
3518  * Atomically updates @v to (@v & @i) with relaxed ordering.
3519  *
3520  * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3521  *
3522  * Return: Nothing.
3523  */
3524 static __always_inline void
3525 raw_atomic64_and(s64 i, atomic64_t *v)
3526 {
3527         arch_atomic64_and(i, v);
3528 }
3529
3530 /**
3531  * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3532  * @i: s64 value
3533  * @v: pointer to atomic64_t
3534  *
3535  * Atomically updates @v to (@v & @i) with full ordering.
3536  *
3537  * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3538  *
3539  * Return: The original value of @v.
3540  */
3541 static __always_inline s64
3542 raw_atomic64_fetch_and(s64 i, atomic64_t *v)
3543 {
3544 #if defined(arch_atomic64_fetch_and)
3545         return arch_atomic64_fetch_and(i, v);
3546 #elif defined(arch_atomic64_fetch_and_relaxed)
3547         s64 ret;
3548         __atomic_pre_full_fence();
3549         ret = arch_atomic64_fetch_and_relaxed(i, v);
3550         __atomic_post_full_fence();
3551         return ret;
3552 #else
3553 #error "Unable to define raw_atomic64_fetch_and"
3554 #endif
3555 }
3556
3557 /**
3558  * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3559  * @i: s64 value
3560  * @v: pointer to atomic64_t
3561  *
3562  * Atomically updates @v to (@v & @i) with acquire ordering.
3563  *
3564  * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
3565  *
3566  * Return: The original value of @v.
3567  */
3568 static __always_inline s64
3569 raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
3570 {
3571 #if defined(arch_atomic64_fetch_and_acquire)
3572         return arch_atomic64_fetch_and_acquire(i, v);
3573 #elif defined(arch_atomic64_fetch_and_relaxed)
3574         s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
3575         __atomic_acquire_fence();
3576         return ret;
3577 #elif defined(arch_atomic64_fetch_and)
3578         return arch_atomic64_fetch_and(i, v);
3579 #else
3580 #error "Unable to define raw_atomic64_fetch_and_acquire"
3581 #endif
3582 }
3583
3584 /**
3585  * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3586  * @i: s64 value
3587  * @v: pointer to atomic64_t
3588  *
3589  * Atomically updates @v to (@v & @i) with release ordering.
3590  *
3591  * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3592  *
3593  * Return: The original value of @v.
3594  */
3595 static __always_inline s64
3596 raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
3597 {
3598 #if defined(arch_atomic64_fetch_and_release)
3599         return arch_atomic64_fetch_and_release(i, v);
3600 #elif defined(arch_atomic64_fetch_and_relaxed)
3601         __atomic_release_fence();
3602         return arch_atomic64_fetch_and_relaxed(i, v);
3603 #elif defined(arch_atomic64_fetch_and)
3604         return arch_atomic64_fetch_and(i, v);
3605 #else
3606 #error "Unable to define raw_atomic64_fetch_and_release"
3607 #endif
3608 }
3609
3610 /**
3611  * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3612  * @i: s64 value
3613  * @v: pointer to atomic64_t
3614  *
3615  * Atomically updates @v to (@v & @i) with relaxed ordering.
3616  *
3617  * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3618  *
3619  * Return: The original value of @v.
3620  */
3621 static __always_inline s64
3622 raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
3623 {
3624 #if defined(arch_atomic64_fetch_and_relaxed)
3625         return arch_atomic64_fetch_and_relaxed(i, v);
3626 #elif defined(arch_atomic64_fetch_and)
3627         return arch_atomic64_fetch_and(i, v);
3628 #else
3629 #error "Unable to define raw_atomic64_fetch_and_relaxed"
3630 #endif
3631 }
3632
3633 /**
3634  * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
3635  * @i: s64 value
3636  * @v: pointer to atomic64_t
3637  *
3638  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3639  *
3640  * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3641  *
3642  * Return: Nothing.
3643  */
3644 static __always_inline void
3645 raw_atomic64_andnot(s64 i, atomic64_t *v)
3646 {
3647 #if defined(arch_atomic64_andnot)
3648         arch_atomic64_andnot(i, v);
3649 #else
3650         raw_atomic64_and(~i, v);
3651 #endif
3652 }
3653
3654 /**
3655  * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3656  * @i: s64 value
3657  * @v: pointer to atomic64_t
3658  *
3659  * Atomically updates @v to (@v & ~@i) with full ordering.
3660  *
3661  * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3662  *
3663  * Return: The original value of @v.
3664  */
3665 static __always_inline s64
3666 raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
3667 {
3668 #if defined(arch_atomic64_fetch_andnot)
3669         return arch_atomic64_fetch_andnot(i, v);
3670 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3671         s64 ret;
3672         __atomic_pre_full_fence();
3673         ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3674         __atomic_post_full_fence();
3675         return ret;
3676 #else
3677         return raw_atomic64_fetch_and(~i, v);
3678 #endif
3679 }
3680
3681 /**
3682  * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3683  * @i: s64 value
3684  * @v: pointer to atomic64_t
3685  *
3686  * Atomically updates @v to (@v & ~@i) with acquire ordering.
3687  *
3688  * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3689  *
3690  * Return: The original value of @v.
3691  */
3692 static __always_inline s64
3693 raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
3694 {
3695 #if defined(arch_atomic64_fetch_andnot_acquire)
3696         return arch_atomic64_fetch_andnot_acquire(i, v);
3697 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3698         s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3699         __atomic_acquire_fence();
3700         return ret;
3701 #elif defined(arch_atomic64_fetch_andnot)
3702         return arch_atomic64_fetch_andnot(i, v);
3703 #else
3704         return raw_atomic64_fetch_and_acquire(~i, v);
3705 #endif
3706 }
3707
3708 /**
3709  * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3710  * @i: s64 value
3711  * @v: pointer to atomic64_t
3712  *
3713  * Atomically updates @v to (@v & ~@i) with release ordering.
3714  *
3715  * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3716  *
3717  * Return: The original value of @v.
3718  */
3719 static __always_inline s64
3720 raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
3721 {
3722 #if defined(arch_atomic64_fetch_andnot_release)
3723         return arch_atomic64_fetch_andnot_release(i, v);
3724 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3725         __atomic_release_fence();
3726         return arch_atomic64_fetch_andnot_relaxed(i, v);
3727 #elif defined(arch_atomic64_fetch_andnot)
3728         return arch_atomic64_fetch_andnot(i, v);
3729 #else
3730         return raw_atomic64_fetch_and_release(~i, v);
3731 #endif
3732 }
3733
3734 /**
3735  * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
3736  * @i: s64 value
3737  * @v: pointer to atomic64_t
3738  *
3739  * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3740  *
3741  * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
3742  *
3743  * Return: The original value of @v.
3744  */
3745 static __always_inline s64
3746 raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
3747 {
3748 #if defined(arch_atomic64_fetch_andnot_relaxed)
3749         return arch_atomic64_fetch_andnot_relaxed(i, v);
3750 #elif defined(arch_atomic64_fetch_andnot)
3751         return arch_atomic64_fetch_andnot(i, v);
3752 #else
3753         return raw_atomic64_fetch_and_relaxed(~i, v);
3754 #endif
3755 }
3756
3757 /**
3758  * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
3759  * @i: s64 value
3760  * @v: pointer to atomic64_t
3761  *
3762  * Atomically updates @v to (@v | @i) with relaxed ordering.
3763  *
3764  * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
3765  *
3766  * Return: Nothing.
3767  */
3768 static __always_inline void
3769 raw_atomic64_or(s64 i, atomic64_t *v)
3770 {
3771         arch_atomic64_or(i, v);
3772 }
3773
3774 /**
3775  * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
3776  * @i: s64 value
3777  * @v: pointer to atomic64_t
3778  *
3779  * Atomically updates @v to (@v | @i) with full ordering.
3780  *
3781  * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
3782  *
3783  * Return: The original value of @v.
3784  */
3785 static __always_inline s64
3786 raw_atomic64_fetch_or(s64 i, atomic64_t *v)
3787 {
3788 #if defined(arch_atomic64_fetch_or)
3789         return arch_atomic64_fetch_or(i, v);
3790 #elif defined(arch_atomic64_fetch_or_relaxed)
3791         s64 ret;
3792         __atomic_pre_full_fence();
3793         ret = arch_atomic64_fetch_or_relaxed(i, v);
3794         __atomic_post_full_fence();
3795         return ret;
3796 #else
3797 #error "Unable to define raw_atomic64_fetch_or"
3798 #endif
3799 }
3800
3801 /**
3802  * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
3803  * @i: s64 value
3804  * @v: pointer to atomic64_t
3805  *
3806  * Atomically updates @v to (@v | @i) with acquire ordering.
3807  *
3808  * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
3809  *
3810  * Return: The original value of @v.
3811  */
3812 static __always_inline s64
3813 raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
3814 {
3815 #if defined(arch_atomic64_fetch_or_acquire)
3816         return arch_atomic64_fetch_or_acquire(i, v);
3817 #elif defined(arch_atomic64_fetch_or_relaxed)
3818         s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
3819         __atomic_acquire_fence();
3820         return ret;
3821 #elif defined(arch_atomic64_fetch_or)
3822         return arch_atomic64_fetch_or(i, v);
3823 #else
3824 #error "Unable to define raw_atomic64_fetch_or_acquire"
3825 #endif
3826 }
3827
3828 /**
3829  * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
3830  * @i: s64 value
3831  * @v: pointer to atomic64_t
3832  *
3833  * Atomically updates @v to (@v | @i) with release ordering.
3834  *
3835  * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
3836  *
3837  * Return: The original value of @v.
3838  */
3839 static __always_inline s64
3840 raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
3841 {
3842 #if defined(arch_atomic64_fetch_or_release)
3843         return arch_atomic64_fetch_or_release(i, v);
3844 #elif defined(arch_atomic64_fetch_or_relaxed)
3845         __atomic_release_fence();
3846         return arch_atomic64_fetch_or_relaxed(i, v);
3847 #elif defined(arch_atomic64_fetch_or)
3848         return arch_atomic64_fetch_or(i, v);
3849 #else
3850 #error "Unable to define raw_atomic64_fetch_or_release"
3851 #endif
3852 }
3853
3854 /**
3855  * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
3856  * @i: s64 value
3857  * @v: pointer to atomic64_t
3858  *
3859  * Atomically updates @v to (@v | @i) with relaxed ordering.
3860  *
3861  * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
3862  *
3863  * Return: The original value of @v.
3864  */
3865 static __always_inline s64
3866 raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
3867 {
3868 #if defined(arch_atomic64_fetch_or_relaxed)
3869         return arch_atomic64_fetch_or_relaxed(i, v);
3870 #elif defined(arch_atomic64_fetch_or)
3871         return arch_atomic64_fetch_or(i, v);
3872 #else
3873 #error "Unable to define raw_atomic64_fetch_or_relaxed"
3874 #endif
3875 }
3876
3877 /**
3878  * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
3879  * @i: s64 value
3880  * @v: pointer to atomic64_t
3881  *
3882  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3883  *
3884  * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
3885  *
3886  * Return: Nothing.
3887  */
3888 static __always_inline void
3889 raw_atomic64_xor(s64 i, atomic64_t *v)
3890 {
3891         arch_atomic64_xor(i, v);
3892 }
3893
3894 /**
3895  * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
3896  * @i: s64 value
3897  * @v: pointer to atomic64_t
3898  *
3899  * Atomically updates @v to (@v ^ @i) with full ordering.
3900  *
3901  * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
3902  *
3903  * Return: The original value of @v.
3904  */
3905 static __always_inline s64
3906 raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
3907 {
3908 #if defined(arch_atomic64_fetch_xor)
3909         return arch_atomic64_fetch_xor(i, v);
3910 #elif defined(arch_atomic64_fetch_xor_relaxed)
3911         s64 ret;
3912         __atomic_pre_full_fence();
3913         ret = arch_atomic64_fetch_xor_relaxed(i, v);
3914         __atomic_post_full_fence();
3915         return ret;
3916 #else
3917 #error "Unable to define raw_atomic64_fetch_xor"
3918 #endif
3919 }
3920
3921 /**
3922  * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
3923  * @i: s64 value
3924  * @v: pointer to atomic64_t
3925  *
3926  * Atomically updates @v to (@v ^ @i) with acquire ordering.
3927  *
3928  * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
3929  *
3930  * Return: The original value of @v.
3931  */
3932 static __always_inline s64
3933 raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
3934 {
3935 #if defined(arch_atomic64_fetch_xor_acquire)
3936         return arch_atomic64_fetch_xor_acquire(i, v);
3937 #elif defined(arch_atomic64_fetch_xor_relaxed)
3938         s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3939         __atomic_acquire_fence();
3940         return ret;
3941 #elif defined(arch_atomic64_fetch_xor)
3942         return arch_atomic64_fetch_xor(i, v);
3943 #else
3944 #error "Unable to define raw_atomic64_fetch_xor_acquire"
3945 #endif
3946 }
3947
3948 /**
3949  * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
3950  * @i: s64 value
3951  * @v: pointer to atomic64_t
3952  *
3953  * Atomically updates @v to (@v ^ @i) with release ordering.
3954  *
3955  * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
3956  *
3957  * Return: The original value of @v.
3958  */
3959 static __always_inline s64
3960 raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
3961 {
3962 #if defined(arch_atomic64_fetch_xor_release)
3963         return arch_atomic64_fetch_xor_release(i, v);
3964 #elif defined(arch_atomic64_fetch_xor_relaxed)
3965         __atomic_release_fence();
3966         return arch_atomic64_fetch_xor_relaxed(i, v);
3967 #elif defined(arch_atomic64_fetch_xor)
3968         return arch_atomic64_fetch_xor(i, v);
3969 #else
3970 #error "Unable to define raw_atomic64_fetch_xor_release"
3971 #endif
3972 }
3973
3974 /**
3975  * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
3976  * @i: s64 value
3977  * @v: pointer to atomic64_t
3978  *
3979  * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3980  *
3981  * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
3982  *
3983  * Return: The original value of @v.
3984  */
3985 static __always_inline s64
3986 raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
3987 {
3988 #if defined(arch_atomic64_fetch_xor_relaxed)
3989         return arch_atomic64_fetch_xor_relaxed(i, v);
3990 #elif defined(arch_atomic64_fetch_xor)
3991         return arch_atomic64_fetch_xor(i, v);
3992 #else
3993 #error "Unable to define raw_atomic64_fetch_xor_relaxed"
3994 #endif
3995 }
3996
3997 /**
3998  * raw_atomic64_xchg() - atomic exchange with full ordering
3999  * @v: pointer to atomic64_t
4000  * @new: s64 value to assign
4001  *
4002  * Atomically updates @v to @new with full ordering.
4003  *
4004  * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
4005  *
4006  * Return: The original value of @v.
4007  */
4008 static __always_inline s64
4009 raw_atomic64_xchg(atomic64_t *v, s64 new)
4010 {
4011 #if defined(arch_atomic64_xchg)
4012         return arch_atomic64_xchg(v, new);
4013 #elif defined(arch_atomic64_xchg_relaxed)
4014         s64 ret;
4015         __atomic_pre_full_fence();
4016         ret = arch_atomic64_xchg_relaxed(v, new);
4017         __atomic_post_full_fence();
4018         return ret;
4019 #else
4020         return raw_xchg(&v->counter, new);
4021 #endif
4022 }
4023
4024 /**
4025  * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
4026  * @v: pointer to atomic64_t
4027  * @new: s64 value to assign
4028  *
4029  * Atomically updates @v to @new with acquire ordering.
4030  *
4031  * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
4032  *
4033  * Return: The original value of @v.
4034  */
4035 static __always_inline s64
4036 raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
4037 {
4038 #if defined(arch_atomic64_xchg_acquire)
4039         return arch_atomic64_xchg_acquire(v, new);
4040 #elif defined(arch_atomic64_xchg_relaxed)
4041         s64 ret = arch_atomic64_xchg_relaxed(v, new);
4042         __atomic_acquire_fence();
4043         return ret;
4044 #elif defined(arch_atomic64_xchg)
4045         return arch_atomic64_xchg(v, new);
4046 #else
4047         return raw_xchg_acquire(&v->counter, new);
4048 #endif
4049 }
4050
4051 /**
4052  * raw_atomic64_xchg_release() - atomic exchange with release ordering
4053  * @v: pointer to atomic64_t
4054  * @new: s64 value to assign
4055  *
4056  * Atomically updates @v to @new with release ordering.
4057  *
4058  * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
4059  *
4060  * Return: The original value of @v.
4061  */
4062 static __always_inline s64
4063 raw_atomic64_xchg_release(atomic64_t *v, s64 new)
4064 {
4065 #if defined(arch_atomic64_xchg_release)
4066         return arch_atomic64_xchg_release(v, new);
4067 #elif defined(arch_atomic64_xchg_relaxed)
4068         __atomic_release_fence();
4069         return arch_atomic64_xchg_relaxed(v, new);
4070 #elif defined(arch_atomic64_xchg)
4071         return arch_atomic64_xchg(v, new);
4072 #else
4073         return raw_xchg_release(&v->counter, new);
4074 #endif
4075 }
4076
4077 /**
4078  * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
4079  * @v: pointer to atomic64_t
4080  * @new: s64 value to assign
4081  *
4082  * Atomically updates @v to @new with relaxed ordering.
4083  *
4084  * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
4085  *
4086  * Return: The original value of @v.
4087  */
4088 static __always_inline s64
4089 raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
4090 {
4091 #if defined(arch_atomic64_xchg_relaxed)
4092         return arch_atomic64_xchg_relaxed(v, new);
4093 #elif defined(arch_atomic64_xchg)
4094         return arch_atomic64_xchg(v, new);
4095 #else
4096         return raw_xchg_relaxed(&v->counter, new);
4097 #endif
4098 }
4099
4100 /**
4101  * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
4102  * @v: pointer to atomic64_t
4103  * @old: s64 value to compare with
4104  * @new: s64 value to assign
4105  *
4106  * If (@v == @old), atomically updates @v to @new with full ordering.
4107  *
4108  * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4109  *
4110  * Return: The original value of @v.
4111  */
4112 static __always_inline s64
4113 raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
4114 {
4115 #if defined(arch_atomic64_cmpxchg)
4116         return arch_atomic64_cmpxchg(v, old, new);
4117 #elif defined(arch_atomic64_cmpxchg_relaxed)
4118         s64 ret;
4119         __atomic_pre_full_fence();
4120         ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4121         __atomic_post_full_fence();
4122         return ret;
4123 #else
4124         return raw_cmpxchg(&v->counter, old, new);
4125 #endif
4126 }
4127
4128 /**
4129  * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4130  * @v: pointer to atomic64_t
4131  * @old: s64 value to compare with
4132  * @new: s64 value to assign
4133  *
4134  * If (@v == @old), atomically updates @v to @new with acquire ordering.
4135  *
4136  * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4137  *
4138  * Return: The original value of @v.
4139  */
4140 static __always_inline s64
4141 raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
4142 {
4143 #if defined(arch_atomic64_cmpxchg_acquire)
4144         return arch_atomic64_cmpxchg_acquire(v, old, new);
4145 #elif defined(arch_atomic64_cmpxchg_relaxed)
4146         s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4147         __atomic_acquire_fence();
4148         return ret;
4149 #elif defined(arch_atomic64_cmpxchg)
4150         return arch_atomic64_cmpxchg(v, old, new);
4151 #else
4152         return raw_cmpxchg_acquire(&v->counter, old, new);
4153 #endif
4154 }
4155
4156 /**
4157  * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
4158  * @v: pointer to atomic64_t
4159  * @old: s64 value to compare with
4160  * @new: s64 value to assign
4161  *
4162  * If (@v == @old), atomically updates @v to @new with release ordering.
4163  *
4164  * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4165  *
4166  * Return: The original value of @v.
4167  */
4168 static __always_inline s64
4169 raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
4170 {
4171 #if defined(arch_atomic64_cmpxchg_release)
4172         return arch_atomic64_cmpxchg_release(v, old, new);
4173 #elif defined(arch_atomic64_cmpxchg_relaxed)
4174         __atomic_release_fence();
4175         return arch_atomic64_cmpxchg_relaxed(v, old, new);
4176 #elif defined(arch_atomic64_cmpxchg)
4177         return arch_atomic64_cmpxchg(v, old, new);
4178 #else
4179         return raw_cmpxchg_release(&v->counter, old, new);
4180 #endif
4181 }
4182
4183 /**
4184  * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4185  * @v: pointer to atomic64_t
4186  * @old: s64 value to compare with
4187  * @new: s64 value to assign
4188  *
4189  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4190  *
4191  * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4192  *
4193  * Return: The original value of @v.
4194  */
4195 static __always_inline s64
4196 raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
4197 {
4198 #if defined(arch_atomic64_cmpxchg_relaxed)
4199         return arch_atomic64_cmpxchg_relaxed(v, old, new);
4200 #elif defined(arch_atomic64_cmpxchg)
4201         return arch_atomic64_cmpxchg(v, old, new);
4202 #else
4203         return raw_cmpxchg_relaxed(&v->counter, old, new);
4204 #endif
4205 }
4206
4207 /**
4208  * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
4209  * @v: pointer to atomic64_t
4210  * @old: pointer to s64 value to compare with
4211  * @new: s64 value to assign
4212  *
4213  * If (@v == @old), atomically updates @v to @new with full ordering.
4214  * Otherwise, updates @old to the current value of @v.
4215  *
4216  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4217  *
4218  * Return: @true if the exchange occured, @false otherwise.
4219  */
4220 static __always_inline bool
4221 raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
4222 {
4223 #if defined(arch_atomic64_try_cmpxchg)
4224         return arch_atomic64_try_cmpxchg(v, old, new);
4225 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4226         bool ret;
4227         __atomic_pre_full_fence();
4228         ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4229         __atomic_post_full_fence();
4230         return ret;
4231 #else
4232         s64 r, o = *old;
4233         r = raw_atomic64_cmpxchg(v, o, new);
4234         if (unlikely(r != o))
4235                 *old = r;
4236         return likely(r == o);
4237 #endif
4238 }
4239
4240 /**
4241  * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4242  * @v: pointer to atomic64_t
4243  * @old: pointer to s64 value to compare with
4244  * @new: s64 value to assign
4245  *
4246  * If (@v == @old), atomically updates @v to @new with acquire ordering.
4247  * Otherwise, updates @old to the current value of @v.
4248  *
4249  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4250  *
4251  * Return: @true if the exchange occured, @false otherwise.
4252  */
4253 static __always_inline bool
4254 raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
4255 {
4256 #if defined(arch_atomic64_try_cmpxchg_acquire)
4257         return arch_atomic64_try_cmpxchg_acquire(v, old, new);
4258 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4259         bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4260         __atomic_acquire_fence();
4261         return ret;
4262 #elif defined(arch_atomic64_try_cmpxchg)
4263         return arch_atomic64_try_cmpxchg(v, old, new);
4264 #else
4265         s64 r, o = *old;
4266         r = raw_atomic64_cmpxchg_acquire(v, o, new);
4267         if (unlikely(r != o))
4268                 *old = r;
4269         return likely(r == o);
4270 #endif
4271 }
4272
4273 /**
4274  * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
4275  * @v: pointer to atomic64_t
4276  * @old: pointer to s64 value to compare with
4277  * @new: s64 value to assign
4278  *
4279  * If (@v == @old), atomically updates @v to @new with release ordering.
4280  * Otherwise, updates @old to the current value of @v.
4281  *
4282  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4283  *
4284  * Return: @true if the exchange occured, @false otherwise.
4285  */
4286 static __always_inline bool
4287 raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
4288 {
4289 #if defined(arch_atomic64_try_cmpxchg_release)
4290         return arch_atomic64_try_cmpxchg_release(v, old, new);
4291 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4292         __atomic_release_fence();
4293         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4294 #elif defined(arch_atomic64_try_cmpxchg)
4295         return arch_atomic64_try_cmpxchg(v, old, new);
4296 #else
4297         s64 r, o = *old;
4298         r = raw_atomic64_cmpxchg_release(v, o, new);
4299         if (unlikely(r != o))
4300                 *old = r;
4301         return likely(r == o);
4302 #endif
4303 }
4304
4305 /**
4306  * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4307  * @v: pointer to atomic64_t
4308  * @old: pointer to s64 value to compare with
4309  * @new: s64 value to assign
4310  *
4311  * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4312  * Otherwise, updates @old to the current value of @v.
4313  *
4314  * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4315  *
4316  * Return: @true if the exchange occured, @false otherwise.
4317  */
4318 static __always_inline bool
4319 raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
4320 {
4321 #if defined(arch_atomic64_try_cmpxchg_relaxed)
4322         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4323 #elif defined(arch_atomic64_try_cmpxchg)
4324         return arch_atomic64_try_cmpxchg(v, old, new);
4325 #else
4326         s64 r, o = *old;
4327         r = raw_atomic64_cmpxchg_relaxed(v, o, new);
4328         if (unlikely(r != o))
4329                 *old = r;
4330         return likely(r == o);
4331 #endif
4332 }
4333
4334 /**
4335  * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
4336  * @i: s64 value to add
4337  * @v: pointer to atomic64_t
4338  *
4339  * Atomically updates @v to (@v - @i) with full ordering.
4340  *
4341  * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
4342  *
4343  * Return: @true if the resulting value of @v is zero, @false otherwise.
4344  */
4345 static __always_inline bool
4346 raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
4347 {
4348 #if defined(arch_atomic64_sub_and_test)
4349         return arch_atomic64_sub_and_test(i, v);
4350 #else
4351         return raw_atomic64_sub_return(i, v) == 0;
4352 #endif
4353 }
4354
4355 /**
4356  * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
4357  * @v: pointer to atomic64_t
4358  *
4359  * Atomically updates @v to (@v - 1) with full ordering.
4360  *
4361  * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
4362  *
4363  * Return: @true if the resulting value of @v is zero, @false otherwise.
4364  */
4365 static __always_inline bool
4366 raw_atomic64_dec_and_test(atomic64_t *v)
4367 {
4368 #if defined(arch_atomic64_dec_and_test)
4369         return arch_atomic64_dec_and_test(v);
4370 #else
4371         return raw_atomic64_dec_return(v) == 0;
4372 #endif
4373 }
4374
4375 /**
4376  * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
4377  * @v: pointer to atomic64_t
4378  *
4379  * Atomically updates @v to (@v + 1) with full ordering.
4380  *
4381  * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
4382  *
4383  * Return: @true if the resulting value of @v is zero, @false otherwise.
4384  */
4385 static __always_inline bool
4386 raw_atomic64_inc_and_test(atomic64_t *v)
4387 {
4388 #if defined(arch_atomic64_inc_and_test)
4389         return arch_atomic64_inc_and_test(v);
4390 #else
4391         return raw_atomic64_inc_return(v) == 0;
4392 #endif
4393 }
4394
4395 /**
4396  * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
4397  * @i: s64 value to add
4398  * @v: pointer to atomic64_t
4399  *
4400  * Atomically updates @v to (@v + @i) with full ordering.
4401  *
4402  * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
4403  *
4404  * Return: @true if the resulting value of @v is negative, @false otherwise.
4405  */
4406 static __always_inline bool
4407 raw_atomic64_add_negative(s64 i, atomic64_t *v)
4408 {
4409 #if defined(arch_atomic64_add_negative)
4410         return arch_atomic64_add_negative(i, v);
4411 #elif defined(arch_atomic64_add_negative_relaxed)
4412         bool ret;
4413         __atomic_pre_full_fence();
4414         ret = arch_atomic64_add_negative_relaxed(i, v);
4415         __atomic_post_full_fence();
4416         return ret;
4417 #else
4418         return raw_atomic64_add_return(i, v) < 0;
4419 #endif
4420 }
4421
4422 /**
4423  * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
4424  * @i: s64 value to add
4425  * @v: pointer to atomic64_t
4426  *
4427  * Atomically updates @v to (@v + @i) with acquire ordering.
4428  *
4429  * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
4430  *
4431  * Return: @true if the resulting value of @v is negative, @false otherwise.
4432  */
4433 static __always_inline bool
4434 raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
4435 {
4436 #if defined(arch_atomic64_add_negative_acquire)
4437         return arch_atomic64_add_negative_acquire(i, v);
4438 #elif defined(arch_atomic64_add_negative_relaxed)
4439         bool ret = arch_atomic64_add_negative_relaxed(i, v);
4440         __atomic_acquire_fence();
4441         return ret;
4442 #elif defined(arch_atomic64_add_negative)
4443         return arch_atomic64_add_negative(i, v);
4444 #else
4445         return raw_atomic64_add_return_acquire(i, v) < 0;
4446 #endif
4447 }
4448
4449 /**
4450  * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
4451  * @i: s64 value to add
4452  * @v: pointer to atomic64_t
4453  *
4454  * Atomically updates @v to (@v + @i) with release ordering.
4455  *
4456  * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
4457  *
4458  * Return: @true if the resulting value of @v is negative, @false otherwise.
4459  */
4460 static __always_inline bool
4461 raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
4462 {
4463 #if defined(arch_atomic64_add_negative_release)
4464         return arch_atomic64_add_negative_release(i, v);
4465 #elif defined(arch_atomic64_add_negative_relaxed)
4466         __atomic_release_fence();
4467         return arch_atomic64_add_negative_relaxed(i, v);
4468 #elif defined(arch_atomic64_add_negative)
4469         return arch_atomic64_add_negative(i, v);
4470 #else
4471         return raw_atomic64_add_return_release(i, v) < 0;
4472 #endif
4473 }
4474
4475 /**
4476  * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4477  * @i: s64 value to add
4478  * @v: pointer to atomic64_t
4479  *
4480  * Atomically updates @v to (@v + @i) with relaxed ordering.
4481  *
4482  * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
4483  *
4484  * Return: @true if the resulting value of @v is negative, @false otherwise.
4485  */
4486 static __always_inline bool
4487 raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
4488 {
4489 #if defined(arch_atomic64_add_negative_relaxed)
4490         return arch_atomic64_add_negative_relaxed(i, v);
4491 #elif defined(arch_atomic64_add_negative)
4492         return arch_atomic64_add_negative(i, v);
4493 #else
4494         return raw_atomic64_add_return_relaxed(i, v) < 0;
4495 #endif
4496 }
4497
4498 /**
4499  * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
4500  * @v: pointer to atomic64_t
4501  * @a: s64 value to add
4502  * @u: s64 value to compare with
4503  *
4504  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4505  *
4506  * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4507  *
4508  * Return: The original value of @v.
4509  */
4510 static __always_inline s64
4511 raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
4512 {
4513 #if defined(arch_atomic64_fetch_add_unless)
4514         return arch_atomic64_fetch_add_unless(v, a, u);
4515 #else
4516         s64 c = raw_atomic64_read(v);
4517
4518         do {
4519                 if (unlikely(c == u))
4520                         break;
4521         } while (!raw_atomic64_try_cmpxchg(v, &c, c + a));
4522
4523         return c;
4524 #endif
4525 }
4526
4527 /**
4528  * raw_atomic64_add_unless() - atomic add unless value with full ordering
4529  * @v: pointer to atomic64_t
4530  * @a: s64 value to add
4531  * @u: s64 value to compare with
4532  *
4533  * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4534  *
4535  * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4536  *
4537  * Return: @true if @v was updated, @false otherwise.
4538  */
4539 static __always_inline bool
4540 raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
4541 {
4542 #if defined(arch_atomic64_add_unless)
4543         return arch_atomic64_add_unless(v, a, u);
4544 #else
4545         return raw_atomic64_fetch_add_unless(v, a, u) != u;
4546 #endif
4547 }
4548
4549 /**
4550  * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
4551  * @v: pointer to atomic64_t
4552  *
4553  * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4554  *
4555  * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4556  *
4557  * Return: @true if @v was updated, @false otherwise.
4558  */
4559 static __always_inline bool
4560 raw_atomic64_inc_not_zero(atomic64_t *v)
4561 {
4562 #if defined(arch_atomic64_inc_not_zero)
4563         return arch_atomic64_inc_not_zero(v);
4564 #else
4565         return raw_atomic64_add_unless(v, 1, 0);
4566 #endif
4567 }
4568
4569 /**
4570  * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
4571  * @v: pointer to atomic64_t
4572  *
4573  * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4574  *
4575  * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4576  *
4577  * Return: @true if @v was updated, @false otherwise.
4578  */
4579 static __always_inline bool
4580 raw_atomic64_inc_unless_negative(atomic64_t *v)
4581 {
4582 #if defined(arch_atomic64_inc_unless_negative)
4583         return arch_atomic64_inc_unless_negative(v);
4584 #else
4585         s64 c = raw_atomic64_read(v);
4586
4587         do {
4588                 if (unlikely(c < 0))
4589                         return false;
4590         } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));
4591
4592         return true;
4593 #endif
4594 }
4595
4596 /**
4597  * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
4598  * @v: pointer to atomic64_t
4599  *
4600  * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4601  *
4602  * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4603  *
4604  * Return: @true if @v was updated, @false otherwise.
4605  */
4606 static __always_inline bool
4607 raw_atomic64_dec_unless_positive(atomic64_t *v)
4608 {
4609 #if defined(arch_atomic64_dec_unless_positive)
4610         return arch_atomic64_dec_unless_positive(v);
4611 #else
4612         s64 c = raw_atomic64_read(v);
4613
4614         do {
4615                 if (unlikely(c > 0))
4616                         return false;
4617         } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));
4618
4619         return true;
4620 #endif
4621 }
4622
4623 /**
4624  * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
4625  * @v: pointer to atomic64_t
4626  *
4627  * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4628  *
4629  * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4630  *
4631  * Return: The old value of (@v - 1), regardless of whether @v was updated.
4632  */
4633 static __always_inline s64
4634 raw_atomic64_dec_if_positive(atomic64_t *v)
4635 {
4636 #if defined(arch_atomic64_dec_if_positive)
4637         return arch_atomic64_dec_if_positive(v);
4638 #else
4639         s64 dec, c = raw_atomic64_read(v);
4640
4641         do {
4642                 dec = c - 1;
4643                 if (unlikely(dec < 0))
4644                         break;
4645         } while (!raw_atomic64_try_cmpxchg(v, &c, dec));
4646
4647         return dec;
4648 #endif
4649 }
4650
4651 #endif /* _LINUX_ATOMIC_FALLBACK_H */
4652 // 2fdd6702823fa842f9cea57a002e6e4476ae780c