1 // SPDX-License-Identifier: GPL-2.0
3 // Generated by scripts/atomic/gen-atomic-fallback.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
6 #ifndef _LINUX_ATOMIC_FALLBACK_H
7 #define _LINUX_ATOMIC_FALLBACK_H
9 #include <linux/compiler.h>
11 #if defined(arch_xchg)
12 #define raw_xchg arch_xchg
13 #elif defined(arch_xchg_relaxed)
14 #define raw_xchg(...) \
15 __atomic_op_fence(arch_xchg, __VA_ARGS__)
17 extern void raw_xchg_not_implemented(void);
18 #define raw_xchg(...) raw_xchg_not_implemented()
21 #if defined(arch_xchg_acquire)
22 #define raw_xchg_acquire arch_xchg_acquire
23 #elif defined(arch_xchg_relaxed)
24 #define raw_xchg_acquire(...) \
25 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
26 #elif defined(arch_xchg)
27 #define raw_xchg_acquire arch_xchg
29 extern void raw_xchg_acquire_not_implemented(void);
30 #define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
33 #if defined(arch_xchg_release)
34 #define raw_xchg_release arch_xchg_release
35 #elif defined(arch_xchg_relaxed)
36 #define raw_xchg_release(...) \
37 __atomic_op_release(arch_xchg, __VA_ARGS__)
38 #elif defined(arch_xchg)
39 #define raw_xchg_release arch_xchg
41 extern void raw_xchg_release_not_implemented(void);
42 #define raw_xchg_release(...) raw_xchg_release_not_implemented()
45 #if defined(arch_xchg_relaxed)
46 #define raw_xchg_relaxed arch_xchg_relaxed
47 #elif defined(arch_xchg)
48 #define raw_xchg_relaxed arch_xchg
50 extern void raw_xchg_relaxed_not_implemented(void);
51 #define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
54 #if defined(arch_cmpxchg)
55 #define raw_cmpxchg arch_cmpxchg
56 #elif defined(arch_cmpxchg_relaxed)
57 #define raw_cmpxchg(...) \
58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
60 extern void raw_cmpxchg_not_implemented(void);
61 #define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
64 #if defined(arch_cmpxchg_acquire)
65 #define raw_cmpxchg_acquire arch_cmpxchg_acquire
66 #elif defined(arch_cmpxchg_relaxed)
67 #define raw_cmpxchg_acquire(...) \
68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
69 #elif defined(arch_cmpxchg)
70 #define raw_cmpxchg_acquire arch_cmpxchg
72 extern void raw_cmpxchg_acquire_not_implemented(void);
73 #define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
76 #if defined(arch_cmpxchg_release)
77 #define raw_cmpxchg_release arch_cmpxchg_release
78 #elif defined(arch_cmpxchg_relaxed)
79 #define raw_cmpxchg_release(...) \
80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
81 #elif defined(arch_cmpxchg)
82 #define raw_cmpxchg_release arch_cmpxchg
84 extern void raw_cmpxchg_release_not_implemented(void);
85 #define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
88 #if defined(arch_cmpxchg_relaxed)
89 #define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
90 #elif defined(arch_cmpxchg)
91 #define raw_cmpxchg_relaxed arch_cmpxchg
93 extern void raw_cmpxchg_relaxed_not_implemented(void);
94 #define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
97 #if defined(arch_cmpxchg64)
98 #define raw_cmpxchg64 arch_cmpxchg64
99 #elif defined(arch_cmpxchg64_relaxed)
100 #define raw_cmpxchg64(...) \
101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
103 extern void raw_cmpxchg64_not_implemented(void);
104 #define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
107 #if defined(arch_cmpxchg64_acquire)
108 #define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
109 #elif defined(arch_cmpxchg64_relaxed)
110 #define raw_cmpxchg64_acquire(...) \
111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
112 #elif defined(arch_cmpxchg64)
113 #define raw_cmpxchg64_acquire arch_cmpxchg64
115 extern void raw_cmpxchg64_acquire_not_implemented(void);
116 #define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
119 #if defined(arch_cmpxchg64_release)
120 #define raw_cmpxchg64_release arch_cmpxchg64_release
121 #elif defined(arch_cmpxchg64_relaxed)
122 #define raw_cmpxchg64_release(...) \
123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
124 #elif defined(arch_cmpxchg64)
125 #define raw_cmpxchg64_release arch_cmpxchg64
127 extern void raw_cmpxchg64_release_not_implemented(void);
128 #define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
131 #if defined(arch_cmpxchg64_relaxed)
132 #define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
133 #elif defined(arch_cmpxchg64)
134 #define raw_cmpxchg64_relaxed arch_cmpxchg64
136 extern void raw_cmpxchg64_relaxed_not_implemented(void);
137 #define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
140 #if defined(arch_cmpxchg128)
141 #define raw_cmpxchg128 arch_cmpxchg128
142 #elif defined(arch_cmpxchg128_relaxed)
143 #define raw_cmpxchg128(...) \
144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
146 extern void raw_cmpxchg128_not_implemented(void);
147 #define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
150 #if defined(arch_cmpxchg128_acquire)
151 #define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
152 #elif defined(arch_cmpxchg128_relaxed)
153 #define raw_cmpxchg128_acquire(...) \
154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
155 #elif defined(arch_cmpxchg128)
156 #define raw_cmpxchg128_acquire arch_cmpxchg128
158 extern void raw_cmpxchg128_acquire_not_implemented(void);
159 #define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
162 #if defined(arch_cmpxchg128_release)
163 #define raw_cmpxchg128_release arch_cmpxchg128_release
164 #elif defined(arch_cmpxchg128_relaxed)
165 #define raw_cmpxchg128_release(...) \
166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
167 #elif defined(arch_cmpxchg128)
168 #define raw_cmpxchg128_release arch_cmpxchg128
170 extern void raw_cmpxchg128_release_not_implemented(void);
171 #define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
174 #if defined(arch_cmpxchg128_relaxed)
175 #define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
176 #elif defined(arch_cmpxchg128)
177 #define raw_cmpxchg128_relaxed arch_cmpxchg128
179 extern void raw_cmpxchg128_relaxed_not_implemented(void);
180 #define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
183 #if defined(arch_try_cmpxchg)
184 #define raw_try_cmpxchg arch_try_cmpxchg
185 #elif defined(arch_try_cmpxchg_relaxed)
186 #define raw_try_cmpxchg(...) \
187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
189 #define raw_try_cmpxchg(_ptr, _oldp, _new) \
191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
193 if (unlikely(___r != ___o)) \
195 likely(___r == ___o); \
199 #if defined(arch_try_cmpxchg_acquire)
200 #define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
201 #elif defined(arch_try_cmpxchg_relaxed)
202 #define raw_try_cmpxchg_acquire(...) \
203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
204 #elif defined(arch_try_cmpxchg)
205 #define raw_try_cmpxchg_acquire arch_try_cmpxchg
207 #define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
211 if (unlikely(___r != ___o)) \
213 likely(___r == ___o); \
217 #if defined(arch_try_cmpxchg_release)
218 #define raw_try_cmpxchg_release arch_try_cmpxchg_release
219 #elif defined(arch_try_cmpxchg_relaxed)
220 #define raw_try_cmpxchg_release(...) \
221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
222 #elif defined(arch_try_cmpxchg)
223 #define raw_try_cmpxchg_release arch_try_cmpxchg
225 #define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
229 if (unlikely(___r != ___o)) \
231 likely(___r == ___o); \
235 #if defined(arch_try_cmpxchg_relaxed)
236 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
237 #elif defined(arch_try_cmpxchg)
238 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg
240 #define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
244 if (unlikely(___r != ___o)) \
246 likely(___r == ___o); \
250 #if defined(arch_try_cmpxchg64)
251 #define raw_try_cmpxchg64 arch_try_cmpxchg64
252 #elif defined(arch_try_cmpxchg64_relaxed)
253 #define raw_try_cmpxchg64(...) \
254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
256 #define raw_try_cmpxchg64(_ptr, _oldp, _new) \
258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
260 if (unlikely(___r != ___o)) \
262 likely(___r == ___o); \
266 #if defined(arch_try_cmpxchg64_acquire)
267 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
268 #elif defined(arch_try_cmpxchg64_relaxed)
269 #define raw_try_cmpxchg64_acquire(...) \
270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
271 #elif defined(arch_try_cmpxchg64)
272 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
274 #define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
278 if (unlikely(___r != ___o)) \
280 likely(___r == ___o); \
284 #if defined(arch_try_cmpxchg64_release)
285 #define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
286 #elif defined(arch_try_cmpxchg64_relaxed)
287 #define raw_try_cmpxchg64_release(...) \
288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
289 #elif defined(arch_try_cmpxchg64)
290 #define raw_try_cmpxchg64_release arch_try_cmpxchg64
292 #define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
296 if (unlikely(___r != ___o)) \
298 likely(___r == ___o); \
302 #if defined(arch_try_cmpxchg64_relaxed)
303 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
304 #elif defined(arch_try_cmpxchg64)
305 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
307 #define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
311 if (unlikely(___r != ___o)) \
313 likely(___r == ___o); \
317 #if defined(arch_try_cmpxchg128)
318 #define raw_try_cmpxchg128 arch_try_cmpxchg128
319 #elif defined(arch_try_cmpxchg128_relaxed)
320 #define raw_try_cmpxchg128(...) \
321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
323 #define raw_try_cmpxchg128(_ptr, _oldp, _new) \
325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
327 if (unlikely(___r != ___o)) \
329 likely(___r == ___o); \
333 #if defined(arch_try_cmpxchg128_acquire)
334 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
335 #elif defined(arch_try_cmpxchg128_relaxed)
336 #define raw_try_cmpxchg128_acquire(...) \
337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
338 #elif defined(arch_try_cmpxchg128)
339 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
341 #define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
345 if (unlikely(___r != ___o)) \
347 likely(___r == ___o); \
351 #if defined(arch_try_cmpxchg128_release)
352 #define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
353 #elif defined(arch_try_cmpxchg128_relaxed)
354 #define raw_try_cmpxchg128_release(...) \
355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
356 #elif defined(arch_try_cmpxchg128)
357 #define raw_try_cmpxchg128_release arch_try_cmpxchg128
359 #define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
363 if (unlikely(___r != ___o)) \
365 likely(___r == ___o); \
369 #if defined(arch_try_cmpxchg128_relaxed)
370 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
371 #elif defined(arch_try_cmpxchg128)
372 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
374 #define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
378 if (unlikely(___r != ___o)) \
380 likely(___r == ___o); \
384 #define raw_cmpxchg_local arch_cmpxchg_local
386 #ifdef arch_try_cmpxchg_local
387 #define raw_try_cmpxchg_local arch_try_cmpxchg_local
389 #define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
393 if (unlikely(___r != ___o)) \
395 likely(___r == ___o); \
399 #define raw_cmpxchg64_local arch_cmpxchg64_local
401 #ifdef arch_try_cmpxchg64_local
402 #define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
404 #define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
408 if (unlikely(___r != ___o)) \
410 likely(___r == ___o); \
414 #define raw_cmpxchg128_local arch_cmpxchg128_local
416 #ifdef arch_try_cmpxchg128_local
417 #define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
419 #define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
423 if (unlikely(___r != ___o)) \
425 likely(___r == ___o); \
429 #define raw_sync_cmpxchg arch_sync_cmpxchg
432 * raw_atomic_read() - atomic load with relaxed ordering
433 * @v: pointer to atomic_t
435 * Atomically loads the value of @v with relaxed ordering.
437 * Safe to use in noinstr code; prefer atomic_read() elsewhere.
439 * Return: The value loaded from @v.
441 static __always_inline int
442 raw_atomic_read(const atomic_t *v)
444 return arch_atomic_read(v);
448 * raw_atomic_read_acquire() - atomic load with acquire ordering
449 * @v: pointer to atomic_t
451 * Atomically loads the value of @v with acquire ordering.
453 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
455 * Return: The value loaded from @v.
457 static __always_inline int
458 raw_atomic_read_acquire(const atomic_t *v)
460 #if defined(arch_atomic_read_acquire)
461 return arch_atomic_read_acquire(v);
465 if (__native_word(atomic_t)) {
466 ret = smp_load_acquire(&(v)->counter);
468 ret = raw_atomic_read(v);
469 __atomic_acquire_fence();
477 * raw_atomic_set() - atomic set with relaxed ordering
478 * @v: pointer to atomic_t
479 * @i: int value to assign
481 * Atomically sets @v to @i with relaxed ordering.
483 * Safe to use in noinstr code; prefer atomic_set() elsewhere.
487 static __always_inline void
488 raw_atomic_set(atomic_t *v, int i)
490 arch_atomic_set(v, i);
494 * raw_atomic_set_release() - atomic set with release ordering
495 * @v: pointer to atomic_t
496 * @i: int value to assign
498 * Atomically sets @v to @i with release ordering.
500 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
504 static __always_inline void
505 raw_atomic_set_release(atomic_t *v, int i)
507 #if defined(arch_atomic_set_release)
508 arch_atomic_set_release(v, i);
510 if (__native_word(atomic_t)) {
511 smp_store_release(&(v)->counter, i);
513 __atomic_release_fence();
514 raw_atomic_set(v, i);
520 * raw_atomic_add() - atomic add with relaxed ordering
521 * @i: int value to add
522 * @v: pointer to atomic_t
524 * Atomically updates @v to (@v + @i) with relaxed ordering.
526 * Safe to use in noinstr code; prefer atomic_add() elsewhere.
530 static __always_inline void
531 raw_atomic_add(int i, atomic_t *v)
533 arch_atomic_add(i, v);
537 * raw_atomic_add_return() - atomic add with full ordering
538 * @i: int value to add
539 * @v: pointer to atomic_t
541 * Atomically updates @v to (@v + @i) with full ordering.
543 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
545 * Return: The updated value of @v.
547 static __always_inline int
548 raw_atomic_add_return(int i, atomic_t *v)
550 #if defined(arch_atomic_add_return)
551 return arch_atomic_add_return(i, v);
552 #elif defined(arch_atomic_add_return_relaxed)
554 __atomic_pre_full_fence();
555 ret = arch_atomic_add_return_relaxed(i, v);
556 __atomic_post_full_fence();
559 #error "Unable to define raw_atomic_add_return"
564 * raw_atomic_add_return_acquire() - atomic add with acquire ordering
565 * @i: int value to add
566 * @v: pointer to atomic_t
568 * Atomically updates @v to (@v + @i) with acquire ordering.
570 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
572 * Return: The updated value of @v.
574 static __always_inline int
575 raw_atomic_add_return_acquire(int i, atomic_t *v)
577 #if defined(arch_atomic_add_return_acquire)
578 return arch_atomic_add_return_acquire(i, v);
579 #elif defined(arch_atomic_add_return_relaxed)
580 int ret = arch_atomic_add_return_relaxed(i, v);
581 __atomic_acquire_fence();
583 #elif defined(arch_atomic_add_return)
584 return arch_atomic_add_return(i, v);
586 #error "Unable to define raw_atomic_add_return_acquire"
591 * raw_atomic_add_return_release() - atomic add with release ordering
592 * @i: int value to add
593 * @v: pointer to atomic_t
595 * Atomically updates @v to (@v + @i) with release ordering.
597 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
599 * Return: The updated value of @v.
601 static __always_inline int
602 raw_atomic_add_return_release(int i, atomic_t *v)
604 #if defined(arch_atomic_add_return_release)
605 return arch_atomic_add_return_release(i, v);
606 #elif defined(arch_atomic_add_return_relaxed)
607 __atomic_release_fence();
608 return arch_atomic_add_return_relaxed(i, v);
609 #elif defined(arch_atomic_add_return)
610 return arch_atomic_add_return(i, v);
612 #error "Unable to define raw_atomic_add_return_release"
617 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
618 * @i: int value to add
619 * @v: pointer to atomic_t
621 * Atomically updates @v to (@v + @i) with relaxed ordering.
623 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
625 * Return: The updated value of @v.
627 static __always_inline int
628 raw_atomic_add_return_relaxed(int i, atomic_t *v)
630 #if defined(arch_atomic_add_return_relaxed)
631 return arch_atomic_add_return_relaxed(i, v);
632 #elif defined(arch_atomic_add_return)
633 return arch_atomic_add_return(i, v);
635 #error "Unable to define raw_atomic_add_return_relaxed"
640 * raw_atomic_fetch_add() - atomic add with full ordering
641 * @i: int value to add
642 * @v: pointer to atomic_t
644 * Atomically updates @v to (@v + @i) with full ordering.
646 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
648 * Return: The original value of @v.
650 static __always_inline int
651 raw_atomic_fetch_add(int i, atomic_t *v)
653 #if defined(arch_atomic_fetch_add)
654 return arch_atomic_fetch_add(i, v);
655 #elif defined(arch_atomic_fetch_add_relaxed)
657 __atomic_pre_full_fence();
658 ret = arch_atomic_fetch_add_relaxed(i, v);
659 __atomic_post_full_fence();
662 #error "Unable to define raw_atomic_fetch_add"
667 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
668 * @i: int value to add
669 * @v: pointer to atomic_t
671 * Atomically updates @v to (@v + @i) with acquire ordering.
673 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
675 * Return: The original value of @v.
677 static __always_inline int
678 raw_atomic_fetch_add_acquire(int i, atomic_t *v)
680 #if defined(arch_atomic_fetch_add_acquire)
681 return arch_atomic_fetch_add_acquire(i, v);
682 #elif defined(arch_atomic_fetch_add_relaxed)
683 int ret = arch_atomic_fetch_add_relaxed(i, v);
684 __atomic_acquire_fence();
686 #elif defined(arch_atomic_fetch_add)
687 return arch_atomic_fetch_add(i, v);
689 #error "Unable to define raw_atomic_fetch_add_acquire"
694 * raw_atomic_fetch_add_release() - atomic add with release ordering
695 * @i: int value to add
696 * @v: pointer to atomic_t
698 * Atomically updates @v to (@v + @i) with release ordering.
700 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
702 * Return: The original value of @v.
704 static __always_inline int
705 raw_atomic_fetch_add_release(int i, atomic_t *v)
707 #if defined(arch_atomic_fetch_add_release)
708 return arch_atomic_fetch_add_release(i, v);
709 #elif defined(arch_atomic_fetch_add_relaxed)
710 __atomic_release_fence();
711 return arch_atomic_fetch_add_relaxed(i, v);
712 #elif defined(arch_atomic_fetch_add)
713 return arch_atomic_fetch_add(i, v);
715 #error "Unable to define raw_atomic_fetch_add_release"
720 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
721 * @i: int value to add
722 * @v: pointer to atomic_t
724 * Atomically updates @v to (@v + @i) with relaxed ordering.
726 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
728 * Return: The original value of @v.
730 static __always_inline int
731 raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
733 #if defined(arch_atomic_fetch_add_relaxed)
734 return arch_atomic_fetch_add_relaxed(i, v);
735 #elif defined(arch_atomic_fetch_add)
736 return arch_atomic_fetch_add(i, v);
738 #error "Unable to define raw_atomic_fetch_add_relaxed"
743 * raw_atomic_sub() - atomic subtract with relaxed ordering
744 * @i: int value to subtract
745 * @v: pointer to atomic_t
747 * Atomically updates @v to (@v - @i) with relaxed ordering.
749 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
753 static __always_inline void
754 raw_atomic_sub(int i, atomic_t *v)
756 arch_atomic_sub(i, v);
760 * raw_atomic_sub_return() - atomic subtract with full ordering
761 * @i: int value to subtract
762 * @v: pointer to atomic_t
764 * Atomically updates @v to (@v - @i) with full ordering.
766 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
768 * Return: The updated value of @v.
770 static __always_inline int
771 raw_atomic_sub_return(int i, atomic_t *v)
773 #if defined(arch_atomic_sub_return)
774 return arch_atomic_sub_return(i, v);
775 #elif defined(arch_atomic_sub_return_relaxed)
777 __atomic_pre_full_fence();
778 ret = arch_atomic_sub_return_relaxed(i, v);
779 __atomic_post_full_fence();
782 #error "Unable to define raw_atomic_sub_return"
787 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
788 * @i: int value to subtract
789 * @v: pointer to atomic_t
791 * Atomically updates @v to (@v - @i) with acquire ordering.
793 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
795 * Return: The updated value of @v.
797 static __always_inline int
798 raw_atomic_sub_return_acquire(int i, atomic_t *v)
800 #if defined(arch_atomic_sub_return_acquire)
801 return arch_atomic_sub_return_acquire(i, v);
802 #elif defined(arch_atomic_sub_return_relaxed)
803 int ret = arch_atomic_sub_return_relaxed(i, v);
804 __atomic_acquire_fence();
806 #elif defined(arch_atomic_sub_return)
807 return arch_atomic_sub_return(i, v);
809 #error "Unable to define raw_atomic_sub_return_acquire"
814 * raw_atomic_sub_return_release() - atomic subtract with release ordering
815 * @i: int value to subtract
816 * @v: pointer to atomic_t
818 * Atomically updates @v to (@v - @i) with release ordering.
820 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
822 * Return: The updated value of @v.
824 static __always_inline int
825 raw_atomic_sub_return_release(int i, atomic_t *v)
827 #if defined(arch_atomic_sub_return_release)
828 return arch_atomic_sub_return_release(i, v);
829 #elif defined(arch_atomic_sub_return_relaxed)
830 __atomic_release_fence();
831 return arch_atomic_sub_return_relaxed(i, v);
832 #elif defined(arch_atomic_sub_return)
833 return arch_atomic_sub_return(i, v);
835 #error "Unable to define raw_atomic_sub_return_release"
840 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
841 * @i: int value to subtract
842 * @v: pointer to atomic_t
844 * Atomically updates @v to (@v - @i) with relaxed ordering.
846 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
848 * Return: The updated value of @v.
850 static __always_inline int
851 raw_atomic_sub_return_relaxed(int i, atomic_t *v)
853 #if defined(arch_atomic_sub_return_relaxed)
854 return arch_atomic_sub_return_relaxed(i, v);
855 #elif defined(arch_atomic_sub_return)
856 return arch_atomic_sub_return(i, v);
858 #error "Unable to define raw_atomic_sub_return_relaxed"
863 * raw_atomic_fetch_sub() - atomic subtract with full ordering
864 * @i: int value to subtract
865 * @v: pointer to atomic_t
867 * Atomically updates @v to (@v - @i) with full ordering.
869 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
871 * Return: The original value of @v.
873 static __always_inline int
874 raw_atomic_fetch_sub(int i, atomic_t *v)
876 #if defined(arch_atomic_fetch_sub)
877 return arch_atomic_fetch_sub(i, v);
878 #elif defined(arch_atomic_fetch_sub_relaxed)
880 __atomic_pre_full_fence();
881 ret = arch_atomic_fetch_sub_relaxed(i, v);
882 __atomic_post_full_fence();
885 #error "Unable to define raw_atomic_fetch_sub"
890 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
891 * @i: int value to subtract
892 * @v: pointer to atomic_t
894 * Atomically updates @v to (@v - @i) with acquire ordering.
896 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
898 * Return: The original value of @v.
900 static __always_inline int
901 raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
903 #if defined(arch_atomic_fetch_sub_acquire)
904 return arch_atomic_fetch_sub_acquire(i, v);
905 #elif defined(arch_atomic_fetch_sub_relaxed)
906 int ret = arch_atomic_fetch_sub_relaxed(i, v);
907 __atomic_acquire_fence();
909 #elif defined(arch_atomic_fetch_sub)
910 return arch_atomic_fetch_sub(i, v);
912 #error "Unable to define raw_atomic_fetch_sub_acquire"
917 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
918 * @i: int value to subtract
919 * @v: pointer to atomic_t
921 * Atomically updates @v to (@v - @i) with release ordering.
923 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
925 * Return: The original value of @v.
927 static __always_inline int
928 raw_atomic_fetch_sub_release(int i, atomic_t *v)
930 #if defined(arch_atomic_fetch_sub_release)
931 return arch_atomic_fetch_sub_release(i, v);
932 #elif defined(arch_atomic_fetch_sub_relaxed)
933 __atomic_release_fence();
934 return arch_atomic_fetch_sub_relaxed(i, v);
935 #elif defined(arch_atomic_fetch_sub)
936 return arch_atomic_fetch_sub(i, v);
938 #error "Unable to define raw_atomic_fetch_sub_release"
943 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
944 * @i: int value to subtract
945 * @v: pointer to atomic_t
947 * Atomically updates @v to (@v - @i) with relaxed ordering.
949 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
951 * Return: The original value of @v.
953 static __always_inline int
954 raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
956 #if defined(arch_atomic_fetch_sub_relaxed)
957 return arch_atomic_fetch_sub_relaxed(i, v);
958 #elif defined(arch_atomic_fetch_sub)
959 return arch_atomic_fetch_sub(i, v);
961 #error "Unable to define raw_atomic_fetch_sub_relaxed"
966 * raw_atomic_inc() - atomic increment with relaxed ordering
967 * @v: pointer to atomic_t
969 * Atomically updates @v to (@v + 1) with relaxed ordering.
971 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
975 static __always_inline void
976 raw_atomic_inc(atomic_t *v)
978 #if defined(arch_atomic_inc)
981 raw_atomic_add(1, v);
986 * raw_atomic_inc_return() - atomic increment with full ordering
987 * @v: pointer to atomic_t
989 * Atomically updates @v to (@v + 1) with full ordering.
991 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
993 * Return: The updated value of @v.
995 static __always_inline int
996 raw_atomic_inc_return(atomic_t *v)
998 #if defined(arch_atomic_inc_return)
999 return arch_atomic_inc_return(v);
1000 #elif defined(arch_atomic_inc_return_relaxed)
1002 __atomic_pre_full_fence();
1003 ret = arch_atomic_inc_return_relaxed(v);
1004 __atomic_post_full_fence();
1007 return raw_atomic_add_return(1, v);
1012 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
1013 * @v: pointer to atomic_t
1015 * Atomically updates @v to (@v + 1) with acquire ordering.
1017 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1019 * Return: The updated value of @v.
1021 static __always_inline int
1022 raw_atomic_inc_return_acquire(atomic_t *v)
1024 #if defined(arch_atomic_inc_return_acquire)
1025 return arch_atomic_inc_return_acquire(v);
1026 #elif defined(arch_atomic_inc_return_relaxed)
1027 int ret = arch_atomic_inc_return_relaxed(v);
1028 __atomic_acquire_fence();
1030 #elif defined(arch_atomic_inc_return)
1031 return arch_atomic_inc_return(v);
1033 return raw_atomic_add_return_acquire(1, v);
1038 * raw_atomic_inc_return_release() - atomic increment with release ordering
1039 * @v: pointer to atomic_t
1041 * Atomically updates @v to (@v + 1) with release ordering.
1043 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1045 * Return: The updated value of @v.
1047 static __always_inline int
1048 raw_atomic_inc_return_release(atomic_t *v)
1050 #if defined(arch_atomic_inc_return_release)
1051 return arch_atomic_inc_return_release(v);
1052 #elif defined(arch_atomic_inc_return_relaxed)
1053 __atomic_release_fence();
1054 return arch_atomic_inc_return_relaxed(v);
1055 #elif defined(arch_atomic_inc_return)
1056 return arch_atomic_inc_return(v);
1058 return raw_atomic_add_return_release(1, v);
1063 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
1064 * @v: pointer to atomic_t
1066 * Atomically updates @v to (@v + 1) with relaxed ordering.
1068 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
1070 * Return: The updated value of @v.
1072 static __always_inline int
1073 raw_atomic_inc_return_relaxed(atomic_t *v)
1075 #if defined(arch_atomic_inc_return_relaxed)
1076 return arch_atomic_inc_return_relaxed(v);
1077 #elif defined(arch_atomic_inc_return)
1078 return arch_atomic_inc_return(v);
1080 return raw_atomic_add_return_relaxed(1, v);
1085 * raw_atomic_fetch_inc() - atomic increment with full ordering
1086 * @v: pointer to atomic_t
1088 * Atomically updates @v to (@v + 1) with full ordering.
1090 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1092 * Return: The original value of @v.
1094 static __always_inline int
1095 raw_atomic_fetch_inc(atomic_t *v)
1097 #if defined(arch_atomic_fetch_inc)
1098 return arch_atomic_fetch_inc(v);
1099 #elif defined(arch_atomic_fetch_inc_relaxed)
1101 __atomic_pre_full_fence();
1102 ret = arch_atomic_fetch_inc_relaxed(v);
1103 __atomic_post_full_fence();
1106 return raw_atomic_fetch_add(1, v);
1111 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
1112 * @v: pointer to atomic_t
1114 * Atomically updates @v to (@v + 1) with acquire ordering.
1116 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1118 * Return: The original value of @v.
1120 static __always_inline int
1121 raw_atomic_fetch_inc_acquire(atomic_t *v)
1123 #if defined(arch_atomic_fetch_inc_acquire)
1124 return arch_atomic_fetch_inc_acquire(v);
1125 #elif defined(arch_atomic_fetch_inc_relaxed)
1126 int ret = arch_atomic_fetch_inc_relaxed(v);
1127 __atomic_acquire_fence();
1129 #elif defined(arch_atomic_fetch_inc)
1130 return arch_atomic_fetch_inc(v);
1132 return raw_atomic_fetch_add_acquire(1, v);
1137 * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1138 * @v: pointer to atomic_t
1140 * Atomically updates @v to (@v + 1) with release ordering.
1142 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1144 * Return: The original value of @v.
1146 static __always_inline int
1147 raw_atomic_fetch_inc_release(atomic_t *v)
1149 #if defined(arch_atomic_fetch_inc_release)
1150 return arch_atomic_fetch_inc_release(v);
1151 #elif defined(arch_atomic_fetch_inc_relaxed)
1152 __atomic_release_fence();
1153 return arch_atomic_fetch_inc_relaxed(v);
1154 #elif defined(arch_atomic_fetch_inc)
1155 return arch_atomic_fetch_inc(v);
1157 return raw_atomic_fetch_add_release(1, v);
1162 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1163 * @v: pointer to atomic_t
1165 * Atomically updates @v to (@v + 1) with relaxed ordering.
1167 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1169 * Return: The original value of @v.
1171 static __always_inline int
1172 raw_atomic_fetch_inc_relaxed(atomic_t *v)
1174 #if defined(arch_atomic_fetch_inc_relaxed)
1175 return arch_atomic_fetch_inc_relaxed(v);
1176 #elif defined(arch_atomic_fetch_inc)
1177 return arch_atomic_fetch_inc(v);
1179 return raw_atomic_fetch_add_relaxed(1, v);
1184 * raw_atomic_dec() - atomic decrement with relaxed ordering
1185 * @v: pointer to atomic_t
1187 * Atomically updates @v to (@v - 1) with relaxed ordering.
1189 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1193 static __always_inline void
1194 raw_atomic_dec(atomic_t *v)
1196 #if defined(arch_atomic_dec)
1199 raw_atomic_sub(1, v);
1204 * raw_atomic_dec_return() - atomic decrement with full ordering
1205 * @v: pointer to atomic_t
1207 * Atomically updates @v to (@v - 1) with full ordering.
1209 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1211 * Return: The updated value of @v.
1213 static __always_inline int
1214 raw_atomic_dec_return(atomic_t *v)
1216 #if defined(arch_atomic_dec_return)
1217 return arch_atomic_dec_return(v);
1218 #elif defined(arch_atomic_dec_return_relaxed)
1220 __atomic_pre_full_fence();
1221 ret = arch_atomic_dec_return_relaxed(v);
1222 __atomic_post_full_fence();
1225 return raw_atomic_sub_return(1, v);
1230 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
1231 * @v: pointer to atomic_t
1233 * Atomically updates @v to (@v - 1) with acquire ordering.
1235 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1237 * Return: The updated value of @v.
1239 static __always_inline int
1240 raw_atomic_dec_return_acquire(atomic_t *v)
1242 #if defined(arch_atomic_dec_return_acquire)
1243 return arch_atomic_dec_return_acquire(v);
1244 #elif defined(arch_atomic_dec_return_relaxed)
1245 int ret = arch_atomic_dec_return_relaxed(v);
1246 __atomic_acquire_fence();
1248 #elif defined(arch_atomic_dec_return)
1249 return arch_atomic_dec_return(v);
1251 return raw_atomic_sub_return_acquire(1, v);
1256 * raw_atomic_dec_return_release() - atomic decrement with release ordering
1257 * @v: pointer to atomic_t
1259 * Atomically updates @v to (@v - 1) with release ordering.
1261 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1263 * Return: The updated value of @v.
1265 static __always_inline int
1266 raw_atomic_dec_return_release(atomic_t *v)
1268 #if defined(arch_atomic_dec_return_release)
1269 return arch_atomic_dec_return_release(v);
1270 #elif defined(arch_atomic_dec_return_relaxed)
1271 __atomic_release_fence();
1272 return arch_atomic_dec_return_relaxed(v);
1273 #elif defined(arch_atomic_dec_return)
1274 return arch_atomic_dec_return(v);
1276 return raw_atomic_sub_return_release(1, v);
1281 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1282 * @v: pointer to atomic_t
1284 * Atomically updates @v to (@v - 1) with relaxed ordering.
1286 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1288 * Return: The updated value of @v.
1290 static __always_inline int
1291 raw_atomic_dec_return_relaxed(atomic_t *v)
1293 #if defined(arch_atomic_dec_return_relaxed)
1294 return arch_atomic_dec_return_relaxed(v);
1295 #elif defined(arch_atomic_dec_return)
1296 return arch_atomic_dec_return(v);
1298 return raw_atomic_sub_return_relaxed(1, v);
1303 * raw_atomic_fetch_dec() - atomic decrement with full ordering
1304 * @v: pointer to atomic_t
1306 * Atomically updates @v to (@v - 1) with full ordering.
1308 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1310 * Return: The original value of @v.
1312 static __always_inline int
1313 raw_atomic_fetch_dec(atomic_t *v)
1315 #if defined(arch_atomic_fetch_dec)
1316 return arch_atomic_fetch_dec(v);
1317 #elif defined(arch_atomic_fetch_dec_relaxed)
1319 __atomic_pre_full_fence();
1320 ret = arch_atomic_fetch_dec_relaxed(v);
1321 __atomic_post_full_fence();
1324 return raw_atomic_fetch_sub(1, v);
1329 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1330 * @v: pointer to atomic_t
1332 * Atomically updates @v to (@v - 1) with acquire ordering.
1334 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1336 * Return: The original value of @v.
1338 static __always_inline int
1339 raw_atomic_fetch_dec_acquire(atomic_t *v)
1341 #if defined(arch_atomic_fetch_dec_acquire)
1342 return arch_atomic_fetch_dec_acquire(v);
1343 #elif defined(arch_atomic_fetch_dec_relaxed)
1344 int ret = arch_atomic_fetch_dec_relaxed(v);
1345 __atomic_acquire_fence();
1347 #elif defined(arch_atomic_fetch_dec)
1348 return arch_atomic_fetch_dec(v);
1350 return raw_atomic_fetch_sub_acquire(1, v);
1355 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1356 * @v: pointer to atomic_t
1358 * Atomically updates @v to (@v - 1) with release ordering.
1360 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1362 * Return: The original value of @v.
1364 static __always_inline int
1365 raw_atomic_fetch_dec_release(atomic_t *v)
1367 #if defined(arch_atomic_fetch_dec_release)
1368 return arch_atomic_fetch_dec_release(v);
1369 #elif defined(arch_atomic_fetch_dec_relaxed)
1370 __atomic_release_fence();
1371 return arch_atomic_fetch_dec_relaxed(v);
1372 #elif defined(arch_atomic_fetch_dec)
1373 return arch_atomic_fetch_dec(v);
1375 return raw_atomic_fetch_sub_release(1, v);
1380 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1381 * @v: pointer to atomic_t
1383 * Atomically updates @v to (@v - 1) with relaxed ordering.
1385 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1387 * Return: The original value of @v.
1389 static __always_inline int
1390 raw_atomic_fetch_dec_relaxed(atomic_t *v)
1392 #if defined(arch_atomic_fetch_dec_relaxed)
1393 return arch_atomic_fetch_dec_relaxed(v);
1394 #elif defined(arch_atomic_fetch_dec)
1395 return arch_atomic_fetch_dec(v);
1397 return raw_atomic_fetch_sub_relaxed(1, v);
1402 * raw_atomic_and() - atomic bitwise AND with relaxed ordering
1404 * @v: pointer to atomic_t
1406 * Atomically updates @v to (@v & @i) with relaxed ordering.
1408 * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1412 static __always_inline void
1413 raw_atomic_and(int i, atomic_t *v)
1415 arch_atomic_and(i, v);
1419 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1421 * @v: pointer to atomic_t
1423 * Atomically updates @v to (@v & @i) with full ordering.
1425 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1427 * Return: The original value of @v.
1429 static __always_inline int
1430 raw_atomic_fetch_and(int i, atomic_t *v)
1432 #if defined(arch_atomic_fetch_and)
1433 return arch_atomic_fetch_and(i, v);
1434 #elif defined(arch_atomic_fetch_and_relaxed)
1436 __atomic_pre_full_fence();
1437 ret = arch_atomic_fetch_and_relaxed(i, v);
1438 __atomic_post_full_fence();
1441 #error "Unable to define raw_atomic_fetch_and"
1446 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
1448 * @v: pointer to atomic_t
1450 * Atomically updates @v to (@v & @i) with acquire ordering.
1452 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
1454 * Return: The original value of @v.
1456 static __always_inline int
1457 raw_atomic_fetch_and_acquire(int i, atomic_t *v)
1459 #if defined(arch_atomic_fetch_and_acquire)
1460 return arch_atomic_fetch_and_acquire(i, v);
1461 #elif defined(arch_atomic_fetch_and_relaxed)
1462 int ret = arch_atomic_fetch_and_relaxed(i, v);
1463 __atomic_acquire_fence();
1465 #elif defined(arch_atomic_fetch_and)
1466 return arch_atomic_fetch_and(i, v);
1468 #error "Unable to define raw_atomic_fetch_and_acquire"
1473 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1475 * @v: pointer to atomic_t
1477 * Atomically updates @v to (@v & @i) with release ordering.
1479 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1481 * Return: The original value of @v.
1483 static __always_inline int
1484 raw_atomic_fetch_and_release(int i, atomic_t *v)
1486 #if defined(arch_atomic_fetch_and_release)
1487 return arch_atomic_fetch_and_release(i, v);
1488 #elif defined(arch_atomic_fetch_and_relaxed)
1489 __atomic_release_fence();
1490 return arch_atomic_fetch_and_relaxed(i, v);
1491 #elif defined(arch_atomic_fetch_and)
1492 return arch_atomic_fetch_and(i, v);
1494 #error "Unable to define raw_atomic_fetch_and_release"
1499 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1501 * @v: pointer to atomic_t
1503 * Atomically updates @v to (@v & @i) with relaxed ordering.
1505 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1507 * Return: The original value of @v.
1509 static __always_inline int
1510 raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
1512 #if defined(arch_atomic_fetch_and_relaxed)
1513 return arch_atomic_fetch_and_relaxed(i, v);
1514 #elif defined(arch_atomic_fetch_and)
1515 return arch_atomic_fetch_and(i, v);
1517 #error "Unable to define raw_atomic_fetch_and_relaxed"
1522 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
1524 * @v: pointer to atomic_t
1526 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1528 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1532 static __always_inline void
1533 raw_atomic_andnot(int i, atomic_t *v)
1535 #if defined(arch_atomic_andnot)
1536 arch_atomic_andnot(i, v);
1538 raw_atomic_and(~i, v);
1543 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
1545 * @v: pointer to atomic_t
1547 * Atomically updates @v to (@v & ~@i) with full ordering.
1549 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
1551 * Return: The original value of @v.
1553 static __always_inline int
1554 raw_atomic_fetch_andnot(int i, atomic_t *v)
1556 #if defined(arch_atomic_fetch_andnot)
1557 return arch_atomic_fetch_andnot(i, v);
1558 #elif defined(arch_atomic_fetch_andnot_relaxed)
1560 __atomic_pre_full_fence();
1561 ret = arch_atomic_fetch_andnot_relaxed(i, v);
1562 __atomic_post_full_fence();
1565 return raw_atomic_fetch_and(~i, v);
1570 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1572 * @v: pointer to atomic_t
1574 * Atomically updates @v to (@v & ~@i) with acquire ordering.
1576 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1578 * Return: The original value of @v.
1580 static __always_inline int
1581 raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
1583 #if defined(arch_atomic_fetch_andnot_acquire)
1584 return arch_atomic_fetch_andnot_acquire(i, v);
1585 #elif defined(arch_atomic_fetch_andnot_relaxed)
1586 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
1587 __atomic_acquire_fence();
1589 #elif defined(arch_atomic_fetch_andnot)
1590 return arch_atomic_fetch_andnot(i, v);
1592 return raw_atomic_fetch_and_acquire(~i, v);
1597 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1599 * @v: pointer to atomic_t
1601 * Atomically updates @v to (@v & ~@i) with release ordering.
1603 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1605 * Return: The original value of @v.
1607 static __always_inline int
1608 raw_atomic_fetch_andnot_release(int i, atomic_t *v)
1610 #if defined(arch_atomic_fetch_andnot_release)
1611 return arch_atomic_fetch_andnot_release(i, v);
1612 #elif defined(arch_atomic_fetch_andnot_relaxed)
1613 __atomic_release_fence();
1614 return arch_atomic_fetch_andnot_relaxed(i, v);
1615 #elif defined(arch_atomic_fetch_andnot)
1616 return arch_atomic_fetch_andnot(i, v);
1618 return raw_atomic_fetch_and_release(~i, v);
1623 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1625 * @v: pointer to atomic_t
1627 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1629 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1631 * Return: The original value of @v.
1633 static __always_inline int
1634 raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1636 #if defined(arch_atomic_fetch_andnot_relaxed)
1637 return arch_atomic_fetch_andnot_relaxed(i, v);
1638 #elif defined(arch_atomic_fetch_andnot)
1639 return arch_atomic_fetch_andnot(i, v);
1641 return raw_atomic_fetch_and_relaxed(~i, v);
1646 * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1648 * @v: pointer to atomic_t
1650 * Atomically updates @v to (@v | @i) with relaxed ordering.
1652 * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1656 static __always_inline void
1657 raw_atomic_or(int i, atomic_t *v)
1659 arch_atomic_or(i, v);
1663 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1665 * @v: pointer to atomic_t
1667 * Atomically updates @v to (@v | @i) with full ordering.
1669 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1671 * Return: The original value of @v.
1673 static __always_inline int
1674 raw_atomic_fetch_or(int i, atomic_t *v)
1676 #if defined(arch_atomic_fetch_or)
1677 return arch_atomic_fetch_or(i, v);
1678 #elif defined(arch_atomic_fetch_or_relaxed)
1680 __atomic_pre_full_fence();
1681 ret = arch_atomic_fetch_or_relaxed(i, v);
1682 __atomic_post_full_fence();
1685 #error "Unable to define raw_atomic_fetch_or"
1690 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1692 * @v: pointer to atomic_t
1694 * Atomically updates @v to (@v | @i) with acquire ordering.
1696 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
1698 * Return: The original value of @v.
1700 static __always_inline int
1701 raw_atomic_fetch_or_acquire(int i, atomic_t *v)
1703 #if defined(arch_atomic_fetch_or_acquire)
1704 return arch_atomic_fetch_or_acquire(i, v);
1705 #elif defined(arch_atomic_fetch_or_relaxed)
1706 int ret = arch_atomic_fetch_or_relaxed(i, v);
1707 __atomic_acquire_fence();
1709 #elif defined(arch_atomic_fetch_or)
1710 return arch_atomic_fetch_or(i, v);
1712 #error "Unable to define raw_atomic_fetch_or_acquire"
1717 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1719 * @v: pointer to atomic_t
1721 * Atomically updates @v to (@v | @i) with release ordering.
1723 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1725 * Return: The original value of @v.
1727 static __always_inline int
1728 raw_atomic_fetch_or_release(int i, atomic_t *v)
1730 #if defined(arch_atomic_fetch_or_release)
1731 return arch_atomic_fetch_or_release(i, v);
1732 #elif defined(arch_atomic_fetch_or_relaxed)
1733 __atomic_release_fence();
1734 return arch_atomic_fetch_or_relaxed(i, v);
1735 #elif defined(arch_atomic_fetch_or)
1736 return arch_atomic_fetch_or(i, v);
1738 #error "Unable to define raw_atomic_fetch_or_release"
1743 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1745 * @v: pointer to atomic_t
1747 * Atomically updates @v to (@v | @i) with relaxed ordering.
1749 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1751 * Return: The original value of @v.
1753 static __always_inline int
1754 raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1756 #if defined(arch_atomic_fetch_or_relaxed)
1757 return arch_atomic_fetch_or_relaxed(i, v);
1758 #elif defined(arch_atomic_fetch_or)
1759 return arch_atomic_fetch_or(i, v);
1761 #error "Unable to define raw_atomic_fetch_or_relaxed"
1766 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1768 * @v: pointer to atomic_t
1770 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1772 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1776 static __always_inline void
1777 raw_atomic_xor(int i, atomic_t *v)
1779 arch_atomic_xor(i, v);
1783 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1785 * @v: pointer to atomic_t
1787 * Atomically updates @v to (@v ^ @i) with full ordering.
1789 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1791 * Return: The original value of @v.
1793 static __always_inline int
1794 raw_atomic_fetch_xor(int i, atomic_t *v)
1796 #if defined(arch_atomic_fetch_xor)
1797 return arch_atomic_fetch_xor(i, v);
1798 #elif defined(arch_atomic_fetch_xor_relaxed)
1800 __atomic_pre_full_fence();
1801 ret = arch_atomic_fetch_xor_relaxed(i, v);
1802 __atomic_post_full_fence();
1805 #error "Unable to define raw_atomic_fetch_xor"
1810 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1812 * @v: pointer to atomic_t
1814 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1816 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
1818 * Return: The original value of @v.
1820 static __always_inline int
1821 raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
1823 #if defined(arch_atomic_fetch_xor_acquire)
1824 return arch_atomic_fetch_xor_acquire(i, v);
1825 #elif defined(arch_atomic_fetch_xor_relaxed)
1826 int ret = arch_atomic_fetch_xor_relaxed(i, v);
1827 __atomic_acquire_fence();
1829 #elif defined(arch_atomic_fetch_xor)
1830 return arch_atomic_fetch_xor(i, v);
1832 #error "Unable to define raw_atomic_fetch_xor_acquire"
1837 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1839 * @v: pointer to atomic_t
1841 * Atomically updates @v to (@v ^ @i) with release ordering.
1843 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1845 * Return: The original value of @v.
1847 static __always_inline int
1848 raw_atomic_fetch_xor_release(int i, atomic_t *v)
1850 #if defined(arch_atomic_fetch_xor_release)
1851 return arch_atomic_fetch_xor_release(i, v);
1852 #elif defined(arch_atomic_fetch_xor_relaxed)
1853 __atomic_release_fence();
1854 return arch_atomic_fetch_xor_relaxed(i, v);
1855 #elif defined(arch_atomic_fetch_xor)
1856 return arch_atomic_fetch_xor(i, v);
1858 #error "Unable to define raw_atomic_fetch_xor_release"
1863 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1865 * @v: pointer to atomic_t
1867 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1869 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1871 * Return: The original value of @v.
1873 static __always_inline int
1874 raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1876 #if defined(arch_atomic_fetch_xor_relaxed)
1877 return arch_atomic_fetch_xor_relaxed(i, v);
1878 #elif defined(arch_atomic_fetch_xor)
1879 return arch_atomic_fetch_xor(i, v);
1881 #error "Unable to define raw_atomic_fetch_xor_relaxed"
1886 * raw_atomic_xchg() - atomic exchange with full ordering
1887 * @v: pointer to atomic_t
1888 * @new: int value to assign
1890 * Atomically updates @v to @new with full ordering.
1892 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1894 * Return: The original value of @v.
1896 static __always_inline int
1897 raw_atomic_xchg(atomic_t *v, int new)
1899 #if defined(arch_atomic_xchg)
1900 return arch_atomic_xchg(v, new);
1901 #elif defined(arch_atomic_xchg_relaxed)
1903 __atomic_pre_full_fence();
1904 ret = arch_atomic_xchg_relaxed(v, new);
1905 __atomic_post_full_fence();
1908 return raw_xchg(&v->counter, new);
1913 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1914 * @v: pointer to atomic_t
1915 * @new: int value to assign
1917 * Atomically updates @v to @new with acquire ordering.
1919 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1921 * Return: The original value of @v.
1923 static __always_inline int
1924 raw_atomic_xchg_acquire(atomic_t *v, int new)
1926 #if defined(arch_atomic_xchg_acquire)
1927 return arch_atomic_xchg_acquire(v, new);
1928 #elif defined(arch_atomic_xchg_relaxed)
1929 int ret = arch_atomic_xchg_relaxed(v, new);
1930 __atomic_acquire_fence();
1932 #elif defined(arch_atomic_xchg)
1933 return arch_atomic_xchg(v, new);
1935 return raw_xchg_acquire(&v->counter, new);
1940 * raw_atomic_xchg_release() - atomic exchange with release ordering
1941 * @v: pointer to atomic_t
1942 * @new: int value to assign
1944 * Atomically updates @v to @new with release ordering.
1946 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1948 * Return: The original value of @v.
1950 static __always_inline int
1951 raw_atomic_xchg_release(atomic_t *v, int new)
1953 #if defined(arch_atomic_xchg_release)
1954 return arch_atomic_xchg_release(v, new);
1955 #elif defined(arch_atomic_xchg_relaxed)
1956 __atomic_release_fence();
1957 return arch_atomic_xchg_relaxed(v, new);
1958 #elif defined(arch_atomic_xchg)
1959 return arch_atomic_xchg(v, new);
1961 return raw_xchg_release(&v->counter, new);
1966 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1967 * @v: pointer to atomic_t
1968 * @new: int value to assign
1970 * Atomically updates @v to @new with relaxed ordering.
1972 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1974 * Return: The original value of @v.
1976 static __always_inline int
1977 raw_atomic_xchg_relaxed(atomic_t *v, int new)
1979 #if defined(arch_atomic_xchg_relaxed)
1980 return arch_atomic_xchg_relaxed(v, new);
1981 #elif defined(arch_atomic_xchg)
1982 return arch_atomic_xchg(v, new);
1984 return raw_xchg_relaxed(&v->counter, new);
1989 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
1990 * @v: pointer to atomic_t
1991 * @old: int value to compare with
1992 * @new: int value to assign
1994 * If (@v == @old), atomically updates @v to @new with full ordering.
1996 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
1998 * Return: The original value of @v.
2000 static __always_inline int
2001 raw_atomic_cmpxchg(atomic_t *v, int old, int new)
2003 #if defined(arch_atomic_cmpxchg)
2004 return arch_atomic_cmpxchg(v, old, new);
2005 #elif defined(arch_atomic_cmpxchg_relaxed)
2007 __atomic_pre_full_fence();
2008 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2009 __atomic_post_full_fence();
2012 return raw_cmpxchg(&v->counter, old, new);
2017 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2018 * @v: pointer to atomic_t
2019 * @old: int value to compare with
2020 * @new: int value to assign
2022 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2024 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2026 * Return: The original value of @v.
2028 static __always_inline int
2029 raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
2031 #if defined(arch_atomic_cmpxchg_acquire)
2032 return arch_atomic_cmpxchg_acquire(v, old, new);
2033 #elif defined(arch_atomic_cmpxchg_relaxed)
2034 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2035 __atomic_acquire_fence();
2037 #elif defined(arch_atomic_cmpxchg)
2038 return arch_atomic_cmpxchg(v, old, new);
2040 return raw_cmpxchg_acquire(&v->counter, old, new);
2045 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2046 * @v: pointer to atomic_t
2047 * @old: int value to compare with
2048 * @new: int value to assign
2050 * If (@v == @old), atomically updates @v to @new with release ordering.
2052 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2054 * Return: The original value of @v.
2056 static __always_inline int
2057 raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
2059 #if defined(arch_atomic_cmpxchg_release)
2060 return arch_atomic_cmpxchg_release(v, old, new);
2061 #elif defined(arch_atomic_cmpxchg_relaxed)
2062 __atomic_release_fence();
2063 return arch_atomic_cmpxchg_relaxed(v, old, new);
2064 #elif defined(arch_atomic_cmpxchg)
2065 return arch_atomic_cmpxchg(v, old, new);
2067 return raw_cmpxchg_release(&v->counter, old, new);
2072 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2073 * @v: pointer to atomic_t
2074 * @old: int value to compare with
2075 * @new: int value to assign
2077 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2079 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2081 * Return: The original value of @v.
2083 static __always_inline int
2084 raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
2086 #if defined(arch_atomic_cmpxchg_relaxed)
2087 return arch_atomic_cmpxchg_relaxed(v, old, new);
2088 #elif defined(arch_atomic_cmpxchg)
2089 return arch_atomic_cmpxchg(v, old, new);
2091 return raw_cmpxchg_relaxed(&v->counter, old, new);
2096 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
2097 * @v: pointer to atomic_t
2098 * @old: pointer to int value to compare with
2099 * @new: int value to assign
2101 * If (@v == @old), atomically updates @v to @new with full ordering.
2102 * Otherwise, updates @old to the current value of @v.
2104 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2106 * Return: @true if the exchange occured, @false otherwise.
2108 static __always_inline bool
2109 raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
2111 #if defined(arch_atomic_try_cmpxchg)
2112 return arch_atomic_try_cmpxchg(v, old, new);
2113 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2115 __atomic_pre_full_fence();
2116 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2117 __atomic_post_full_fence();
2121 r = raw_atomic_cmpxchg(v, o, new);
2122 if (unlikely(r != o))
2124 return likely(r == o);
2129 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2130 * @v: pointer to atomic_t
2131 * @old: pointer to int value to compare with
2132 * @new: int value to assign
2134 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2135 * Otherwise, updates @old to the current value of @v.
2137 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2139 * Return: @true if the exchange occured, @false otherwise.
2141 static __always_inline bool
2142 raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
2144 #if defined(arch_atomic_try_cmpxchg_acquire)
2145 return arch_atomic_try_cmpxchg_acquire(v, old, new);
2146 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2147 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2148 __atomic_acquire_fence();
2150 #elif defined(arch_atomic_try_cmpxchg)
2151 return arch_atomic_try_cmpxchg(v, old, new);
2154 r = raw_atomic_cmpxchg_acquire(v, o, new);
2155 if (unlikely(r != o))
2157 return likely(r == o);
2162 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2163 * @v: pointer to atomic_t
2164 * @old: pointer to int value to compare with
2165 * @new: int value to assign
2167 * If (@v == @old), atomically updates @v to @new with release ordering.
2168 * Otherwise, updates @old to the current value of @v.
2170 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2172 * Return: @true if the exchange occured, @false otherwise.
2174 static __always_inline bool
2175 raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
2177 #if defined(arch_atomic_try_cmpxchg_release)
2178 return arch_atomic_try_cmpxchg_release(v, old, new);
2179 #elif defined(arch_atomic_try_cmpxchg_relaxed)
2180 __atomic_release_fence();
2181 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2182 #elif defined(arch_atomic_try_cmpxchg)
2183 return arch_atomic_try_cmpxchg(v, old, new);
2186 r = raw_atomic_cmpxchg_release(v, o, new);
2187 if (unlikely(r != o))
2189 return likely(r == o);
2194 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2195 * @v: pointer to atomic_t
2196 * @old: pointer to int value to compare with
2197 * @new: int value to assign
2199 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2200 * Otherwise, updates @old to the current value of @v.
2202 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2204 * Return: @true if the exchange occured, @false otherwise.
2206 static __always_inline bool
2207 raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
2209 #if defined(arch_atomic_try_cmpxchg_relaxed)
2210 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2211 #elif defined(arch_atomic_try_cmpxchg)
2212 return arch_atomic_try_cmpxchg(v, old, new);
2215 r = raw_atomic_cmpxchg_relaxed(v, o, new);
2216 if (unlikely(r != o))
2218 return likely(r == o);
2223 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
2224 * @i: int value to add
2225 * @v: pointer to atomic_t
2227 * Atomically updates @v to (@v - @i) with full ordering.
2229 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
2231 * Return: @true if the resulting value of @v is zero, @false otherwise.
2233 static __always_inline bool
2234 raw_atomic_sub_and_test(int i, atomic_t *v)
2236 #if defined(arch_atomic_sub_and_test)
2237 return arch_atomic_sub_and_test(i, v);
2239 return raw_atomic_sub_return(i, v) == 0;
2244 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
2245 * @v: pointer to atomic_t
2247 * Atomically updates @v to (@v - 1) with full ordering.
2249 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
2251 * Return: @true if the resulting value of @v is zero, @false otherwise.
2253 static __always_inline bool
2254 raw_atomic_dec_and_test(atomic_t *v)
2256 #if defined(arch_atomic_dec_and_test)
2257 return arch_atomic_dec_and_test(v);
2259 return raw_atomic_dec_return(v) == 0;
2264 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
2265 * @v: pointer to atomic_t
2267 * Atomically updates @v to (@v + 1) with full ordering.
2269 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
2271 * Return: @true if the resulting value of @v is zero, @false otherwise.
2273 static __always_inline bool
2274 raw_atomic_inc_and_test(atomic_t *v)
2276 #if defined(arch_atomic_inc_and_test)
2277 return arch_atomic_inc_and_test(v);
2279 return raw_atomic_inc_return(v) == 0;
2284 * raw_atomic_add_negative() - atomic add and test if negative with full ordering
2285 * @i: int value to add
2286 * @v: pointer to atomic_t
2288 * Atomically updates @v to (@v + @i) with full ordering.
2290 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2292 * Return: @true if the resulting value of @v is negative, @false otherwise.
2294 static __always_inline bool
2295 raw_atomic_add_negative(int i, atomic_t *v)
2297 #if defined(arch_atomic_add_negative)
2298 return arch_atomic_add_negative(i, v);
2299 #elif defined(arch_atomic_add_negative_relaxed)
2301 __atomic_pre_full_fence();
2302 ret = arch_atomic_add_negative_relaxed(i, v);
2303 __atomic_post_full_fence();
2306 return raw_atomic_add_return(i, v) < 0;
2311 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
2312 * @i: int value to add
2313 * @v: pointer to atomic_t
2315 * Atomically updates @v to (@v + @i) with acquire ordering.
2317 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
2319 * Return: @true if the resulting value of @v is negative, @false otherwise.
2321 static __always_inline bool
2322 raw_atomic_add_negative_acquire(int i, atomic_t *v)
2324 #if defined(arch_atomic_add_negative_acquire)
2325 return arch_atomic_add_negative_acquire(i, v);
2326 #elif defined(arch_atomic_add_negative_relaxed)
2327 bool ret = arch_atomic_add_negative_relaxed(i, v);
2328 __atomic_acquire_fence();
2330 #elif defined(arch_atomic_add_negative)
2331 return arch_atomic_add_negative(i, v);
2333 return raw_atomic_add_return_acquire(i, v) < 0;
2338 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
2339 * @i: int value to add
2340 * @v: pointer to atomic_t
2342 * Atomically updates @v to (@v + @i) with release ordering.
2344 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2346 * Return: @true if the resulting value of @v is negative, @false otherwise.
2348 static __always_inline bool
2349 raw_atomic_add_negative_release(int i, atomic_t *v)
2351 #if defined(arch_atomic_add_negative_release)
2352 return arch_atomic_add_negative_release(i, v);
2353 #elif defined(arch_atomic_add_negative_relaxed)
2354 __atomic_release_fence();
2355 return arch_atomic_add_negative_relaxed(i, v);
2356 #elif defined(arch_atomic_add_negative)
2357 return arch_atomic_add_negative(i, v);
2359 return raw_atomic_add_return_release(i, v) < 0;
2364 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
2365 * @i: int value to add
2366 * @v: pointer to atomic_t
2368 * Atomically updates @v to (@v + @i) with relaxed ordering.
2370 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
2372 * Return: @true if the resulting value of @v is negative, @false otherwise.
2374 static __always_inline bool
2375 raw_atomic_add_negative_relaxed(int i, atomic_t *v)
2377 #if defined(arch_atomic_add_negative_relaxed)
2378 return arch_atomic_add_negative_relaxed(i, v);
2379 #elif defined(arch_atomic_add_negative)
2380 return arch_atomic_add_negative(i, v);
2382 return raw_atomic_add_return_relaxed(i, v) < 0;
2387 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
2388 * @v: pointer to atomic_t
2389 * @a: int value to add
2390 * @u: int value to compare with
2392 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2394 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2396 * Return: The original value of @v.
2398 static __always_inline int
2399 raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
2401 #if defined(arch_atomic_fetch_add_unless)
2402 return arch_atomic_fetch_add_unless(v, a, u);
2404 int c = raw_atomic_read(v);
2407 if (unlikely(c == u))
2409 } while (!raw_atomic_try_cmpxchg(v, &c, c + a));
2416 * raw_atomic_add_unless() - atomic add unless value with full ordering
2417 * @v: pointer to atomic_t
2418 * @a: int value to add
2419 * @u: int value to compare with
2421 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2423 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2425 * Return: @true if @v was updated, @false otherwise.
2427 static __always_inline bool
2428 raw_atomic_add_unless(atomic_t *v, int a, int u)
2430 #if defined(arch_atomic_add_unless)
2431 return arch_atomic_add_unless(v, a, u);
2433 return raw_atomic_fetch_add_unless(v, a, u) != u;
2438 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
2439 * @v: pointer to atomic_t
2441 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2443 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2445 * Return: @true if @v was updated, @false otherwise.
2447 static __always_inline bool
2448 raw_atomic_inc_not_zero(atomic_t *v)
2450 #if defined(arch_atomic_inc_not_zero)
2451 return arch_atomic_inc_not_zero(v);
2453 return raw_atomic_add_unless(v, 1, 0);
2458 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2459 * @v: pointer to atomic_t
2461 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2463 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2465 * Return: @true if @v was updated, @false otherwise.
2467 static __always_inline bool
2468 raw_atomic_inc_unless_negative(atomic_t *v)
2470 #if defined(arch_atomic_inc_unless_negative)
2471 return arch_atomic_inc_unless_negative(v);
2473 int c = raw_atomic_read(v);
2476 if (unlikely(c < 0))
2478 } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
2485 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2486 * @v: pointer to atomic_t
2488 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2490 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2492 * Return: @true if @v was updated, @false otherwise.
2494 static __always_inline bool
2495 raw_atomic_dec_unless_positive(atomic_t *v)
2497 #if defined(arch_atomic_dec_unless_positive)
2498 return arch_atomic_dec_unless_positive(v);
2500 int c = raw_atomic_read(v);
2503 if (unlikely(c > 0))
2505 } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
2512 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2513 * @v: pointer to atomic_t
2515 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2517 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2519 * Return: The old value of (@v - 1), regardless of whether @v was updated.
2521 static __always_inline int
2522 raw_atomic_dec_if_positive(atomic_t *v)
2524 #if defined(arch_atomic_dec_if_positive)
2525 return arch_atomic_dec_if_positive(v);
2527 int dec, c = raw_atomic_read(v);
2531 if (unlikely(dec < 0))
2533 } while (!raw_atomic_try_cmpxchg(v, &c, dec));
2539 #ifdef CONFIG_GENERIC_ATOMIC64
2540 #include <asm-generic/atomic64.h>
2544 * raw_atomic64_read() - atomic load with relaxed ordering
2545 * @v: pointer to atomic64_t
2547 * Atomically loads the value of @v with relaxed ordering.
2549 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2551 * Return: The value loaded from @v.
2553 static __always_inline s64
2554 raw_atomic64_read(const atomic64_t *v)
2556 return arch_atomic64_read(v);
2560 * raw_atomic64_read_acquire() - atomic load with acquire ordering
2561 * @v: pointer to atomic64_t
2563 * Atomically loads the value of @v with acquire ordering.
2565 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2567 * Return: The value loaded from @v.
2569 static __always_inline s64
2570 raw_atomic64_read_acquire(const atomic64_t *v)
2572 #if defined(arch_atomic64_read_acquire)
2573 return arch_atomic64_read_acquire(v);
2577 if (__native_word(atomic64_t)) {
2578 ret = smp_load_acquire(&(v)->counter);
2580 ret = raw_atomic64_read(v);
2581 __atomic_acquire_fence();
2589 * raw_atomic64_set() - atomic set with relaxed ordering
2590 * @v: pointer to atomic64_t
2591 * @i: s64 value to assign
2593 * Atomically sets @v to @i with relaxed ordering.
2595 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2599 static __always_inline void
2600 raw_atomic64_set(atomic64_t *v, s64 i)
2602 arch_atomic64_set(v, i);
2606 * raw_atomic64_set_release() - atomic set with release ordering
2607 * @v: pointer to atomic64_t
2608 * @i: s64 value to assign
2610 * Atomically sets @v to @i with release ordering.
2612 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2616 static __always_inline void
2617 raw_atomic64_set_release(atomic64_t *v, s64 i)
2619 #if defined(arch_atomic64_set_release)
2620 arch_atomic64_set_release(v, i);
2622 if (__native_word(atomic64_t)) {
2623 smp_store_release(&(v)->counter, i);
2625 __atomic_release_fence();
2626 raw_atomic64_set(v, i);
2632 * raw_atomic64_add() - atomic add with relaxed ordering
2633 * @i: s64 value to add
2634 * @v: pointer to atomic64_t
2636 * Atomically updates @v to (@v + @i) with relaxed ordering.
2638 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2642 static __always_inline void
2643 raw_atomic64_add(s64 i, atomic64_t *v)
2645 arch_atomic64_add(i, v);
2649 * raw_atomic64_add_return() - atomic add with full ordering
2650 * @i: s64 value to add
2651 * @v: pointer to atomic64_t
2653 * Atomically updates @v to (@v + @i) with full ordering.
2655 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2657 * Return: The updated value of @v.
2659 static __always_inline s64
2660 raw_atomic64_add_return(s64 i, atomic64_t *v)
2662 #if defined(arch_atomic64_add_return)
2663 return arch_atomic64_add_return(i, v);
2664 #elif defined(arch_atomic64_add_return_relaxed)
2666 __atomic_pre_full_fence();
2667 ret = arch_atomic64_add_return_relaxed(i, v);
2668 __atomic_post_full_fence();
2671 #error "Unable to define raw_atomic64_add_return"
2676 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2677 * @i: s64 value to add
2678 * @v: pointer to atomic64_t
2680 * Atomically updates @v to (@v + @i) with acquire ordering.
2682 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2684 * Return: The updated value of @v.
2686 static __always_inline s64
2687 raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
2689 #if defined(arch_atomic64_add_return_acquire)
2690 return arch_atomic64_add_return_acquire(i, v);
2691 #elif defined(arch_atomic64_add_return_relaxed)
2692 s64 ret = arch_atomic64_add_return_relaxed(i, v);
2693 __atomic_acquire_fence();
2695 #elif defined(arch_atomic64_add_return)
2696 return arch_atomic64_add_return(i, v);
2698 #error "Unable to define raw_atomic64_add_return_acquire"
2703 * raw_atomic64_add_return_release() - atomic add with release ordering
2704 * @i: s64 value to add
2705 * @v: pointer to atomic64_t
2707 * Atomically updates @v to (@v + @i) with release ordering.
2709 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2711 * Return: The updated value of @v.
2713 static __always_inline s64
2714 raw_atomic64_add_return_release(s64 i, atomic64_t *v)
2716 #if defined(arch_atomic64_add_return_release)
2717 return arch_atomic64_add_return_release(i, v);
2718 #elif defined(arch_atomic64_add_return_relaxed)
2719 __atomic_release_fence();
2720 return arch_atomic64_add_return_relaxed(i, v);
2721 #elif defined(arch_atomic64_add_return)
2722 return arch_atomic64_add_return(i, v);
2724 #error "Unable to define raw_atomic64_add_return_release"
2729 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2730 * @i: s64 value to add
2731 * @v: pointer to atomic64_t
2733 * Atomically updates @v to (@v + @i) with relaxed ordering.
2735 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2737 * Return: The updated value of @v.
2739 static __always_inline s64
2740 raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2742 #if defined(arch_atomic64_add_return_relaxed)
2743 return arch_atomic64_add_return_relaxed(i, v);
2744 #elif defined(arch_atomic64_add_return)
2745 return arch_atomic64_add_return(i, v);
2747 #error "Unable to define raw_atomic64_add_return_relaxed"
2752 * raw_atomic64_fetch_add() - atomic add with full ordering
2753 * @i: s64 value to add
2754 * @v: pointer to atomic64_t
2756 * Atomically updates @v to (@v + @i) with full ordering.
2758 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2760 * Return: The original value of @v.
2762 static __always_inline s64
2763 raw_atomic64_fetch_add(s64 i, atomic64_t *v)
2765 #if defined(arch_atomic64_fetch_add)
2766 return arch_atomic64_fetch_add(i, v);
2767 #elif defined(arch_atomic64_fetch_add_relaxed)
2769 __atomic_pre_full_fence();
2770 ret = arch_atomic64_fetch_add_relaxed(i, v);
2771 __atomic_post_full_fence();
2774 #error "Unable to define raw_atomic64_fetch_add"
2779 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
2780 * @i: s64 value to add
2781 * @v: pointer to atomic64_t
2783 * Atomically updates @v to (@v + @i) with acquire ordering.
2785 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
2787 * Return: The original value of @v.
2789 static __always_inline s64
2790 raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
2792 #if defined(arch_atomic64_fetch_add_acquire)
2793 return arch_atomic64_fetch_add_acquire(i, v);
2794 #elif defined(arch_atomic64_fetch_add_relaxed)
2795 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
2796 __atomic_acquire_fence();
2798 #elif defined(arch_atomic64_fetch_add)
2799 return arch_atomic64_fetch_add(i, v);
2801 #error "Unable to define raw_atomic64_fetch_add_acquire"
2806 * raw_atomic64_fetch_add_release() - atomic add with release ordering
2807 * @i: s64 value to add
2808 * @v: pointer to atomic64_t
2810 * Atomically updates @v to (@v + @i) with release ordering.
2812 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2814 * Return: The original value of @v.
2816 static __always_inline s64
2817 raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
2819 #if defined(arch_atomic64_fetch_add_release)
2820 return arch_atomic64_fetch_add_release(i, v);
2821 #elif defined(arch_atomic64_fetch_add_relaxed)
2822 __atomic_release_fence();
2823 return arch_atomic64_fetch_add_relaxed(i, v);
2824 #elif defined(arch_atomic64_fetch_add)
2825 return arch_atomic64_fetch_add(i, v);
2827 #error "Unable to define raw_atomic64_fetch_add_release"
2832 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2833 * @i: s64 value to add
2834 * @v: pointer to atomic64_t
2836 * Atomically updates @v to (@v + @i) with relaxed ordering.
2838 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2840 * Return: The original value of @v.
2842 static __always_inline s64
2843 raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2845 #if defined(arch_atomic64_fetch_add_relaxed)
2846 return arch_atomic64_fetch_add_relaxed(i, v);
2847 #elif defined(arch_atomic64_fetch_add)
2848 return arch_atomic64_fetch_add(i, v);
2850 #error "Unable to define raw_atomic64_fetch_add_relaxed"
2855 * raw_atomic64_sub() - atomic subtract with relaxed ordering
2856 * @i: s64 value to subtract
2857 * @v: pointer to atomic64_t
2859 * Atomically updates @v to (@v - @i) with relaxed ordering.
2861 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2865 static __always_inline void
2866 raw_atomic64_sub(s64 i, atomic64_t *v)
2868 arch_atomic64_sub(i, v);
2872 * raw_atomic64_sub_return() - atomic subtract with full ordering
2873 * @i: s64 value to subtract
2874 * @v: pointer to atomic64_t
2876 * Atomically updates @v to (@v - @i) with full ordering.
2878 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2880 * Return: The updated value of @v.
2882 static __always_inline s64
2883 raw_atomic64_sub_return(s64 i, atomic64_t *v)
2885 #if defined(arch_atomic64_sub_return)
2886 return arch_atomic64_sub_return(i, v);
2887 #elif defined(arch_atomic64_sub_return_relaxed)
2889 __atomic_pre_full_fence();
2890 ret = arch_atomic64_sub_return_relaxed(i, v);
2891 __atomic_post_full_fence();
2894 #error "Unable to define raw_atomic64_sub_return"
2899 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
2900 * @i: s64 value to subtract
2901 * @v: pointer to atomic64_t
2903 * Atomically updates @v to (@v - @i) with acquire ordering.
2905 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
2907 * Return: The updated value of @v.
2909 static __always_inline s64
2910 raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
2912 #if defined(arch_atomic64_sub_return_acquire)
2913 return arch_atomic64_sub_return_acquire(i, v);
2914 #elif defined(arch_atomic64_sub_return_relaxed)
2915 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
2916 __atomic_acquire_fence();
2918 #elif defined(arch_atomic64_sub_return)
2919 return arch_atomic64_sub_return(i, v);
2921 #error "Unable to define raw_atomic64_sub_return_acquire"
2926 * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2927 * @i: s64 value to subtract
2928 * @v: pointer to atomic64_t
2930 * Atomically updates @v to (@v - @i) with release ordering.
2932 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2934 * Return: The updated value of @v.
2936 static __always_inline s64
2937 raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
2939 #if defined(arch_atomic64_sub_return_release)
2940 return arch_atomic64_sub_return_release(i, v);
2941 #elif defined(arch_atomic64_sub_return_relaxed)
2942 __atomic_release_fence();
2943 return arch_atomic64_sub_return_relaxed(i, v);
2944 #elif defined(arch_atomic64_sub_return)
2945 return arch_atomic64_sub_return(i, v);
2947 #error "Unable to define raw_atomic64_sub_return_release"
2952 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2953 * @i: s64 value to subtract
2954 * @v: pointer to atomic64_t
2956 * Atomically updates @v to (@v - @i) with relaxed ordering.
2958 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2960 * Return: The updated value of @v.
2962 static __always_inline s64
2963 raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2965 #if defined(arch_atomic64_sub_return_relaxed)
2966 return arch_atomic64_sub_return_relaxed(i, v);
2967 #elif defined(arch_atomic64_sub_return)
2968 return arch_atomic64_sub_return(i, v);
2970 #error "Unable to define raw_atomic64_sub_return_relaxed"
2975 * raw_atomic64_fetch_sub() - atomic subtract with full ordering
2976 * @i: s64 value to subtract
2977 * @v: pointer to atomic64_t
2979 * Atomically updates @v to (@v - @i) with full ordering.
2981 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
2983 * Return: The original value of @v.
2985 static __always_inline s64
2986 raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
2988 #if defined(arch_atomic64_fetch_sub)
2989 return arch_atomic64_fetch_sub(i, v);
2990 #elif defined(arch_atomic64_fetch_sub_relaxed)
2992 __atomic_pre_full_fence();
2993 ret = arch_atomic64_fetch_sub_relaxed(i, v);
2994 __atomic_post_full_fence();
2997 #error "Unable to define raw_atomic64_fetch_sub"
3002 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
3003 * @i: s64 value to subtract
3004 * @v: pointer to atomic64_t
3006 * Atomically updates @v to (@v - @i) with acquire ordering.
3008 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
3010 * Return: The original value of @v.
3012 static __always_inline s64
3013 raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
3015 #if defined(arch_atomic64_fetch_sub_acquire)
3016 return arch_atomic64_fetch_sub_acquire(i, v);
3017 #elif defined(arch_atomic64_fetch_sub_relaxed)
3018 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3019 __atomic_acquire_fence();
3021 #elif defined(arch_atomic64_fetch_sub)
3022 return arch_atomic64_fetch_sub(i, v);
3024 #error "Unable to define raw_atomic64_fetch_sub_acquire"
3029 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3030 * @i: s64 value to subtract
3031 * @v: pointer to atomic64_t
3033 * Atomically updates @v to (@v - @i) with release ordering.
3035 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3037 * Return: The original value of @v.
3039 static __always_inline s64
3040 raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
3042 #if defined(arch_atomic64_fetch_sub_release)
3043 return arch_atomic64_fetch_sub_release(i, v);
3044 #elif defined(arch_atomic64_fetch_sub_relaxed)
3045 __atomic_release_fence();
3046 return arch_atomic64_fetch_sub_relaxed(i, v);
3047 #elif defined(arch_atomic64_fetch_sub)
3048 return arch_atomic64_fetch_sub(i, v);
3050 #error "Unable to define raw_atomic64_fetch_sub_release"
3055 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3056 * @i: s64 value to subtract
3057 * @v: pointer to atomic64_t
3059 * Atomically updates @v to (@v - @i) with relaxed ordering.
3061 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3063 * Return: The original value of @v.
3065 static __always_inline s64
3066 raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
3068 #if defined(arch_atomic64_fetch_sub_relaxed)
3069 return arch_atomic64_fetch_sub_relaxed(i, v);
3070 #elif defined(arch_atomic64_fetch_sub)
3071 return arch_atomic64_fetch_sub(i, v);
3073 #error "Unable to define raw_atomic64_fetch_sub_relaxed"
3078 * raw_atomic64_inc() - atomic increment with relaxed ordering
3079 * @v: pointer to atomic64_t
3081 * Atomically updates @v to (@v + 1) with relaxed ordering.
3083 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
3087 static __always_inline void
3088 raw_atomic64_inc(atomic64_t *v)
3090 #if defined(arch_atomic64_inc)
3091 arch_atomic64_inc(v);
3093 raw_atomic64_add(1, v);
3098 * raw_atomic64_inc_return() - atomic increment with full ordering
3099 * @v: pointer to atomic64_t
3101 * Atomically updates @v to (@v + 1) with full ordering.
3103 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3105 * Return: The updated value of @v.
3107 static __always_inline s64
3108 raw_atomic64_inc_return(atomic64_t *v)
3110 #if defined(arch_atomic64_inc_return)
3111 return arch_atomic64_inc_return(v);
3112 #elif defined(arch_atomic64_inc_return_relaxed)
3114 __atomic_pre_full_fence();
3115 ret = arch_atomic64_inc_return_relaxed(v);
3116 __atomic_post_full_fence();
3119 return raw_atomic64_add_return(1, v);
3124 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
3125 * @v: pointer to atomic64_t
3127 * Atomically updates @v to (@v + 1) with acquire ordering.
3129 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3131 * Return: The updated value of @v.
3133 static __always_inline s64
3134 raw_atomic64_inc_return_acquire(atomic64_t *v)
3136 #if defined(arch_atomic64_inc_return_acquire)
3137 return arch_atomic64_inc_return_acquire(v);
3138 #elif defined(arch_atomic64_inc_return_relaxed)
3139 s64 ret = arch_atomic64_inc_return_relaxed(v);
3140 __atomic_acquire_fence();
3142 #elif defined(arch_atomic64_inc_return)
3143 return arch_atomic64_inc_return(v);
3145 return raw_atomic64_add_return_acquire(1, v);
3150 * raw_atomic64_inc_return_release() - atomic increment with release ordering
3151 * @v: pointer to atomic64_t
3153 * Atomically updates @v to (@v + 1) with release ordering.
3155 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3157 * Return: The updated value of @v.
3159 static __always_inline s64
3160 raw_atomic64_inc_return_release(atomic64_t *v)
3162 #if defined(arch_atomic64_inc_return_release)
3163 return arch_atomic64_inc_return_release(v);
3164 #elif defined(arch_atomic64_inc_return_relaxed)
3165 __atomic_release_fence();
3166 return arch_atomic64_inc_return_relaxed(v);
3167 #elif defined(arch_atomic64_inc_return)
3168 return arch_atomic64_inc_return(v);
3170 return raw_atomic64_add_return_release(1, v);
3175 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
3176 * @v: pointer to atomic64_t
3178 * Atomically updates @v to (@v + 1) with relaxed ordering.
3180 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
3182 * Return: The updated value of @v.
3184 static __always_inline s64
3185 raw_atomic64_inc_return_relaxed(atomic64_t *v)
3187 #if defined(arch_atomic64_inc_return_relaxed)
3188 return arch_atomic64_inc_return_relaxed(v);
3189 #elif defined(arch_atomic64_inc_return)
3190 return arch_atomic64_inc_return(v);
3192 return raw_atomic64_add_return_relaxed(1, v);
3197 * raw_atomic64_fetch_inc() - atomic increment with full ordering
3198 * @v: pointer to atomic64_t
3200 * Atomically updates @v to (@v + 1) with full ordering.
3202 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3204 * Return: The original value of @v.
3206 static __always_inline s64
3207 raw_atomic64_fetch_inc(atomic64_t *v)
3209 #if defined(arch_atomic64_fetch_inc)
3210 return arch_atomic64_fetch_inc(v);
3211 #elif defined(arch_atomic64_fetch_inc_relaxed)
3213 __atomic_pre_full_fence();
3214 ret = arch_atomic64_fetch_inc_relaxed(v);
3215 __atomic_post_full_fence();
3218 return raw_atomic64_fetch_add(1, v);
3223 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3224 * @v: pointer to atomic64_t
3226 * Atomically updates @v to (@v + 1) with acquire ordering.
3228 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3230 * Return: The original value of @v.
3232 static __always_inline s64
3233 raw_atomic64_fetch_inc_acquire(atomic64_t *v)
3235 #if defined(arch_atomic64_fetch_inc_acquire)
3236 return arch_atomic64_fetch_inc_acquire(v);
3237 #elif defined(arch_atomic64_fetch_inc_relaxed)
3238 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
3239 __atomic_acquire_fence();
3241 #elif defined(arch_atomic64_fetch_inc)
3242 return arch_atomic64_fetch_inc(v);
3244 return raw_atomic64_fetch_add_acquire(1, v);
3249 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3250 * @v: pointer to atomic64_t
3252 * Atomically updates @v to (@v + 1) with release ordering.
3254 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3256 * Return: The original value of @v.
3258 static __always_inline s64
3259 raw_atomic64_fetch_inc_release(atomic64_t *v)
3261 #if defined(arch_atomic64_fetch_inc_release)
3262 return arch_atomic64_fetch_inc_release(v);
3263 #elif defined(arch_atomic64_fetch_inc_relaxed)
3264 __atomic_release_fence();
3265 return arch_atomic64_fetch_inc_relaxed(v);
3266 #elif defined(arch_atomic64_fetch_inc)
3267 return arch_atomic64_fetch_inc(v);
3269 return raw_atomic64_fetch_add_release(1, v);
3274 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
3275 * @v: pointer to atomic64_t
3277 * Atomically updates @v to (@v + 1) with relaxed ordering.
3279 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
3281 * Return: The original value of @v.
3283 static __always_inline s64
3284 raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
3286 #if defined(arch_atomic64_fetch_inc_relaxed)
3287 return arch_atomic64_fetch_inc_relaxed(v);
3288 #elif defined(arch_atomic64_fetch_inc)
3289 return arch_atomic64_fetch_inc(v);
3291 return raw_atomic64_fetch_add_relaxed(1, v);
3296 * raw_atomic64_dec() - atomic decrement with relaxed ordering
3297 * @v: pointer to atomic64_t
3299 * Atomically updates @v to (@v - 1) with relaxed ordering.
3301 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3305 static __always_inline void
3306 raw_atomic64_dec(atomic64_t *v)
3308 #if defined(arch_atomic64_dec)
3309 arch_atomic64_dec(v);
3311 raw_atomic64_sub(1, v);
3316 * raw_atomic64_dec_return() - atomic decrement with full ordering
3317 * @v: pointer to atomic64_t
3319 * Atomically updates @v to (@v - 1) with full ordering.
3321 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3323 * Return: The updated value of @v.
3325 static __always_inline s64
3326 raw_atomic64_dec_return(atomic64_t *v)
3328 #if defined(arch_atomic64_dec_return)
3329 return arch_atomic64_dec_return(v);
3330 #elif defined(arch_atomic64_dec_return_relaxed)
3332 __atomic_pre_full_fence();
3333 ret = arch_atomic64_dec_return_relaxed(v);
3334 __atomic_post_full_fence();
3337 return raw_atomic64_sub_return(1, v);
3342 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
3343 * @v: pointer to atomic64_t
3345 * Atomically updates @v to (@v - 1) with acquire ordering.
3347 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3349 * Return: The updated value of @v.
3351 static __always_inline s64
3352 raw_atomic64_dec_return_acquire(atomic64_t *v)
3354 #if defined(arch_atomic64_dec_return_acquire)
3355 return arch_atomic64_dec_return_acquire(v);
3356 #elif defined(arch_atomic64_dec_return_relaxed)
3357 s64 ret = arch_atomic64_dec_return_relaxed(v);
3358 __atomic_acquire_fence();
3360 #elif defined(arch_atomic64_dec_return)
3361 return arch_atomic64_dec_return(v);
3363 return raw_atomic64_sub_return_acquire(1, v);
3368 * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3369 * @v: pointer to atomic64_t
3371 * Atomically updates @v to (@v - 1) with release ordering.
3373 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3375 * Return: The updated value of @v.
3377 static __always_inline s64
3378 raw_atomic64_dec_return_release(atomic64_t *v)
3380 #if defined(arch_atomic64_dec_return_release)
3381 return arch_atomic64_dec_return_release(v);
3382 #elif defined(arch_atomic64_dec_return_relaxed)
3383 __atomic_release_fence();
3384 return arch_atomic64_dec_return_relaxed(v);
3385 #elif defined(arch_atomic64_dec_return)
3386 return arch_atomic64_dec_return(v);
3388 return raw_atomic64_sub_return_release(1, v);
3393 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3394 * @v: pointer to atomic64_t
3396 * Atomically updates @v to (@v - 1) with relaxed ordering.
3398 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3400 * Return: The updated value of @v.
3402 static __always_inline s64
3403 raw_atomic64_dec_return_relaxed(atomic64_t *v)
3405 #if defined(arch_atomic64_dec_return_relaxed)
3406 return arch_atomic64_dec_return_relaxed(v);
3407 #elif defined(arch_atomic64_dec_return)
3408 return arch_atomic64_dec_return(v);
3410 return raw_atomic64_sub_return_relaxed(1, v);
3415 * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3416 * @v: pointer to atomic64_t
3418 * Atomically updates @v to (@v - 1) with full ordering.
3420 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3422 * Return: The original value of @v.
3424 static __always_inline s64
3425 raw_atomic64_fetch_dec(atomic64_t *v)
3427 #if defined(arch_atomic64_fetch_dec)
3428 return arch_atomic64_fetch_dec(v);
3429 #elif defined(arch_atomic64_fetch_dec_relaxed)
3431 __atomic_pre_full_fence();
3432 ret = arch_atomic64_fetch_dec_relaxed(v);
3433 __atomic_post_full_fence();
3436 return raw_atomic64_fetch_sub(1, v);
3441 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3442 * @v: pointer to atomic64_t
3444 * Atomically updates @v to (@v - 1) with acquire ordering.
3446 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3448 * Return: The original value of @v.
3450 static __always_inline s64
3451 raw_atomic64_fetch_dec_acquire(atomic64_t *v)
3453 #if defined(arch_atomic64_fetch_dec_acquire)
3454 return arch_atomic64_fetch_dec_acquire(v);
3455 #elif defined(arch_atomic64_fetch_dec_relaxed)
3456 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3457 __atomic_acquire_fence();
3459 #elif defined(arch_atomic64_fetch_dec)
3460 return arch_atomic64_fetch_dec(v);
3462 return raw_atomic64_fetch_sub_acquire(1, v);
3467 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3468 * @v: pointer to atomic64_t
3470 * Atomically updates @v to (@v - 1) with release ordering.
3472 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3474 * Return: The original value of @v.
3476 static __always_inline s64
3477 raw_atomic64_fetch_dec_release(atomic64_t *v)
3479 #if defined(arch_atomic64_fetch_dec_release)
3480 return arch_atomic64_fetch_dec_release(v);
3481 #elif defined(arch_atomic64_fetch_dec_relaxed)
3482 __atomic_release_fence();
3483 return arch_atomic64_fetch_dec_relaxed(v);
3484 #elif defined(arch_atomic64_fetch_dec)
3485 return arch_atomic64_fetch_dec(v);
3487 return raw_atomic64_fetch_sub_release(1, v);
3492 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3493 * @v: pointer to atomic64_t
3495 * Atomically updates @v to (@v - 1) with relaxed ordering.
3497 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3499 * Return: The original value of @v.
3501 static __always_inline s64
3502 raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
3504 #if defined(arch_atomic64_fetch_dec_relaxed)
3505 return arch_atomic64_fetch_dec_relaxed(v);
3506 #elif defined(arch_atomic64_fetch_dec)
3507 return arch_atomic64_fetch_dec(v);
3509 return raw_atomic64_fetch_sub_relaxed(1, v);
3514 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
3516 * @v: pointer to atomic64_t
3518 * Atomically updates @v to (@v & @i) with relaxed ordering.
3520 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3524 static __always_inline void
3525 raw_atomic64_and(s64 i, atomic64_t *v)
3527 arch_atomic64_and(i, v);
3531 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3533 * @v: pointer to atomic64_t
3535 * Atomically updates @v to (@v & @i) with full ordering.
3537 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3539 * Return: The original value of @v.
3541 static __always_inline s64
3542 raw_atomic64_fetch_and(s64 i, atomic64_t *v)
3544 #if defined(arch_atomic64_fetch_and)
3545 return arch_atomic64_fetch_and(i, v);
3546 #elif defined(arch_atomic64_fetch_and_relaxed)
3548 __atomic_pre_full_fence();
3549 ret = arch_atomic64_fetch_and_relaxed(i, v);
3550 __atomic_post_full_fence();
3553 #error "Unable to define raw_atomic64_fetch_and"
3558 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3560 * @v: pointer to atomic64_t
3562 * Atomically updates @v to (@v & @i) with acquire ordering.
3564 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
3566 * Return: The original value of @v.
3568 static __always_inline s64
3569 raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
3571 #if defined(arch_atomic64_fetch_and_acquire)
3572 return arch_atomic64_fetch_and_acquire(i, v);
3573 #elif defined(arch_atomic64_fetch_and_relaxed)
3574 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
3575 __atomic_acquire_fence();
3577 #elif defined(arch_atomic64_fetch_and)
3578 return arch_atomic64_fetch_and(i, v);
3580 #error "Unable to define raw_atomic64_fetch_and_acquire"
3585 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3587 * @v: pointer to atomic64_t
3589 * Atomically updates @v to (@v & @i) with release ordering.
3591 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3593 * Return: The original value of @v.
3595 static __always_inline s64
3596 raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
3598 #if defined(arch_atomic64_fetch_and_release)
3599 return arch_atomic64_fetch_and_release(i, v);
3600 #elif defined(arch_atomic64_fetch_and_relaxed)
3601 __atomic_release_fence();
3602 return arch_atomic64_fetch_and_relaxed(i, v);
3603 #elif defined(arch_atomic64_fetch_and)
3604 return arch_atomic64_fetch_and(i, v);
3606 #error "Unable to define raw_atomic64_fetch_and_release"
3611 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3613 * @v: pointer to atomic64_t
3615 * Atomically updates @v to (@v & @i) with relaxed ordering.
3617 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3619 * Return: The original value of @v.
3621 static __always_inline s64
3622 raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
3624 #if defined(arch_atomic64_fetch_and_relaxed)
3625 return arch_atomic64_fetch_and_relaxed(i, v);
3626 #elif defined(arch_atomic64_fetch_and)
3627 return arch_atomic64_fetch_and(i, v);
3629 #error "Unable to define raw_atomic64_fetch_and_relaxed"
3634 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
3636 * @v: pointer to atomic64_t
3638 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3640 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3644 static __always_inline void
3645 raw_atomic64_andnot(s64 i, atomic64_t *v)
3647 #if defined(arch_atomic64_andnot)
3648 arch_atomic64_andnot(i, v);
3650 raw_atomic64_and(~i, v);
3655 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3657 * @v: pointer to atomic64_t
3659 * Atomically updates @v to (@v & ~@i) with full ordering.
3661 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3663 * Return: The original value of @v.
3665 static __always_inline s64
3666 raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
3668 #if defined(arch_atomic64_fetch_andnot)
3669 return arch_atomic64_fetch_andnot(i, v);
3670 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3672 __atomic_pre_full_fence();
3673 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3674 __atomic_post_full_fence();
3677 return raw_atomic64_fetch_and(~i, v);
3682 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3684 * @v: pointer to atomic64_t
3686 * Atomically updates @v to (@v & ~@i) with acquire ordering.
3688 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3690 * Return: The original value of @v.
3692 static __always_inline s64
3693 raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
3695 #if defined(arch_atomic64_fetch_andnot_acquire)
3696 return arch_atomic64_fetch_andnot_acquire(i, v);
3697 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3698 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3699 __atomic_acquire_fence();
3701 #elif defined(arch_atomic64_fetch_andnot)
3702 return arch_atomic64_fetch_andnot(i, v);
3704 return raw_atomic64_fetch_and_acquire(~i, v);
3709 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3711 * @v: pointer to atomic64_t
3713 * Atomically updates @v to (@v & ~@i) with release ordering.
3715 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3717 * Return: The original value of @v.
3719 static __always_inline s64
3720 raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
3722 #if defined(arch_atomic64_fetch_andnot_release)
3723 return arch_atomic64_fetch_andnot_release(i, v);
3724 #elif defined(arch_atomic64_fetch_andnot_relaxed)
3725 __atomic_release_fence();
3726 return arch_atomic64_fetch_andnot_relaxed(i, v);
3727 #elif defined(arch_atomic64_fetch_andnot)
3728 return arch_atomic64_fetch_andnot(i, v);
3730 return raw_atomic64_fetch_and_release(~i, v);
3735 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
3737 * @v: pointer to atomic64_t
3739 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3741 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
3743 * Return: The original value of @v.
3745 static __always_inline s64
3746 raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
3748 #if defined(arch_atomic64_fetch_andnot_relaxed)
3749 return arch_atomic64_fetch_andnot_relaxed(i, v);
3750 #elif defined(arch_atomic64_fetch_andnot)
3751 return arch_atomic64_fetch_andnot(i, v);
3753 return raw_atomic64_fetch_and_relaxed(~i, v);
3758 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
3760 * @v: pointer to atomic64_t
3762 * Atomically updates @v to (@v | @i) with relaxed ordering.
3764 * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
3768 static __always_inline void
3769 raw_atomic64_or(s64 i, atomic64_t *v)
3771 arch_atomic64_or(i, v);
3775 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
3777 * @v: pointer to atomic64_t
3779 * Atomically updates @v to (@v | @i) with full ordering.
3781 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
3783 * Return: The original value of @v.
3785 static __always_inline s64
3786 raw_atomic64_fetch_or(s64 i, atomic64_t *v)
3788 #if defined(arch_atomic64_fetch_or)
3789 return arch_atomic64_fetch_or(i, v);
3790 #elif defined(arch_atomic64_fetch_or_relaxed)
3792 __atomic_pre_full_fence();
3793 ret = arch_atomic64_fetch_or_relaxed(i, v);
3794 __atomic_post_full_fence();
3797 #error "Unable to define raw_atomic64_fetch_or"
3802 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
3804 * @v: pointer to atomic64_t
3806 * Atomically updates @v to (@v | @i) with acquire ordering.
3808 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
3810 * Return: The original value of @v.
3812 static __always_inline s64
3813 raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
3815 #if defined(arch_atomic64_fetch_or_acquire)
3816 return arch_atomic64_fetch_or_acquire(i, v);
3817 #elif defined(arch_atomic64_fetch_or_relaxed)
3818 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
3819 __atomic_acquire_fence();
3821 #elif defined(arch_atomic64_fetch_or)
3822 return arch_atomic64_fetch_or(i, v);
3824 #error "Unable to define raw_atomic64_fetch_or_acquire"
3829 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
3831 * @v: pointer to atomic64_t
3833 * Atomically updates @v to (@v | @i) with release ordering.
3835 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
3837 * Return: The original value of @v.
3839 static __always_inline s64
3840 raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
3842 #if defined(arch_atomic64_fetch_or_release)
3843 return arch_atomic64_fetch_or_release(i, v);
3844 #elif defined(arch_atomic64_fetch_or_relaxed)
3845 __atomic_release_fence();
3846 return arch_atomic64_fetch_or_relaxed(i, v);
3847 #elif defined(arch_atomic64_fetch_or)
3848 return arch_atomic64_fetch_or(i, v);
3850 #error "Unable to define raw_atomic64_fetch_or_release"
3855 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
3857 * @v: pointer to atomic64_t
3859 * Atomically updates @v to (@v | @i) with relaxed ordering.
3861 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
3863 * Return: The original value of @v.
3865 static __always_inline s64
3866 raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
3868 #if defined(arch_atomic64_fetch_or_relaxed)
3869 return arch_atomic64_fetch_or_relaxed(i, v);
3870 #elif defined(arch_atomic64_fetch_or)
3871 return arch_atomic64_fetch_or(i, v);
3873 #error "Unable to define raw_atomic64_fetch_or_relaxed"
3878 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
3880 * @v: pointer to atomic64_t
3882 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3884 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
3888 static __always_inline void
3889 raw_atomic64_xor(s64 i, atomic64_t *v)
3891 arch_atomic64_xor(i, v);
3895 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
3897 * @v: pointer to atomic64_t
3899 * Atomically updates @v to (@v ^ @i) with full ordering.
3901 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
3903 * Return: The original value of @v.
3905 static __always_inline s64
3906 raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
3908 #if defined(arch_atomic64_fetch_xor)
3909 return arch_atomic64_fetch_xor(i, v);
3910 #elif defined(arch_atomic64_fetch_xor_relaxed)
3912 __atomic_pre_full_fence();
3913 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3914 __atomic_post_full_fence();
3917 #error "Unable to define raw_atomic64_fetch_xor"
3922 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
3924 * @v: pointer to atomic64_t
3926 * Atomically updates @v to (@v ^ @i) with acquire ordering.
3928 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
3930 * Return: The original value of @v.
3932 static __always_inline s64
3933 raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
3935 #if defined(arch_atomic64_fetch_xor_acquire)
3936 return arch_atomic64_fetch_xor_acquire(i, v);
3937 #elif defined(arch_atomic64_fetch_xor_relaxed)
3938 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3939 __atomic_acquire_fence();
3941 #elif defined(arch_atomic64_fetch_xor)
3942 return arch_atomic64_fetch_xor(i, v);
3944 #error "Unable to define raw_atomic64_fetch_xor_acquire"
3949 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
3951 * @v: pointer to atomic64_t
3953 * Atomically updates @v to (@v ^ @i) with release ordering.
3955 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
3957 * Return: The original value of @v.
3959 static __always_inline s64
3960 raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
3962 #if defined(arch_atomic64_fetch_xor_release)
3963 return arch_atomic64_fetch_xor_release(i, v);
3964 #elif defined(arch_atomic64_fetch_xor_relaxed)
3965 __atomic_release_fence();
3966 return arch_atomic64_fetch_xor_relaxed(i, v);
3967 #elif defined(arch_atomic64_fetch_xor)
3968 return arch_atomic64_fetch_xor(i, v);
3970 #error "Unable to define raw_atomic64_fetch_xor_release"
3975 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
3977 * @v: pointer to atomic64_t
3979 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3981 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
3983 * Return: The original value of @v.
3985 static __always_inline s64
3986 raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
3988 #if defined(arch_atomic64_fetch_xor_relaxed)
3989 return arch_atomic64_fetch_xor_relaxed(i, v);
3990 #elif defined(arch_atomic64_fetch_xor)
3991 return arch_atomic64_fetch_xor(i, v);
3993 #error "Unable to define raw_atomic64_fetch_xor_relaxed"
3998 * raw_atomic64_xchg() - atomic exchange with full ordering
3999 * @v: pointer to atomic64_t
4000 * @new: s64 value to assign
4002 * Atomically updates @v to @new with full ordering.
4004 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
4006 * Return: The original value of @v.
4008 static __always_inline s64
4009 raw_atomic64_xchg(atomic64_t *v, s64 new)
4011 #if defined(arch_atomic64_xchg)
4012 return arch_atomic64_xchg(v, new);
4013 #elif defined(arch_atomic64_xchg_relaxed)
4015 __atomic_pre_full_fence();
4016 ret = arch_atomic64_xchg_relaxed(v, new);
4017 __atomic_post_full_fence();
4020 return raw_xchg(&v->counter, new);
4025 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
4026 * @v: pointer to atomic64_t
4027 * @new: s64 value to assign
4029 * Atomically updates @v to @new with acquire ordering.
4031 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
4033 * Return: The original value of @v.
4035 static __always_inline s64
4036 raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
4038 #if defined(arch_atomic64_xchg_acquire)
4039 return arch_atomic64_xchg_acquire(v, new);
4040 #elif defined(arch_atomic64_xchg_relaxed)
4041 s64 ret = arch_atomic64_xchg_relaxed(v, new);
4042 __atomic_acquire_fence();
4044 #elif defined(arch_atomic64_xchg)
4045 return arch_atomic64_xchg(v, new);
4047 return raw_xchg_acquire(&v->counter, new);
4052 * raw_atomic64_xchg_release() - atomic exchange with release ordering
4053 * @v: pointer to atomic64_t
4054 * @new: s64 value to assign
4056 * Atomically updates @v to @new with release ordering.
4058 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
4060 * Return: The original value of @v.
4062 static __always_inline s64
4063 raw_atomic64_xchg_release(atomic64_t *v, s64 new)
4065 #if defined(arch_atomic64_xchg_release)
4066 return arch_atomic64_xchg_release(v, new);
4067 #elif defined(arch_atomic64_xchg_relaxed)
4068 __atomic_release_fence();
4069 return arch_atomic64_xchg_relaxed(v, new);
4070 #elif defined(arch_atomic64_xchg)
4071 return arch_atomic64_xchg(v, new);
4073 return raw_xchg_release(&v->counter, new);
4078 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
4079 * @v: pointer to atomic64_t
4080 * @new: s64 value to assign
4082 * Atomically updates @v to @new with relaxed ordering.
4084 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
4086 * Return: The original value of @v.
4088 static __always_inline s64
4089 raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
4091 #if defined(arch_atomic64_xchg_relaxed)
4092 return arch_atomic64_xchg_relaxed(v, new);
4093 #elif defined(arch_atomic64_xchg)
4094 return arch_atomic64_xchg(v, new);
4096 return raw_xchg_relaxed(&v->counter, new);
4101 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
4102 * @v: pointer to atomic64_t
4103 * @old: s64 value to compare with
4104 * @new: s64 value to assign
4106 * If (@v == @old), atomically updates @v to @new with full ordering.
4108 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4110 * Return: The original value of @v.
4112 static __always_inline s64
4113 raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
4115 #if defined(arch_atomic64_cmpxchg)
4116 return arch_atomic64_cmpxchg(v, old, new);
4117 #elif defined(arch_atomic64_cmpxchg_relaxed)
4119 __atomic_pre_full_fence();
4120 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4121 __atomic_post_full_fence();
4124 return raw_cmpxchg(&v->counter, old, new);
4129 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4130 * @v: pointer to atomic64_t
4131 * @old: s64 value to compare with
4132 * @new: s64 value to assign
4134 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4136 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4138 * Return: The original value of @v.
4140 static __always_inline s64
4141 raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
4143 #if defined(arch_atomic64_cmpxchg_acquire)
4144 return arch_atomic64_cmpxchg_acquire(v, old, new);
4145 #elif defined(arch_atomic64_cmpxchg_relaxed)
4146 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4147 __atomic_acquire_fence();
4149 #elif defined(arch_atomic64_cmpxchg)
4150 return arch_atomic64_cmpxchg(v, old, new);
4152 return raw_cmpxchg_acquire(&v->counter, old, new);
4157 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
4158 * @v: pointer to atomic64_t
4159 * @old: s64 value to compare with
4160 * @new: s64 value to assign
4162 * If (@v == @old), atomically updates @v to @new with release ordering.
4164 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4166 * Return: The original value of @v.
4168 static __always_inline s64
4169 raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
4171 #if defined(arch_atomic64_cmpxchg_release)
4172 return arch_atomic64_cmpxchg_release(v, old, new);
4173 #elif defined(arch_atomic64_cmpxchg_relaxed)
4174 __atomic_release_fence();
4175 return arch_atomic64_cmpxchg_relaxed(v, old, new);
4176 #elif defined(arch_atomic64_cmpxchg)
4177 return arch_atomic64_cmpxchg(v, old, new);
4179 return raw_cmpxchg_release(&v->counter, old, new);
4184 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4185 * @v: pointer to atomic64_t
4186 * @old: s64 value to compare with
4187 * @new: s64 value to assign
4189 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4191 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4193 * Return: The original value of @v.
4195 static __always_inline s64
4196 raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
4198 #if defined(arch_atomic64_cmpxchg_relaxed)
4199 return arch_atomic64_cmpxchg_relaxed(v, old, new);
4200 #elif defined(arch_atomic64_cmpxchg)
4201 return arch_atomic64_cmpxchg(v, old, new);
4203 return raw_cmpxchg_relaxed(&v->counter, old, new);
4208 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
4209 * @v: pointer to atomic64_t
4210 * @old: pointer to s64 value to compare with
4211 * @new: s64 value to assign
4213 * If (@v == @old), atomically updates @v to @new with full ordering.
4214 * Otherwise, updates @old to the current value of @v.
4216 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4218 * Return: @true if the exchange occured, @false otherwise.
4220 static __always_inline bool
4221 raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
4223 #if defined(arch_atomic64_try_cmpxchg)
4224 return arch_atomic64_try_cmpxchg(v, old, new);
4225 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4227 __atomic_pre_full_fence();
4228 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4229 __atomic_post_full_fence();
4233 r = raw_atomic64_cmpxchg(v, o, new);
4234 if (unlikely(r != o))
4236 return likely(r == o);
4241 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4242 * @v: pointer to atomic64_t
4243 * @old: pointer to s64 value to compare with
4244 * @new: s64 value to assign
4246 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4247 * Otherwise, updates @old to the current value of @v.
4249 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4251 * Return: @true if the exchange occured, @false otherwise.
4253 static __always_inline bool
4254 raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
4256 #if defined(arch_atomic64_try_cmpxchg_acquire)
4257 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
4258 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4259 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4260 __atomic_acquire_fence();
4262 #elif defined(arch_atomic64_try_cmpxchg)
4263 return arch_atomic64_try_cmpxchg(v, old, new);
4266 r = raw_atomic64_cmpxchg_acquire(v, o, new);
4267 if (unlikely(r != o))
4269 return likely(r == o);
4274 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
4275 * @v: pointer to atomic64_t
4276 * @old: pointer to s64 value to compare with
4277 * @new: s64 value to assign
4279 * If (@v == @old), atomically updates @v to @new with release ordering.
4280 * Otherwise, updates @old to the current value of @v.
4282 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4284 * Return: @true if the exchange occured, @false otherwise.
4286 static __always_inline bool
4287 raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
4289 #if defined(arch_atomic64_try_cmpxchg_release)
4290 return arch_atomic64_try_cmpxchg_release(v, old, new);
4291 #elif defined(arch_atomic64_try_cmpxchg_relaxed)
4292 __atomic_release_fence();
4293 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4294 #elif defined(arch_atomic64_try_cmpxchg)
4295 return arch_atomic64_try_cmpxchg(v, old, new);
4298 r = raw_atomic64_cmpxchg_release(v, o, new);
4299 if (unlikely(r != o))
4301 return likely(r == o);
4306 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4307 * @v: pointer to atomic64_t
4308 * @old: pointer to s64 value to compare with
4309 * @new: s64 value to assign
4311 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4312 * Otherwise, updates @old to the current value of @v.
4314 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4316 * Return: @true if the exchange occured, @false otherwise.
4318 static __always_inline bool
4319 raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
4321 #if defined(arch_atomic64_try_cmpxchg_relaxed)
4322 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4323 #elif defined(arch_atomic64_try_cmpxchg)
4324 return arch_atomic64_try_cmpxchg(v, old, new);
4327 r = raw_atomic64_cmpxchg_relaxed(v, o, new);
4328 if (unlikely(r != o))
4330 return likely(r == o);
4335 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
4336 * @i: s64 value to add
4337 * @v: pointer to atomic64_t
4339 * Atomically updates @v to (@v - @i) with full ordering.
4341 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
4343 * Return: @true if the resulting value of @v is zero, @false otherwise.
4345 static __always_inline bool
4346 raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
4348 #if defined(arch_atomic64_sub_and_test)
4349 return arch_atomic64_sub_and_test(i, v);
4351 return raw_atomic64_sub_return(i, v) == 0;
4356 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
4357 * @v: pointer to atomic64_t
4359 * Atomically updates @v to (@v - 1) with full ordering.
4361 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
4363 * Return: @true if the resulting value of @v is zero, @false otherwise.
4365 static __always_inline bool
4366 raw_atomic64_dec_and_test(atomic64_t *v)
4368 #if defined(arch_atomic64_dec_and_test)
4369 return arch_atomic64_dec_and_test(v);
4371 return raw_atomic64_dec_return(v) == 0;
4376 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
4377 * @v: pointer to atomic64_t
4379 * Atomically updates @v to (@v + 1) with full ordering.
4381 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
4383 * Return: @true if the resulting value of @v is zero, @false otherwise.
4385 static __always_inline bool
4386 raw_atomic64_inc_and_test(atomic64_t *v)
4388 #if defined(arch_atomic64_inc_and_test)
4389 return arch_atomic64_inc_and_test(v);
4391 return raw_atomic64_inc_return(v) == 0;
4396 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
4397 * @i: s64 value to add
4398 * @v: pointer to atomic64_t
4400 * Atomically updates @v to (@v + @i) with full ordering.
4402 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
4404 * Return: @true if the resulting value of @v is negative, @false otherwise.
4406 static __always_inline bool
4407 raw_atomic64_add_negative(s64 i, atomic64_t *v)
4409 #if defined(arch_atomic64_add_negative)
4410 return arch_atomic64_add_negative(i, v);
4411 #elif defined(arch_atomic64_add_negative_relaxed)
4413 __atomic_pre_full_fence();
4414 ret = arch_atomic64_add_negative_relaxed(i, v);
4415 __atomic_post_full_fence();
4418 return raw_atomic64_add_return(i, v) < 0;
4423 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
4424 * @i: s64 value to add
4425 * @v: pointer to atomic64_t
4427 * Atomically updates @v to (@v + @i) with acquire ordering.
4429 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
4431 * Return: @true if the resulting value of @v is negative, @false otherwise.
4433 static __always_inline bool
4434 raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
4436 #if defined(arch_atomic64_add_negative_acquire)
4437 return arch_atomic64_add_negative_acquire(i, v);
4438 #elif defined(arch_atomic64_add_negative_relaxed)
4439 bool ret = arch_atomic64_add_negative_relaxed(i, v);
4440 __atomic_acquire_fence();
4442 #elif defined(arch_atomic64_add_negative)
4443 return arch_atomic64_add_negative(i, v);
4445 return raw_atomic64_add_return_acquire(i, v) < 0;
4450 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
4451 * @i: s64 value to add
4452 * @v: pointer to atomic64_t
4454 * Atomically updates @v to (@v + @i) with release ordering.
4456 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
4458 * Return: @true if the resulting value of @v is negative, @false otherwise.
4460 static __always_inline bool
4461 raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
4463 #if defined(arch_atomic64_add_negative_release)
4464 return arch_atomic64_add_negative_release(i, v);
4465 #elif defined(arch_atomic64_add_negative_relaxed)
4466 __atomic_release_fence();
4467 return arch_atomic64_add_negative_relaxed(i, v);
4468 #elif defined(arch_atomic64_add_negative)
4469 return arch_atomic64_add_negative(i, v);
4471 return raw_atomic64_add_return_release(i, v) < 0;
4476 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4477 * @i: s64 value to add
4478 * @v: pointer to atomic64_t
4480 * Atomically updates @v to (@v + @i) with relaxed ordering.
4482 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
4484 * Return: @true if the resulting value of @v is negative, @false otherwise.
4486 static __always_inline bool
4487 raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
4489 #if defined(arch_atomic64_add_negative_relaxed)
4490 return arch_atomic64_add_negative_relaxed(i, v);
4491 #elif defined(arch_atomic64_add_negative)
4492 return arch_atomic64_add_negative(i, v);
4494 return raw_atomic64_add_return_relaxed(i, v) < 0;
4499 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
4500 * @v: pointer to atomic64_t
4501 * @a: s64 value to add
4502 * @u: s64 value to compare with
4504 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4506 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4508 * Return: The original value of @v.
4510 static __always_inline s64
4511 raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
4513 #if defined(arch_atomic64_fetch_add_unless)
4514 return arch_atomic64_fetch_add_unless(v, a, u);
4516 s64 c = raw_atomic64_read(v);
4519 if (unlikely(c == u))
4521 } while (!raw_atomic64_try_cmpxchg(v, &c, c + a));
4528 * raw_atomic64_add_unless() - atomic add unless value with full ordering
4529 * @v: pointer to atomic64_t
4530 * @a: s64 value to add
4531 * @u: s64 value to compare with
4533 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4535 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4537 * Return: @true if @v was updated, @false otherwise.
4539 static __always_inline bool
4540 raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
4542 #if defined(arch_atomic64_add_unless)
4543 return arch_atomic64_add_unless(v, a, u);
4545 return raw_atomic64_fetch_add_unless(v, a, u) != u;
4550 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
4551 * @v: pointer to atomic64_t
4553 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4555 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4557 * Return: @true if @v was updated, @false otherwise.
4559 static __always_inline bool
4560 raw_atomic64_inc_not_zero(atomic64_t *v)
4562 #if defined(arch_atomic64_inc_not_zero)
4563 return arch_atomic64_inc_not_zero(v);
4565 return raw_atomic64_add_unless(v, 1, 0);
4570 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
4571 * @v: pointer to atomic64_t
4573 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4575 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4577 * Return: @true if @v was updated, @false otherwise.
4579 static __always_inline bool
4580 raw_atomic64_inc_unless_negative(atomic64_t *v)
4582 #if defined(arch_atomic64_inc_unless_negative)
4583 return arch_atomic64_inc_unless_negative(v);
4585 s64 c = raw_atomic64_read(v);
4588 if (unlikely(c < 0))
4590 } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));
4597 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
4598 * @v: pointer to atomic64_t
4600 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4602 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4604 * Return: @true if @v was updated, @false otherwise.
4606 static __always_inline bool
4607 raw_atomic64_dec_unless_positive(atomic64_t *v)
4609 #if defined(arch_atomic64_dec_unless_positive)
4610 return arch_atomic64_dec_unless_positive(v);
4612 s64 c = raw_atomic64_read(v);
4615 if (unlikely(c > 0))
4617 } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));
4624 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
4625 * @v: pointer to atomic64_t
4627 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4629 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4631 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4633 static __always_inline s64
4634 raw_atomic64_dec_if_positive(atomic64_t *v)
4636 #if defined(arch_atomic64_dec_if_positive)
4637 return arch_atomic64_dec_if_positive(v);
4639 s64 dec, c = raw_atomic64_read(v);
4643 if (unlikely(dec < 0))
4645 } while (!raw_atomic64_try_cmpxchg(v, &c, dec));
4651 #endif /* _LINUX_ATOMIC_FALLBACK_H */
4652 // 2fdd6702823fa842f9cea57a002e6e4476ae780c