Merge tag 'regulator-fix-v5.17-rc5' of git://git.kernel.org/pub/scm/linux/kernel...
[platform/kernel/linux-starfive.git] / lib / atomic64.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Generic implementation of 64-bit atomics using spinlocks,
4  * useful on processors that don't have 64-bit atomic instructions.
5  *
6  * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7  */
8 #include <linux/types.h>
9 #include <linux/cache.h>
10 #include <linux/spinlock.h>
11 #include <linux/init.h>
12 #include <linux/export.h>
13 #include <linux/atomic.h>
14
15 /*
16  * We use a hashed array of spinlocks to provide exclusive access
17  * to each atomic64_t variable.  Since this is expected to used on
18  * systems with small numbers of CPUs (<= 4 or so), we use a
19  * relatively small array of 16 spinlocks to avoid wasting too much
20  * memory on the spinlock array.
21  */
22 #define NR_LOCKS        16
23
24 /*
25  * Ensure each lock is in a separate cacheline.
26  */
27 static union {
28         raw_spinlock_t lock;
29         char pad[L1_CACHE_BYTES];
30 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
31         [0 ... (NR_LOCKS - 1)] = {
32                 .lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
33         },
34 };
35
36 static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
37 {
38         unsigned long addr = (unsigned long) v;
39
40         addr >>= L1_CACHE_SHIFT;
41         addr ^= (addr >> 8) ^ (addr >> 16);
42         return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
43 }
44
45 s64 generic_atomic64_read(const atomic64_t *v)
46 {
47         unsigned long flags;
48         raw_spinlock_t *lock = lock_addr(v);
49         s64 val;
50
51         raw_spin_lock_irqsave(lock, flags);
52         val = v->counter;
53         raw_spin_unlock_irqrestore(lock, flags);
54         return val;
55 }
56 EXPORT_SYMBOL(generic_atomic64_read);
57
58 void generic_atomic64_set(atomic64_t *v, s64 i)
59 {
60         unsigned long flags;
61         raw_spinlock_t *lock = lock_addr(v);
62
63         raw_spin_lock_irqsave(lock, flags);
64         v->counter = i;
65         raw_spin_unlock_irqrestore(lock, flags);
66 }
67 EXPORT_SYMBOL(generic_atomic64_set);
68
69 #define ATOMIC64_OP(op, c_op)                                           \
70 void generic_atomic64_##op(s64 a, atomic64_t *v)                        \
71 {                                                                       \
72         unsigned long flags;                                            \
73         raw_spinlock_t *lock = lock_addr(v);                            \
74                                                                         \
75         raw_spin_lock_irqsave(lock, flags);                             \
76         v->counter c_op a;                                              \
77         raw_spin_unlock_irqrestore(lock, flags);                        \
78 }                                                                       \
79 EXPORT_SYMBOL(generic_atomic64_##op);
80
81 #define ATOMIC64_OP_RETURN(op, c_op)                                    \
82 s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v)                \
83 {                                                                       \
84         unsigned long flags;                                            \
85         raw_spinlock_t *lock = lock_addr(v);                            \
86         s64 val;                                                        \
87                                                                         \
88         raw_spin_lock_irqsave(lock, flags);                             \
89         val = (v->counter c_op a);                                      \
90         raw_spin_unlock_irqrestore(lock, flags);                        \
91         return val;                                                     \
92 }                                                                       \
93 EXPORT_SYMBOL(generic_atomic64_##op##_return);
94
95 #define ATOMIC64_FETCH_OP(op, c_op)                                     \
96 s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v)                   \
97 {                                                                       \
98         unsigned long flags;                                            \
99         raw_spinlock_t *lock = lock_addr(v);                            \
100         s64 val;                                                        \
101                                                                         \
102         raw_spin_lock_irqsave(lock, flags);                             \
103         val = v->counter;                                               \
104         v->counter c_op a;                                              \
105         raw_spin_unlock_irqrestore(lock, flags);                        \
106         return val;                                                     \
107 }                                                                       \
108 EXPORT_SYMBOL(generic_atomic64_fetch_##op);
109
110 #define ATOMIC64_OPS(op, c_op)                                          \
111         ATOMIC64_OP(op, c_op)                                           \
112         ATOMIC64_OP_RETURN(op, c_op)                                    \
113         ATOMIC64_FETCH_OP(op, c_op)
114
115 ATOMIC64_OPS(add, +=)
116 ATOMIC64_OPS(sub, -=)
117
118 #undef ATOMIC64_OPS
119 #define ATOMIC64_OPS(op, c_op)                                          \
120         ATOMIC64_OP(op, c_op)                                           \
121         ATOMIC64_FETCH_OP(op, c_op)
122
123 ATOMIC64_OPS(and, &=)
124 ATOMIC64_OPS(or, |=)
125 ATOMIC64_OPS(xor, ^=)
126
127 #undef ATOMIC64_OPS
128 #undef ATOMIC64_FETCH_OP
129 #undef ATOMIC64_OP
130
131 s64 generic_atomic64_dec_if_positive(atomic64_t *v)
132 {
133         unsigned long flags;
134         raw_spinlock_t *lock = lock_addr(v);
135         s64 val;
136
137         raw_spin_lock_irqsave(lock, flags);
138         val = v->counter - 1;
139         if (val >= 0)
140                 v->counter = val;
141         raw_spin_unlock_irqrestore(lock, flags);
142         return val;
143 }
144 EXPORT_SYMBOL(generic_atomic64_dec_if_positive);
145
146 s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
147 {
148         unsigned long flags;
149         raw_spinlock_t *lock = lock_addr(v);
150         s64 val;
151
152         raw_spin_lock_irqsave(lock, flags);
153         val = v->counter;
154         if (val == o)
155                 v->counter = n;
156         raw_spin_unlock_irqrestore(lock, flags);
157         return val;
158 }
159 EXPORT_SYMBOL(generic_atomic64_cmpxchg);
160
161 s64 generic_atomic64_xchg(atomic64_t *v, s64 new)
162 {
163         unsigned long flags;
164         raw_spinlock_t *lock = lock_addr(v);
165         s64 val;
166
167         raw_spin_lock_irqsave(lock, flags);
168         val = v->counter;
169         v->counter = new;
170         raw_spin_unlock_irqrestore(lock, flags);
171         return val;
172 }
173 EXPORT_SYMBOL(generic_atomic64_xchg);
174
175 s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
176 {
177         unsigned long flags;
178         raw_spinlock_t *lock = lock_addr(v);
179         s64 val;
180
181         raw_spin_lock_irqsave(lock, flags);
182         val = v->counter;
183         if (val != u)
184                 v->counter += a;
185         raw_spin_unlock_irqrestore(lock, flags);
186
187         return val;
188 }
189 EXPORT_SYMBOL(generic_atomic64_fetch_add_unless);