include/asm-generic/percpu.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_GENERIC_PERCPU_H_
   3 #define _ASM_GENERIC_PERCPU_H_
   4
   5 #include <linux/compiler.h>
   6 #include <linux/threads.h>
   7 #include <linux/percpu-defs.h>
   8
   9 #ifdef CONFIG_SMP
  10
  11 /*
  12  * per_cpu_offset() is the offset that has to be added to a
  13  * percpu variable to get to the instance for a certain processor.
  14  *
  15  * Most arches use the __per_cpu_offset array for those offsets but
  16  * some arches have their own ways of determining the offset (x86_64, s390).
  17  */
  18 #ifndef __per_cpu_offset
  19 extern unsigned long __per_cpu_offset[NR_CPUS];
  20
  21 #define per_cpu_offset(x) (__per_cpu_offset[x])
  22 #endif
  23
  24 /*
  25  * Determine the offset for the currently active processor.
  26  * An arch may define __my_cpu_offset to provide a more effective
  27  * means of obtaining the offset to the per cpu variables of the
  28  * current processor.
  29  */
  30 #ifndef __my_cpu_offset
  31 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
  32 #endif
  33 #ifdef CONFIG_DEBUG_PREEMPT
  34 #define my_cpu_offset per_cpu_offset(smp_processor_id())
  35 #else
  36 #define my_cpu_offset __my_cpu_offset
  37 #endif
  38
  39 /*
  40  * Arch may define arch_raw_cpu_ptr() to provide more efficient address
  41  * translations for raw_cpu_ptr().
  42  */
  43 #ifndef arch_raw_cpu_ptr
  44 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
  45 #endif
  46
  47 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
  48 extern void setup_per_cpu_areas(void);
  49 #endif
  50
  51 #endif  /* SMP */
  52
  53 #ifndef PER_CPU_BASE_SECTION
  54 #ifdef CONFIG_SMP
  55 #define PER_CPU_BASE_SECTION ".data..percpu"
  56 #else
  57 #define PER_CPU_BASE_SECTION ".data"
  58 #endif
  59 #endif
  60
  61 #ifndef PER_CPU_ATTRIBUTES
  62 #define PER_CPU_ATTRIBUTES
  63 #endif
  64
  65 #define raw_cpu_generic_read(pcp)                                       \
  66 ({                                                                      \
  67         *raw_cpu_ptr(&(pcp));                                           \
  68 })
  69
  70 #define raw_cpu_generic_to_op(pcp, val, op)                             \
  71 do {                                                                    \
  72         *raw_cpu_ptr(&(pcp)) op val;                                    \
  73 } while (0)
  74
  75 #define raw_cpu_generic_add_return(pcp, val)                            \
  76 ({                                                                      \
  77         typeof(pcp) *__p = raw_cpu_ptr(&(pcp));                         \
  78                                                                         \
  79         *__p += val;                                                    \
  80         *__p;                                                           \
  81 })
  82
  83 #define raw_cpu_generic_xchg(pcp, nval)                                 \
  84 ({                                                                      \
  85         typeof(pcp) *__p = raw_cpu_ptr(&(pcp));                         \
  86         typeof(pcp) __ret;                                              \
  87         __ret = *__p;                                                   \
  88         *__p = nval;                                                    \
  89         __ret;                                                          \
  90 })
  91
  92 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)                        \
  93 ({                                                                      \
  94         typeof(pcp) *__p = raw_cpu_ptr(&(pcp));                         \
  95         typeof(pcp) __ret;                                              \
  96         __ret = *__p;                                                   \
  97         if (__ret == (oval))                                            \
  98                 *__p = nval;                                            \
  99         __ret;                                                          \
 100 })
 101
 102 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 103 ({                                                                      \
 104         typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1));                      \
 105         typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2));                      \
 106         int __ret = 0;                                                  \
 107         if (*__p1 == (oval1) && *__p2  == (oval2)) {                    \
 108                 *__p1 = nval1;                                          \
 109                 *__p2 = nval2;                                          \
 110                 __ret = 1;                                              \
 111         }                                                               \
 112         (__ret);                                                        \
 113 })
 114
 115 #define __this_cpu_generic_read_nopreempt(pcp)                          \
 116 ({                                                                      \
 117         typeof(pcp) ___ret;                                             \
 118         preempt_disable_notrace();                                      \
 119         ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));                       \
 120         preempt_enable_notrace();                                       \
 121         ___ret;                                                         \
 122 })
 123
 124 #define __this_cpu_generic_read_noirq(pcp)                              \
 125 ({                                                                      \
 126         typeof(pcp) ___ret;                                             \
 127         unsigned long ___flags;                                         \
 128         raw_local_irq_save(___flags);                                   \
 129         ___ret = raw_cpu_generic_read(pcp);                             \
 130         raw_local_irq_restore(___flags);                                \
 131         ___ret;                                                         \
 132 })
 133
 134 #define this_cpu_generic_read(pcp)                                      \
 135 ({                                                                      \
 136         typeof(pcp) __ret;                                              \
 137         if (__native_word(pcp))                                         \
 138                 __ret = __this_cpu_generic_read_nopreempt(pcp);         \
 139         else                                                            \
 140                 __ret = __this_cpu_generic_read_noirq(pcp);             \
 141         __ret;                                                          \
 142 })
 143
 144 #define this_cpu_generic_to_op(pcp, val, op)                            \
 145 do {                                                                    \
 146         unsigned long __flags;                                          \
 147         raw_local_irq_save(__flags);                                    \
 148         raw_cpu_generic_to_op(pcp, val, op);                            \
 149         raw_local_irq_restore(__flags);                                 \
 150 } while (0)
 151
 152
 153 #define this_cpu_generic_add_return(pcp, val)                           \
 154 ({                                                                      \
 155         typeof(pcp) __ret;                                              \
 156         unsigned long __flags;                                          \
 157         raw_local_irq_save(__flags);                                    \
 158         __ret = raw_cpu_generic_add_return(pcp, val);                   \
 159         raw_local_irq_restore(__flags);                                 \
 160         __ret;                                                          \
 161 })
 162
 163 #define this_cpu_generic_xchg(pcp, nval)                                \
 164 ({                                                                      \
 165         typeof(pcp) __ret;                                              \
 166         unsigned long __flags;                                          \
 167         raw_local_irq_save(__flags);                                    \
 168         __ret = raw_cpu_generic_xchg(pcp, nval);                        \
 169         raw_local_irq_restore(__flags);                                 \
 170         __ret;                                                          \
 171 })
 172
 173 #define this_cpu_generic_cmpxchg(pcp, oval, nval)                       \
 174 ({                                                                      \
 175         typeof(pcp) __ret;                                              \
 176         unsigned long __flags;                                          \
 177         raw_local_irq_save(__flags);                                    \
 178         __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);               \
 179         raw_local_irq_restore(__flags);                                 \
 180         __ret;                                                          \
 181 })
 182
 183 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 184 ({                                                                      \
 185         int __ret;                                                      \
 186         unsigned long __flags;                                          \
 187         raw_local_irq_save(__flags);                                    \
 188         __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,              \
 189                         oval1, oval2, nval1, nval2);                    \
 190         raw_local_irq_restore(__flags);                                 \
 191         __ret;                                                          \
 192 })
 193
 194 #ifndef raw_cpu_read_1
 195 #define raw_cpu_read_1(pcp)             raw_cpu_generic_read(pcp)
 196 #endif
 197 #ifndef raw_cpu_read_2
 198 #define raw_cpu_read_2(pcp)             raw_cpu_generic_read(pcp)
 199 #endif
 200 #ifndef raw_cpu_read_4
 201 #define raw_cpu_read_4(pcp)             raw_cpu_generic_read(pcp)
 202 #endif
 203 #ifndef raw_cpu_read_8
 204 #define raw_cpu_read_8(pcp)             raw_cpu_generic_read(pcp)
 205 #endif
 206
 207 #ifndef raw_cpu_write_1
 208 #define raw_cpu_write_1(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 209 #endif
 210 #ifndef raw_cpu_write_2
 211 #define raw_cpu_write_2(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 212 #endif
 213 #ifndef raw_cpu_write_4
 214 #define raw_cpu_write_4(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 215 #endif
 216 #ifndef raw_cpu_write_8
 217 #define raw_cpu_write_8(pcp, val)       raw_cpu_generic_to_op(pcp, val, =)
 218 #endif
 219
 220 #ifndef raw_cpu_add_1
 221 #define raw_cpu_add_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 222 #endif
 223 #ifndef raw_cpu_add_2
 224 #define raw_cpu_add_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 225 #endif
 226 #ifndef raw_cpu_add_4
 227 #define raw_cpu_add_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 228 #endif
 229 #ifndef raw_cpu_add_8
 230 #define raw_cpu_add_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, +=)
 231 #endif
 232
 233 #ifndef raw_cpu_and_1
 234 #define raw_cpu_and_1(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 235 #endif
 236 #ifndef raw_cpu_and_2
 237 #define raw_cpu_and_2(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 238 #endif
 239 #ifndef raw_cpu_and_4
 240 #define raw_cpu_and_4(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 241 #endif
 242 #ifndef raw_cpu_and_8
 243 #define raw_cpu_and_8(pcp, val)         raw_cpu_generic_to_op(pcp, val, &=)
 244 #endif
 245
 246 #ifndef raw_cpu_or_1
 247 #define raw_cpu_or_1(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 248 #endif
 249 #ifndef raw_cpu_or_2
 250 #define raw_cpu_or_2(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 251 #endif
 252 #ifndef raw_cpu_or_4
 253 #define raw_cpu_or_4(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 254 #endif
 255 #ifndef raw_cpu_or_8
 256 #define raw_cpu_or_8(pcp, val)          raw_cpu_generic_to_op(pcp, val, |=)
 257 #endif
 258
 259 #ifndef raw_cpu_add_return_1
 260 #define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 261 #endif
 262 #ifndef raw_cpu_add_return_2
 263 #define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 264 #endif
 265 #ifndef raw_cpu_add_return_4
 266 #define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 267 #endif
 268 #ifndef raw_cpu_add_return_8
 269 #define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
 270 #endif
 271
 272 #ifndef raw_cpu_xchg_1
 273 #define raw_cpu_xchg_1(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 274 #endif
 275 #ifndef raw_cpu_xchg_2
 276 #define raw_cpu_xchg_2(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 277 #endif
 278 #ifndef raw_cpu_xchg_4
 279 #define raw_cpu_xchg_4(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 280 #endif
 281 #ifndef raw_cpu_xchg_8
 282 #define raw_cpu_xchg_8(pcp, nval)       raw_cpu_generic_xchg(pcp, nval)
 283 #endif
 284
 285 #ifndef raw_cpu_cmpxchg_1
 286 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
 287         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 288 #endif
 289 #ifndef raw_cpu_cmpxchg_2
 290 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
 291         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 292 #endif
 293 #ifndef raw_cpu_cmpxchg_4
 294 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
 295         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 296 #endif
 297 #ifndef raw_cpu_cmpxchg_8
 298 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
 299         raw_cpu_generic_cmpxchg(pcp, oval, nval)
 300 #endif
 301
 302 #ifndef raw_cpu_cmpxchg_double_1
 303 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 304         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 305 #endif
 306 #ifndef raw_cpu_cmpxchg_double_2
 307 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 308         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 309 #endif
 310 #ifndef raw_cpu_cmpxchg_double_4
 311 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 312         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 313 #endif
 314 #ifndef raw_cpu_cmpxchg_double_8
 315 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 316         raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 317 #endif
 318
 319 #ifndef this_cpu_read_1
 320 #define this_cpu_read_1(pcp)            this_cpu_generic_read(pcp)
 321 #endif
 322 #ifndef this_cpu_read_2
 323 #define this_cpu_read_2(pcp)            this_cpu_generic_read(pcp)
 324 #endif
 325 #ifndef this_cpu_read_4
 326 #define this_cpu_read_4(pcp)            this_cpu_generic_read(pcp)
 327 #endif
 328 #ifndef this_cpu_read_8
 329 #define this_cpu_read_8(pcp)            this_cpu_generic_read(pcp)
 330 #endif
 331
 332 #ifndef this_cpu_write_1
 333 #define this_cpu_write_1(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 334 #endif
 335 #ifndef this_cpu_write_2
 336 #define this_cpu_write_2(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 337 #endif
 338 #ifndef this_cpu_write_4
 339 #define this_cpu_write_4(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 340 #endif
 341 #ifndef this_cpu_write_8
 342 #define this_cpu_write_8(pcp, val)      this_cpu_generic_to_op(pcp, val, =)
 343 #endif
 344
 345 #ifndef this_cpu_add_1
 346 #define this_cpu_add_1(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 347 #endif
 348 #ifndef this_cpu_add_2
 349 #define this_cpu_add_2(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 350 #endif
 351 #ifndef this_cpu_add_4
 352 #define this_cpu_add_4(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 353 #endif
 354 #ifndef this_cpu_add_8
 355 #define this_cpu_add_8(pcp, val)        this_cpu_generic_to_op(pcp, val, +=)
 356 #endif
 357
 358 #ifndef this_cpu_and_1
 359 #define this_cpu_and_1(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 360 #endif
 361 #ifndef this_cpu_and_2
 362 #define this_cpu_and_2(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 363 #endif
 364 #ifndef this_cpu_and_4
 365 #define this_cpu_and_4(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 366 #endif
 367 #ifndef this_cpu_and_8
 368 #define this_cpu_and_8(pcp, val)        this_cpu_generic_to_op(pcp, val, &=)
 369 #endif
 370
 371 #ifndef this_cpu_or_1
 372 #define this_cpu_or_1(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 373 #endif
 374 #ifndef this_cpu_or_2
 375 #define this_cpu_or_2(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 376 #endif
 377 #ifndef this_cpu_or_4
 378 #define this_cpu_or_4(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 379 #endif
 380 #ifndef this_cpu_or_8
 381 #define this_cpu_or_8(pcp, val)         this_cpu_generic_to_op(pcp, val, |=)
 382 #endif
 383
 384 #ifndef this_cpu_add_return_1
 385 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
 386 #endif
 387 #ifndef this_cpu_add_return_2
 388 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
 389 #endif
 390 #ifndef this_cpu_add_return_4
 391 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
 392 #endif
 393 #ifndef this_cpu_add_return_8
 394 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
 395 #endif
 396
 397 #ifndef this_cpu_xchg_1
 398 #define this_cpu_xchg_1(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 399 #endif
 400 #ifndef this_cpu_xchg_2
 401 #define this_cpu_xchg_2(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 402 #endif
 403 #ifndef this_cpu_xchg_4
 404 #define this_cpu_xchg_4(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 405 #endif
 406 #ifndef this_cpu_xchg_8
 407 #define this_cpu_xchg_8(pcp, nval)      this_cpu_generic_xchg(pcp, nval)
 408 #endif
 409
 410 #ifndef this_cpu_cmpxchg_1
 411 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
 412         this_cpu_generic_cmpxchg(pcp, oval, nval)
 413 #endif
 414 #ifndef this_cpu_cmpxchg_2
 415 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
 416         this_cpu_generic_cmpxchg(pcp, oval, nval)
 417 #endif
 418 #ifndef this_cpu_cmpxchg_4
 419 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
 420         this_cpu_generic_cmpxchg(pcp, oval, nval)
 421 #endif
 422 #ifndef this_cpu_cmpxchg_8
 423 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
 424         this_cpu_generic_cmpxchg(pcp, oval, nval)
 425 #endif
 426
 427 #ifndef this_cpu_cmpxchg_double_1
 428 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 429         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 430 #endif
 431 #ifndef this_cpu_cmpxchg_double_2
 432 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 433         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 434 #endif
 435 #ifndef this_cpu_cmpxchg_double_4
 436 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 437         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 438 #endif
 439 #ifndef this_cpu_cmpxchg_double_8
 440 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
 441         this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
 442 #endif
 443
 444 #endif /* _ASM_GENERIC_PERCPU_H_ */