arch/loongarch/include/asm/percpu.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   4  */
   5 #ifndef __ASM_PERCPU_H
   6 #define __ASM_PERCPU_H
   7
   8 #include <asm/cmpxchg.h>
   9 #include <asm/loongarch.h>
  10
  11 /*
  12  * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
  13  * the loading address of main kernel image, but far from where the modules are
  14  * loaded. Tell the compiler this fact when using explicit relocs.
  15  */
  16 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
  17 # if __has_attribute(model)
  18 #  define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
  19 # else
  20 #  error compiler support for the model attribute is necessary when a recent assembler is used
  21 # endif
  22 #endif
  23
  24 /* Use r21 for fast access */
  25 register unsigned long __my_cpu_offset __asm__("$r21");
  26
  27 static inline void set_my_cpu_offset(unsigned long off)
  28 {
  29         __my_cpu_offset = off;
  30         csr_write64(off, PERCPU_BASE_KS);
  31 }
  32 #define __my_cpu_offset __my_cpu_offset
  33
  34 #define PERCPU_OP(op, asm_op, c_op)                                     \
  35 static __always_inline unsigned long __percpu_##op(void *ptr,           \
  36                         unsigned long val, int size)                    \
  37 {                                                                       \
  38         unsigned long ret;                                              \
  39                                                                         \
  40         switch (size) {                                                 \
  41         case 4:                                                         \
  42                 __asm__ __volatile__(                                   \
  43                 "am"#asm_op".w" " %[ret], %[val], %[ptr]        \n"             \
  44                 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr)           \
  45                 : [val] "r" (val));                                     \
  46                 break;                                                  \
  47         case 8:                                                         \
  48                 __asm__ __volatile__(                                   \
  49                 "am"#asm_op".d" " %[ret], %[val], %[ptr]        \n"             \
  50                 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr)           \
  51                 : [val] "r" (val));                                     \
  52                 break;                                                  \
  53         default:                                                        \
  54                 ret = 0;                                                \
  55                 BUILD_BUG();                                            \
  56         }                                                               \
  57                                                                         \
  58         return ret c_op val;                                            \
  59 }
  60
  61 PERCPU_OP(add, add, +)
  62 PERCPU_OP(and, and, &)
  63 PERCPU_OP(or, or, |)
  64 #undef PERCPU_OP
  65
  66 static __always_inline unsigned long __percpu_read(void *ptr, int size)
  67 {
  68         unsigned long ret;
  69
  70         switch (size) {
  71         case 1:
  72                 __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr]       \n"
  73                 : [ret] "=&r"(ret)
  74                 : [ptr] "r"(ptr)
  75                 : "memory");
  76                 break;
  77         case 2:
  78                 __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr]       \n"
  79                 : [ret] "=&r"(ret)
  80                 : [ptr] "r"(ptr)
  81                 : "memory");
  82                 break;
  83         case 4:
  84                 __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr]       \n"
  85                 : [ret] "=&r"(ret)
  86                 : [ptr] "r"(ptr)
  87                 : "memory");
  88                 break;
  89         case 8:
  90                 __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr]       \n"
  91                 : [ret] "=&r"(ret)
  92                 : [ptr] "r"(ptr)
  93                 : "memory");
  94                 break;
  95         default:
  96                 ret = 0;
  97                 BUILD_BUG();
  98         }
  99
 100         return ret;
 101 }
 102
 103 static __always_inline void __percpu_write(void *ptr, unsigned long val, int size)
 104 {
 105         switch (size) {
 106         case 1:
 107                 __asm__ __volatile__("stx.b %[val], $r21, %[ptr]        \n"
 108                 :
 109                 : [val] "r" (val), [ptr] "r" (ptr)
 110                 : "memory");
 111                 break;
 112         case 2:
 113                 __asm__ __volatile__("stx.h %[val], $r21, %[ptr]        \n"
 114                 :
 115                 : [val] "r" (val), [ptr] "r" (ptr)
 116                 : "memory");
 117                 break;
 118         case 4:
 119                 __asm__ __volatile__("stx.w %[val], $r21, %[ptr]        \n"
 120                 :
 121                 : [val] "r" (val), [ptr] "r" (ptr)
 122                 : "memory");
 123                 break;
 124         case 8:
 125                 __asm__ __volatile__("stx.d %[val], $r21, %[ptr]        \n"
 126                 :
 127                 : [val] "r" (val), [ptr] "r" (ptr)
 128                 : "memory");
 129                 break;
 130         default:
 131                 BUILD_BUG();
 132         }
 133 }
 134
 135 static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
 136                                                    int size)
 137 {
 138         switch (size) {
 139         case 1:
 140         case 2:
 141                 return __xchg_small((volatile void *)ptr, val, size);
 142
 143         case 4:
 144                 return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
 145
 146         case 8:
 147                 return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
 148
 149         default:
 150                 BUILD_BUG();
 151         }
 152
 153         return 0;
 154 }
 155
 156 /* this_cpu_cmpxchg */
 157 #define _protect_cmpxchg_local(pcp, o, n)                       \
 158 ({                                                              \
 159         typeof(*raw_cpu_ptr(&(pcp))) __ret;                     \
 160         preempt_disable_notrace();                              \
 161         __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);       \
 162         preempt_enable_notrace();                               \
 163         __ret;                                                  \
 164 })
 165
 166 #define _percpu_read(pcp)                                               \
 167 ({                                                                      \
 168         typeof(pcp) __retval;                                           \
 169         __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));     \
 170         __retval;                                                       \
 171 })
 172
 173 #define _percpu_write(pcp, val)                                         \
 174 do {                                                                    \
 175         __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp));      \
 176 } while (0)                                                             \
 177
 178 #define _pcp_protect(operation, pcp, val)                       \
 179 ({                                                              \
 180         typeof(pcp) __retval;                                   \
 181         preempt_disable_notrace();                              \
 182         __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),  \
 183                                           (val), sizeof(pcp));  \
 184         preempt_enable_notrace();                               \
 185         __retval;                                               \
 186 })
 187
 188 #define _percpu_add(pcp, val) \
 189         _pcp_protect(__percpu_add, pcp, val)
 190
 191 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
 192
 193 #define _percpu_and(pcp, val) \
 194         _pcp_protect(__percpu_and, pcp, val)
 195
 196 #define _percpu_or(pcp, val) \
 197         _pcp_protect(__percpu_or, pcp, val)
 198
 199 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \
 200         _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
 201
 202 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
 203 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
 204
 205 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
 206 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
 207
 208 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
 209 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
 210
 211 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
 212 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
 213
 214 #define this_cpu_read_1(pcp) _percpu_read(pcp)
 215 #define this_cpu_read_2(pcp) _percpu_read(pcp)
 216 #define this_cpu_read_4(pcp) _percpu_read(pcp)
 217 #define this_cpu_read_8(pcp) _percpu_read(pcp)
 218
 219 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
 220 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
 221 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
 222 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
 223
 224 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
 225 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
 226 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
 227 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
 228
 229 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 230 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 231 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 232 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
 233
 234 #include <asm-generic/percpu.h>
 235
 236 #endif /* __ASM_PERCPU_H */