tools/testing/selftests/rseq/rseq-riscv.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
   4  * other architectures, the ebreak instruction has no immediate field for
   5  * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
   6  * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
   7  * is an uncommon instruction and will raise an illegal instruction
   8  * exception when executed in all modes.
   9  */
  10 #include <endian.h>
  11
  12 #if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
  13 #define RSEQ_SIG   0xf1401073  /* csrr mhartid, x0 */
  14 #else
  15 #error "Currently, RSEQ only supports Little-Endian version"
  16 #endif
  17
  18 #if __riscv_xlen == 64
  19 #define __REG_SEL(a, b) a
  20 #elif __riscv_xlen == 32
  21 #define __REG_SEL(a, b) b
  22 #endif
  23
  24 #define REG_L   __REG_SEL("ld ", "lw ")
  25 #define REG_S   __REG_SEL("sd ", "sw ")
  26
  27 #define RISCV_FENCE(p, s) \
  28         __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
  29 #define rseq_smp_mb()   RISCV_FENCE(rw, rw)
  30 #define rseq_smp_rmb()  RISCV_FENCE(r, r)
  31 #define rseq_smp_wmb()  RISCV_FENCE(w, w)
  32 #define RSEQ_ASM_TMP_REG_1      "t6"
  33 #define RSEQ_ASM_TMP_REG_2      "t5"
  34 #define RSEQ_ASM_TMP_REG_3      "t4"
  35 #define RSEQ_ASM_TMP_REG_4      "t3"
  36
  37 #define rseq_smp_load_acquire(p)                                        \
  38 __extension__ ({                                                        \
  39         __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));                   \
  40         RISCV_FENCE(r, rw)                                              \
  41         ____p1;                                                         \
  42 })
  43
  44 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  45
  46 #define rseq_smp_store_release(p, v)                                    \
  47 do {                                                                    \
  48         RISCV_FENCE(rw, w);                                             \
  49         RSEQ_WRITE_ONCE(*(p), v);                                               \
  50 } while (0)
  51
  52 #ifdef RSEQ_SKIP_FASTPATH
  53 #include "rseq-skip.h"
  54 #else /* !RSEQ_SKIP_FASTPATH */
  55
  56 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,        \
  57                                 post_commit_offset, abort_ip)           \
  58         ".pushsection   __rseq_cs, \"aw\"\n"                            \
  59         ".balign        32\n"                                           \
  60         __rseq_str(label) ":\n"                                         \
  61         ".long  " __rseq_str(version) ", " __rseq_str(flags) "\n"       \
  62         ".quad  " __rseq_str(start_ip) ", "                             \
  63                           __rseq_str(post_commit_offset) ", "           \
  64                           __rseq_str(abort_ip) "\n"                     \
  65         ".popsection\n\t"                                               \
  66         ".pushsection __rseq_cs_ptr_array, \"aw\"\n"                    \
  67         ".quad " __rseq_str(label) "b\n"                                \
  68         ".popsection\n"
  69
  70 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  71         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,               \
  72                                 ((post_commit_ip) - (start_ip)), abort_ip)
  73
  74 /*
  75  * Exit points of a rseq critical section consist of all instructions outside
  76  * of the critical section where a critical section can either branch to or
  77  * reach through the normal course of its execution. The abort IP and the
  78  * post-commit IP are already part of the __rseq_cs section and should not be
  79  * explicitly defined as additional exit points. Knowing all exit points is
  80  * useful to assist debuggers stepping over the critical section.
  81  */
  82 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  83         ".pushsection __rseq_exit_point_array, \"aw\"\n"                \
  84         ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"     \
  85         ".popsection\n"
  86
  87 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  88         RSEQ_INJECT_ASM(1)                                              \
  89         "la     " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n"     \
  90         REG_S   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n"     \
  91         __rseq_str(label) ":\n"
  92
  93 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                       \
  94         "j      222f\n"                                                 \
  95         ".balign        4\n"                                            \
  96         ".long "        __rseq_str(RSEQ_SIG) "\n"                       \
  97         __rseq_str(label) ":\n"                                         \
  98         "j      %l[" __rseq_str(abort_label) "]\n"                      \
  99         "222:\n"
 100
 101 #define RSEQ_ASM_OP_STORE(value, var)                                   \
 102         REG_S   "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 103
 104 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                           \
 105         REG_L   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"         \
 106         "bne    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
 107                   __rseq_str(label) "\n"
 108
 109 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label)                         \
 110         "lw     " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"       \
 111         "bne    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
 112                   __rseq_str(label) "\n"
 113
 114 #define RSEQ_ASM_OP_CMPNE(var, expect, label)                           \
 115         REG_L   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"         \
 116         "beq    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
 117                   __rseq_str(label) "\n"
 118
 119 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 120         RSEQ_INJECT_ASM(2)                                              \
 121         RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
 122
 123 #define RSEQ_ASM_OP_R_LOAD(var)                                         \
 124         REG_L   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
 125
 126 #define RSEQ_ASM_OP_R_STORE(var)                                        \
 127         REG_S   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
 128
 129 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                                  \
 130         "add    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], "    \
 131                  RSEQ_ASM_TMP_REG_1 "\n"                                \
 132         REG_L   RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
 133
 134 #define RSEQ_ASM_OP_R_ADD(count)                                        \
 135         "add    " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1            \
 136                 ", %[" __rseq_str(count) "]\n"
 137
 138 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)          \
 139         RSEQ_ASM_OP_STORE(value, var)                                   \
 140         __rseq_str(post_commit_label) ":\n"
 141
 142 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)  \
 143         "fence  rw, w\n"                                                \
 144         RSEQ_ASM_OP_STORE(value, var)                                   \
 145         __rseq_str(post_commit_label) ":\n"
 146
 147 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)               \
 148         REG_S   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"         \
 149         __rseq_str(post_commit_label) ":\n"
 150
 151 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)                         \
 152         "beqz   %[" __rseq_str(len) "], 333f\n"                         \
 153         "mv     " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n"       \
 154         "mv     " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n"       \
 155         "mv     " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n"       \
 156         "222:\n"                                                        \
 157         "lb     " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n"    \
 158         "sb     " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n"    \
 159         "addi   " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n"   \
 160         "addi   " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n"    \
 161         "addi   " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n"    \
 162         "bnez   " RSEQ_ASM_TMP_REG_1 ", 222b\n"                         \
 163         "333:\n"
 164
 165 #define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label)           \
 166         "mv     " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n"       \
 167         RSEQ_ASM_OP_R_ADD(off)                                          \
 168         REG_L     RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n"    \
 169         RSEQ_ASM_OP_R_ADD(inc)                                          \
 170         __rseq_str(post_commit_label) ":\n"
 171
 172 static inline __always_inline
 173 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 174 {
 175         RSEQ_INJECT_C(9)
 176
 177         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 178                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 179 #ifdef RSEQ_COMPARE_TWICE
 180                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 181                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 182 #endif
 183                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 184                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 185                                   RSEQ_INJECT_ASM(3)
 186                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 187                                   RSEQ_INJECT_ASM(4)
 188 #ifdef RSEQ_COMPARE_TWICE
 189                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 190                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 191 #endif
 192                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 193                                   RSEQ_INJECT_ASM(5)
 194                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 195                                   : /* gcc asm goto does not allow outputs */
 196                                   : [cpu_id]            "r" (cpu),
 197                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 198                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 199                                     [v]                 "m" (*v),
 200                                     [expect]            "r" (expect),
 201                                     [newv]              "r" (newv)
 202                                     RSEQ_INJECT_INPUT
 203                                   : "memory", RSEQ_ASM_TMP_REG_1
 204                                     RSEQ_INJECT_CLOBBER
 205                                   : abort, cmpfail
 206 #ifdef RSEQ_COMPARE_TWICE
 207                                     , error1, error2
 208 #endif
 209         );
 210
 211         return 0;
 212 abort:
 213         RSEQ_INJECT_FAILED
 214         return -1;
 215 cmpfail:
 216         return 1;
 217 #ifdef RSEQ_COMPARE_TWICE
 218 error1:
 219         rseq_bug("cpu_id comparison failed");
 220 error2:
 221         rseq_bug("expected value comparison failed");
 222 #endif
 223 }
 224
 225 static inline __always_inline
 226 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 227                                off_t voffp, intptr_t *load, int cpu)
 228 {
 229         RSEQ_INJECT_C(9)
 230
 231         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 232                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 233 #ifdef RSEQ_COMPARE_TWICE
 234                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 235                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 236 #endif
 237                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 238                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 239                                   RSEQ_INJECT_ASM(3)
 240                                   RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
 241                                   RSEQ_INJECT_ASM(4)
 242 #ifdef RSEQ_COMPARE_TWICE
 243                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 244                                   RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
 245 #endif
 246                                   RSEQ_ASM_OP_R_LOAD(v)
 247                                   RSEQ_ASM_OP_R_STORE(load)
 248                                   RSEQ_ASM_OP_R_LOAD_OFF(voffp)
 249                                   RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 250                                   RSEQ_INJECT_ASM(5)
 251                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 252                                   : /* gcc asm goto does not allow outputs */
 253                                   : [cpu_id]            "r" (cpu),
 254                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 255                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 256                                     [v]                 "m" (*v),
 257                                     [expectnot]         "r" (expectnot),
 258                                     [load]              "m" (*load),
 259                                     [voffp]             "r" (voffp)
 260                                     RSEQ_INJECT_INPUT
 261                                   : "memory", RSEQ_ASM_TMP_REG_1
 262                                     RSEQ_INJECT_CLOBBER
 263                                   : abort, cmpfail
 264 #ifdef RSEQ_COMPARE_TWICE
 265                                     , error1, error2
 266 #endif
 267         );
 268         return 0;
 269 abort:
 270         RSEQ_INJECT_FAILED
 271         return -1;
 272 cmpfail:
 273         return 1;
 274 #ifdef RSEQ_COMPARE_TWICE
 275 error1:
 276         rseq_bug("cpu_id comparison failed");
 277 error2:
 278         rseq_bug("expected value comparison failed");
 279 #endif
 280 }
 281
 282 static inline __always_inline
 283 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 284 {
 285         RSEQ_INJECT_C(9)
 286
 287         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 288 #ifdef RSEQ_COMPARE_TWICE
 289                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 290 #endif
 291                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 292                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 293                                   RSEQ_INJECT_ASM(3)
 294 #ifdef RSEQ_COMPARE_TWICE
 295                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 296 #endif
 297                                   RSEQ_ASM_OP_R_LOAD(v)
 298                                   RSEQ_ASM_OP_R_ADD(count)
 299                                   RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 300                                   RSEQ_INJECT_ASM(4)
 301                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 302                                   : /* gcc asm goto does not allow outputs */
 303                                   : [cpu_id]            "r" (cpu),
 304                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 305                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 306                                     [v]                 "m" (*v),
 307                                     [count]             "r" (count)
 308                                     RSEQ_INJECT_INPUT
 309                                   : "memory", RSEQ_ASM_TMP_REG_1
 310                                     RSEQ_INJECT_CLOBBER
 311                                   : abort
 312 #ifdef RSEQ_COMPARE_TWICE
 313                                     , error1
 314 #endif
 315         );
 316         return 0;
 317 abort:
 318         RSEQ_INJECT_FAILED
 319         return -1;
 320 #ifdef RSEQ_COMPARE_TWICE
 321 error1:
 322         rseq_bug("cpu_id comparison failed");
 323 #endif
 324 }
 325
 326 static inline __always_inline
 327 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 328                                  intptr_t *v2, intptr_t newv2,
 329                                  intptr_t newv, int cpu)
 330 {
 331         RSEQ_INJECT_C(9)
 332
 333         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 334                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 335 #ifdef RSEQ_COMPARE_TWICE
 336                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 337                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 338 #endif
 339                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 340                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 341                                   RSEQ_INJECT_ASM(3)
 342                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 343                                   RSEQ_INJECT_ASM(4)
 344 #ifdef RSEQ_COMPARE_TWICE
 345                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 346                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 347 #endif
 348                                   RSEQ_ASM_OP_STORE(newv2, v2)
 349                                   RSEQ_INJECT_ASM(5)
 350                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 351                                   RSEQ_INJECT_ASM(6)
 352                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 353                                   : /* gcc asm goto does not allow outputs */
 354                                   : [cpu_id]            "r" (cpu),
 355                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 356                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 357                                     [expect]            "r" (expect),
 358                                     [v]                 "m" (*v),
 359                                     [newv]              "r" (newv),
 360                                     [v2]                        "m" (*v2),
 361                                     [newv2]             "r" (newv2)
 362                                     RSEQ_INJECT_INPUT
 363                                   : "memory", RSEQ_ASM_TMP_REG_1
 364                                     RSEQ_INJECT_CLOBBER
 365                                   : abort, cmpfail
 366 #ifdef RSEQ_COMPARE_TWICE
 367                                     , error1, error2
 368 #endif
 369         );
 370
 371         return 0;
 372 abort:
 373         RSEQ_INJECT_FAILED
 374         return -1;
 375 cmpfail:
 376         return 1;
 377 #ifdef RSEQ_COMPARE_TWICE
 378 error1:
 379         rseq_bug("cpu_id comparison failed");
 380 error2:
 381         rseq_bug("expected value comparison failed");
 382 #endif
 383 }
 384
 385 static inline __always_inline
 386 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 387                                          intptr_t *v2, intptr_t newv2,
 388                                          intptr_t newv, int cpu)
 389 {
 390         RSEQ_INJECT_C(9)
 391
 392         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 393                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 394 #ifdef RSEQ_COMPARE_TWICE
 395                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 396                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 397 #endif
 398                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 399                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 400                                   RSEQ_INJECT_ASM(3)
 401                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 402                                   RSEQ_INJECT_ASM(4)
 403 #ifdef RSEQ_COMPARE_TWICE
 404                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 405                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 406 #endif
 407                                   RSEQ_ASM_OP_STORE(newv2, v2)
 408                                   RSEQ_INJECT_ASM(5)
 409                                   RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 410                                   RSEQ_INJECT_ASM(6)
 411                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 412                                   : /* gcc asm goto does not allow outputs */
 413                                   : [cpu_id]            "r" (cpu),
 414                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 415                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 416                                     [expect]            "r" (expect),
 417                                     [v]                 "m" (*v),
 418                                     [newv]              "r" (newv),
 419                                     [v2]                        "m" (*v2),
 420                                     [newv2]             "r" (newv2)
 421                                     RSEQ_INJECT_INPUT
 422                                   : "memory", RSEQ_ASM_TMP_REG_1
 423                                     RSEQ_INJECT_CLOBBER
 424                                   : abort, cmpfail
 425 #ifdef RSEQ_COMPARE_TWICE
 426                                     , error1, error2
 427 #endif
 428         );
 429
 430         return 0;
 431 abort:
 432         RSEQ_INJECT_FAILED
 433         return -1;
 434 cmpfail:
 435         return 1;
 436 #ifdef RSEQ_COMPARE_TWICE
 437 error1:
 438         rseq_bug("cpu_id comparison failed");
 439 error2:
 440         rseq_bug("expected value comparison failed");
 441 #endif
 442 }
 443
 444 static inline __always_inline
 445 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 446                               intptr_t *v2, intptr_t expect2,
 447                               intptr_t newv, int cpu)
 448 {
 449         RSEQ_INJECT_C(9)
 450
 451         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 452                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 453 #ifdef RSEQ_COMPARE_TWICE
 454                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 455                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 456                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
 457 #endif
 458                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 459                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 460                                   RSEQ_INJECT_ASM(3)
 461                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 462                                   RSEQ_INJECT_ASM(4)
 463                                   RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
 464                                   RSEQ_INJECT_ASM(5)
 465 #ifdef RSEQ_COMPARE_TWICE
 466                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 467                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 468                                   RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
 469 #endif
 470                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 471                                   RSEQ_INJECT_ASM(6)
 472                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 473                                   : /* gcc asm goto does not allow outputs */
 474                                   : [cpu_id]            "r" (cpu),
 475                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 476                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 477                                     [v]                 "m" (*v),
 478                                     [expect]            "r" (expect),
 479                                     [v2]                        "m" (*v2),
 480                                     [expect2]           "r" (expect2),
 481                                     [newv]              "r" (newv)
 482                                     RSEQ_INJECT_INPUT
 483                                   : "memory", RSEQ_ASM_TMP_REG_1
 484                                     RSEQ_INJECT_CLOBBER
 485                                   : abort, cmpfail
 486 #ifdef RSEQ_COMPARE_TWICE
 487                                     , error1, error2, error3
 488 #endif
 489         );
 490
 491         return 0;
 492 abort:
 493         RSEQ_INJECT_FAILED
 494         return -1;
 495 cmpfail:
 496         return 1;
 497 #ifdef RSEQ_COMPARE_TWICE
 498 error1:
 499         rseq_bug("cpu_id comparison failed");
 500 error2:
 501         rseq_bug("expected value comparison failed");
 502 error3:
 503         rseq_bug("2nd expected value comparison failed");
 504 #endif
 505 }
 506
 507 static inline __always_inline
 508 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 509                                  void *dst, void *src, size_t len,
 510                                  intptr_t newv, int cpu)
 511 {
 512         RSEQ_INJECT_C(9)
 513         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 514                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 515 #ifdef RSEQ_COMPARE_TWICE
 516                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 517                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 518 #endif
 519                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 520                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 521                                   RSEQ_INJECT_ASM(3)
 522                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 523                                   RSEQ_INJECT_ASM(4)
 524 #ifdef RSEQ_COMPARE_TWICE
 525                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 526                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 527 #endif
 528                                   RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 529                                   RSEQ_INJECT_ASM(5)
 530                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 531                                   RSEQ_INJECT_ASM(6)
 532                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 533                                   : /* gcc asm goto does not allow outputs */
 534                                   : [cpu_id]            "r" (cpu),
 535                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 536                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 537                                     [expect]            "r" (expect),
 538                                     [v]                 "m" (*v),
 539                                     [newv]              "r" (newv),
 540                                     [dst]                       "r" (dst),
 541                                     [src]                       "r" (src),
 542                                     [len]                       "r" (len)
 543                                     RSEQ_INJECT_INPUT
 544                                   : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
 545                                     RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
 546                                     RSEQ_INJECT_CLOBBER
 547                                   : abort, cmpfail
 548 #ifdef RSEQ_COMPARE_TWICE
 549                                     , error1, error2
 550 #endif
 551         );
 552
 553         return 0;
 554 abort:
 555         RSEQ_INJECT_FAILED
 556         return -1;
 557 cmpfail:
 558         return 1;
 559 #ifdef RSEQ_COMPARE_TWICE
 560 error1:
 561         rseq_bug("cpu_id comparison failed");
 562 error2:
 563         rseq_bug("expected value comparison failed");
 564 #endif
 565 }
 566
 567 static inline __always_inline
 568 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 569                                          void *dst, void *src, size_t len,
 570                                          intptr_t newv, int cpu)
 571 {
 572         RSEQ_INJECT_C(9)
 573
 574         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 575                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 576 #ifdef RSEQ_COMPARE_TWICE
 577                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 578                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 579 #endif
 580                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 581                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 582                                   RSEQ_INJECT_ASM(3)
 583                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 584                                   RSEQ_INJECT_ASM(4)
 585 #ifdef RSEQ_COMPARE_TWICE
 586                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 587                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 588 #endif
 589                                   RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 590                                   RSEQ_INJECT_ASM(5)
 591                                   RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 592                                   RSEQ_INJECT_ASM(6)
 593                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 594                                   : /* gcc asm goto does not allow outputs */
 595                                   : [cpu_id]            "r" (cpu),
 596                                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
 597                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 598                                     [expect]            "r" (expect),
 599                                     [v]                 "m" (*v),
 600                                     [newv]              "r" (newv),
 601                                     [dst]                       "r" (dst),
 602                                     [src]                       "r" (src),
 603                                     [len]                       "r" (len)
 604                                     RSEQ_INJECT_INPUT
 605                                   : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
 606                                     RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
 607                                     RSEQ_INJECT_CLOBBER
 608                                   : abort, cmpfail
 609 #ifdef RSEQ_COMPARE_TWICE
 610                                     , error1, error2
 611 #endif
 612         );
 613
 614         return 0;
 615 abort:
 616         RSEQ_INJECT_FAILED
 617         return -1;
 618 cmpfail:
 619         return 1;
 620 #ifdef RSEQ_COMPARE_TWICE
 621 error1:
 622         rseq_bug("cpu_id comparison failed");
 623 error2:
 624         rseq_bug("expected value comparison failed");
 625 #endif
 626 }
 627
 628 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
 629
 630 /*
 631  *   pval = *(ptr+off)
 632  *  *pval += inc;
 633  */
 634 static inline __always_inline
 635 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
 636 {
 637         RSEQ_INJECT_C(9)
 638
 639         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 640 #ifdef RSEQ_COMPARE_TWICE
 641                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 642 #endif
 643                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 644                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 645                                   RSEQ_INJECT_ASM(3)
 646 #ifdef RSEQ_COMPARE_TWICE
 647                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 648 #endif
 649                                   RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
 650                                   RSEQ_INJECT_ASM(4)
 651                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 652                                   : /* gcc asm goto does not allow outputs */
 653                                   : [cpu_id]            "r" (cpu),
 654                                     [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 655                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 656                                     [ptr]                       "r" (ptr),
 657                                     [off]                       "er" (off),
 658                                     [inc]                       "er" (inc)
 659                                     RSEQ_INJECT_INPUT
 660                                   : "memory", RSEQ_ASM_TMP_REG_1
 661                                     RSEQ_INJECT_CLOBBER
 662                                   : abort
 663 #ifdef RSEQ_COMPARE_TWICE
 664                                     , error1
 665 #endif
 666         );
 667         return 0;
 668 abort:
 669         RSEQ_INJECT_FAILED
 670         return -1;
 671 #ifdef RSEQ_COMPARE_TWICE
 672 error1:
 673         rseq_bug("cpu_id comparison failed");
 674 #endif
 675 }
 676
 677 #endif /* !RSEQ_SKIP_FASTPATH */