tools/testing/selftests/rseq/rseq-s390.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2
   3 /*
   4  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
   5  * access-register mode nor the linkage stack this instruction will always
   6  * cause a special-operation exception (the trap-enabled bit in the DUCT
   7  * is and will stay 0). The instruction pattern is
   8  *      b2 ff 0f ff     trap4   4095(%r0)
   9  */
  10 #define RSEQ_SIG        0xB2FF0FFF
  11
  12 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  13 #define rseq_smp_rmb()  rseq_smp_mb()
  14 #define rseq_smp_wmb()  rseq_smp_mb()
  15
  16 #define rseq_smp_load_acquire(p)                                        \
  17 __extension__ ({                                                        \
  18         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  19         rseq_barrier();                                                 \
  20         ____p1;                                                         \
  21 })
  22
  23 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  24
  25 #define rseq_smp_store_release(p, v)                                    \
  26 do {                                                                    \
  27         rseq_barrier();                                                 \
  28         RSEQ_WRITE_ONCE(*p, v);                                         \
  29 } while (0)
  30
  31 #ifdef RSEQ_SKIP_FASTPATH
  32 #include "rseq-skip.h"
  33 #else /* !RSEQ_SKIP_FASTPATH */
  34
  35 #ifdef __s390x__
  36
  37 #define LONG_L                  "lg"
  38 #define LONG_S                  "stg"
  39 #define LONG_LT_R               "ltgr"
  40 #define LONG_CMP                "cg"
  41 #define LONG_CMP_R              "cgr"
  42 #define LONG_ADDI               "aghi"
  43 #define LONG_ADD_R              "agr"
  44
  45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  46                                 start_ip, post_commit_offset, abort_ip) \
  47                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
  48                 ".balign 32\n\t"                                        \
  49                 __rseq_str(label) ":\n\t"                               \
  50                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  51                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  52                 ".popsection\n\t"
  53
  54 #elif __s390__
  55
  56 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  57                                 start_ip, post_commit_offset, abort_ip) \
  58                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
  59                 ".balign 32\n\t"                                        \
  60                 __rseq_str(label) ":\n\t"                               \
  61                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  62                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  63                 ".popsection\n\t"
  64
  65 #define LONG_L                  "l"
  66 #define LONG_S                  "st"
  67 #define LONG_LT_R               "ltr"
  68 #define LONG_CMP                "c"
  69 #define LONG_CMP_R              "cr"
  70 #define LONG_ADDI               "ahi"
  71 #define LONG_ADD_R              "ar"
  72
  73 #endif
  74
  75 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  76         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  77                                 (post_commit_ip - start_ip), abort_ip)
  78
  79 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  80                 RSEQ_INJECT_ASM(1)                                      \
  81                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
  82                 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"          \
  83                 __rseq_str(label) ":\n\t"
  84
  85 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  86                 RSEQ_INJECT_ASM(2)                                      \
  87                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
  88                 "jnz " __rseq_str(label) "\n\t"
  89
  90 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  91                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  92                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
  93                 __rseq_str(label) ":\n\t"                               \
  94                 teardown                                                \
  95                 "j %l[" __rseq_str(abort_label) "]\n\t"                 \
  96                 ".popsection\n\t"
  97
  98 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
  99                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 100                 __rseq_str(label) ":\n\t"                               \
 101                 teardown                                                \
 102                 "j %l[" __rseq_str(cmpfail_label) "]\n\t"               \
 103                 ".popsection\n\t"
 104
 105 static inline __attribute__((always_inline))
 106 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 107 {
 108         RSEQ_INJECT_C(9)
 109
 110         __asm__ __volatile__ goto (
 111                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 112                 /* Start rseq by storing table entry pointer into rseq_cs. */
 113                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 114                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 115                 RSEQ_INJECT_ASM(3)
 116                 LONG_CMP " %[expect], %[v]\n\t"
 117                 "jnz %l[cmpfail]\n\t"
 118                 RSEQ_INJECT_ASM(4)
 119 #ifdef RSEQ_COMPARE_TWICE
 120                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 121                 LONG_CMP " %[expect], %[v]\n\t"
 122                 "jnz %l[error2]\n\t"
 123 #endif
 124                 /* final store */
 125                 LONG_S " %[newv], %[v]\n\t"
 126                 "2:\n\t"
 127                 RSEQ_INJECT_ASM(5)
 128                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 129                 : /* gcc asm goto does not allow outputs */
 130                 : [cpu_id]              "r" (cpu),
 131                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 132                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 133                   [v]                   "m" (*v),
 134                   [expect]              "r" (expect),
 135                   [newv]                "r" (newv)
 136                   RSEQ_INJECT_INPUT
 137                 : "memory", "cc", "r0"
 138                   RSEQ_INJECT_CLOBBER
 139                 : abort, cmpfail
 140 #ifdef RSEQ_COMPARE_TWICE
 141                   , error1, error2
 142 #endif
 143         );
 144         return 0;
 145 abort:
 146         RSEQ_INJECT_FAILED
 147         return -1;
 148 cmpfail:
 149         return 1;
 150 #ifdef RSEQ_COMPARE_TWICE
 151 error1:
 152         rseq_bug("cpu_id comparison failed");
 153 error2:
 154         rseq_bug("expected value comparison failed");
 155 #endif
 156 }
 157
 158 /*
 159  * Compare @v against @expectnot. When it does _not_ match, load @v
 160  * into @load, and store the content of *@v + voffp into @v.
 161  */
 162 static inline __attribute__((always_inline))
 163 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 164                                off_t voffp, intptr_t *load, int cpu)
 165 {
 166         RSEQ_INJECT_C(9)
 167
 168         __asm__ __volatile__ goto (
 169                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 170                 /* Start rseq by storing table entry pointer into rseq_cs. */
 171                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 172                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 173                 RSEQ_INJECT_ASM(3)
 174                 LONG_L " %%r1, %[v]\n\t"
 175                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 176                 "je %l[cmpfail]\n\t"
 177                 RSEQ_INJECT_ASM(4)
 178 #ifdef RSEQ_COMPARE_TWICE
 179                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 180                 LONG_L " %%r1, %[v]\n\t"
 181                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 182                 "je %l[error2]\n\t"
 183 #endif
 184                 LONG_S " %%r1, %[load]\n\t"
 185                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 186                 LONG_L " %%r1, 0(%%r1)\n\t"
 187                 /* final store */
 188                 LONG_S " %%r1, %[v]\n\t"
 189                 "2:\n\t"
 190                 RSEQ_INJECT_ASM(5)
 191                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 192                 : /* gcc asm goto does not allow outputs */
 193                 : [cpu_id]              "r" (cpu),
 194                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 195                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 196                   /* final store input */
 197                   [v]                   "m" (*v),
 198                   [expectnot]           "r" (expectnot),
 199                   [voffp]               "r" (voffp),
 200                   [load]                "m" (*load)
 201                   RSEQ_INJECT_INPUT
 202                 : "memory", "cc", "r0", "r1"
 203                   RSEQ_INJECT_CLOBBER
 204                 : abort, cmpfail
 205 #ifdef RSEQ_COMPARE_TWICE
 206                   , error1, error2
 207 #endif
 208         );
 209         return 0;
 210 abort:
 211         RSEQ_INJECT_FAILED
 212         return -1;
 213 cmpfail:
 214         return 1;
 215 #ifdef RSEQ_COMPARE_TWICE
 216 error1:
 217         rseq_bug("cpu_id comparison failed");
 218 error2:
 219         rseq_bug("expected value comparison failed");
 220 #endif
 221 }
 222
 223 static inline __attribute__((always_inline))
 224 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 225 {
 226         RSEQ_INJECT_C(9)
 227
 228         __asm__ __volatile__ goto (
 229                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 230                 /* Start rseq by storing table entry pointer into rseq_cs. */
 231                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 232                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 233                 RSEQ_INJECT_ASM(3)
 234 #ifdef RSEQ_COMPARE_TWICE
 235                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 236 #endif
 237                 LONG_L " %%r0, %[v]\n\t"
 238                 LONG_ADD_R " %%r0, %[count]\n\t"
 239                 /* final store */
 240                 LONG_S " %%r0, %[v]\n\t"
 241                 "2:\n\t"
 242                 RSEQ_INJECT_ASM(4)
 243                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 244                 : /* gcc asm goto does not allow outputs */
 245                 : [cpu_id]              "r" (cpu),
 246                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 247                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 248                   /* final store input */
 249                   [v]                   "m" (*v),
 250                   [count]               "r" (count)
 251                   RSEQ_INJECT_INPUT
 252                 : "memory", "cc", "r0"
 253                   RSEQ_INJECT_CLOBBER
 254                 : abort
 255 #ifdef RSEQ_COMPARE_TWICE
 256                   , error1
 257 #endif
 258         );
 259         return 0;
 260 abort:
 261         RSEQ_INJECT_FAILED
 262         return -1;
 263 #ifdef RSEQ_COMPARE_TWICE
 264 error1:
 265         rseq_bug("cpu_id comparison failed");
 266 #endif
 267 }
 268
 269 static inline __attribute__((always_inline))
 270 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 271                                  intptr_t *v2, intptr_t newv2,
 272                                  intptr_t newv, int cpu)
 273 {
 274         RSEQ_INJECT_C(9)
 275
 276         __asm__ __volatile__ goto (
 277                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 278                 /* Start rseq by storing table entry pointer into rseq_cs. */
 279                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 280                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 281                 RSEQ_INJECT_ASM(3)
 282                 LONG_CMP " %[expect], %[v]\n\t"
 283                 "jnz %l[cmpfail]\n\t"
 284                 RSEQ_INJECT_ASM(4)
 285 #ifdef RSEQ_COMPARE_TWICE
 286                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 287                 LONG_CMP " %[expect], %[v]\n\t"
 288                 "jnz %l[error2]\n\t"
 289 #endif
 290                 /* try store */
 291                 LONG_S " %[newv2], %[v2]\n\t"
 292                 RSEQ_INJECT_ASM(5)
 293                 /* final store */
 294                 LONG_S " %[newv], %[v]\n\t"
 295                 "2:\n\t"
 296                 RSEQ_INJECT_ASM(6)
 297                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 298                 : /* gcc asm goto does not allow outputs */
 299                 : [cpu_id]              "r" (cpu),
 300                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 301                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 302                   /* try store input */
 303                   [v2]                  "m" (*v2),
 304                   [newv2]               "r" (newv2),
 305                   /* final store input */
 306                   [v]                   "m" (*v),
 307                   [expect]              "r" (expect),
 308                   [newv]                "r" (newv)
 309                   RSEQ_INJECT_INPUT
 310                 : "memory", "cc", "r0"
 311                   RSEQ_INJECT_CLOBBER
 312                 : abort, cmpfail
 313 #ifdef RSEQ_COMPARE_TWICE
 314                   , error1, error2
 315 #endif
 316         );
 317         return 0;
 318 abort:
 319         RSEQ_INJECT_FAILED
 320         return -1;
 321 cmpfail:
 322         return 1;
 323 #ifdef RSEQ_COMPARE_TWICE
 324 error1:
 325         rseq_bug("cpu_id comparison failed");
 326 error2:
 327         rseq_bug("expected value comparison failed");
 328 #endif
 329 }
 330
 331 /* s390 is TSO. */
 332 static inline __attribute__((always_inline))
 333 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 334                                          intptr_t *v2, intptr_t newv2,
 335                                          intptr_t newv, int cpu)
 336 {
 337         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 338 }
 339
 340 static inline __attribute__((always_inline))
 341 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 342                               intptr_t *v2, intptr_t expect2,
 343                               intptr_t newv, int cpu)
 344 {
 345         RSEQ_INJECT_C(9)
 346
 347         __asm__ __volatile__ goto (
 348                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 349                 /* Start rseq by storing table entry pointer into rseq_cs. */
 350                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 351                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 352                 RSEQ_INJECT_ASM(3)
 353                 LONG_CMP " %[expect], %[v]\n\t"
 354                 "jnz %l[cmpfail]\n\t"
 355                 RSEQ_INJECT_ASM(4)
 356                 LONG_CMP " %[expect2], %[v2]\n\t"
 357                 "jnz %l[cmpfail]\n\t"
 358                 RSEQ_INJECT_ASM(5)
 359 #ifdef RSEQ_COMPARE_TWICE
 360                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 361                 LONG_CMP " %[expect], %[v]\n\t"
 362                 "jnz %l[error2]\n\t"
 363                 LONG_CMP " %[expect2], %[v2]\n\t"
 364                 "jnz %l[error3]\n\t"
 365 #endif
 366                 /* final store */
 367                 LONG_S " %[newv], %[v]\n\t"
 368                 "2:\n\t"
 369                 RSEQ_INJECT_ASM(6)
 370                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 371                 : /* gcc asm goto does not allow outputs */
 372                 : [cpu_id]              "r" (cpu),
 373                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 374                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 375                   /* cmp2 input */
 376                   [v2]                  "m" (*v2),
 377                   [expect2]             "r" (expect2),
 378                   /* final store input */
 379                   [v]                   "m" (*v),
 380                   [expect]              "r" (expect),
 381                   [newv]                "r" (newv)
 382                   RSEQ_INJECT_INPUT
 383                 : "memory", "cc", "r0"
 384                   RSEQ_INJECT_CLOBBER
 385                 : abort, cmpfail
 386 #ifdef RSEQ_COMPARE_TWICE
 387                   , error1, error2, error3
 388 #endif
 389         );
 390         return 0;
 391 abort:
 392         RSEQ_INJECT_FAILED
 393         return -1;
 394 cmpfail:
 395         return 1;
 396 #ifdef RSEQ_COMPARE_TWICE
 397 error1:
 398         rseq_bug("cpu_id comparison failed");
 399 error2:
 400         rseq_bug("1st expected value comparison failed");
 401 error3:
 402         rseq_bug("2nd expected value comparison failed");
 403 #endif
 404 }
 405
 406 static inline __attribute__((always_inline))
 407 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 408                                  void *dst, void *src, size_t len,
 409                                  intptr_t newv, int cpu)
 410 {
 411         uint64_t rseq_scratch[3];
 412
 413         RSEQ_INJECT_C(9)
 414
 415         __asm__ __volatile__ goto (
 416                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 417                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 418                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 419                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 420                 /* Start rseq by storing table entry pointer into rseq_cs. */
 421                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 422                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 423                 RSEQ_INJECT_ASM(3)
 424                 LONG_CMP " %[expect], %[v]\n\t"
 425                 "jnz 5f\n\t"
 426                 RSEQ_INJECT_ASM(4)
 427 #ifdef RSEQ_COMPARE_TWICE
 428                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 429                 LONG_CMP " %[expect], %[v]\n\t"
 430                 "jnz 7f\n\t"
 431 #endif
 432                 /* try memcpy */
 433                 LONG_LT_R " %[len], %[len]\n\t"
 434                 "jz 333f\n\t"
 435                 "222:\n\t"
 436                 "ic %%r0,0(%[src])\n\t"
 437                 "stc %%r0,0(%[dst])\n\t"
 438                 LONG_ADDI " %[src], 1\n\t"
 439                 LONG_ADDI " %[dst], 1\n\t"
 440                 LONG_ADDI " %[len], -1\n\t"
 441                 "jnz 222b\n\t"
 442                 "333:\n\t"
 443                 RSEQ_INJECT_ASM(5)
 444                 /* final store */
 445                 LONG_S " %[newv], %[v]\n\t"
 446                 "2:\n\t"
 447                 RSEQ_INJECT_ASM(6)
 448                 /* teardown */
 449                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 450                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 451                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 452                 RSEQ_ASM_DEFINE_ABORT(4,
 453                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 454                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 455                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 456                         abort)
 457                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 458                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 459                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 460                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 461                         cmpfail)
 462 #ifdef RSEQ_COMPARE_TWICE
 463                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 464                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 465                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 466                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 467                         error1)
 468                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 469                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 470                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 471                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 472                         error2)
 473 #endif
 474                 : /* gcc asm goto does not allow outputs */
 475                 : [cpu_id]              "r" (cpu),
 476                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 477                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 478                   /* final store input */
 479                   [v]                   "m" (*v),
 480                   [expect]              "r" (expect),
 481                   [newv]                "r" (newv),
 482                   /* try memcpy input */
 483                   [dst]                 "r" (dst),
 484                   [src]                 "r" (src),
 485                   [len]                 "r" (len),
 486                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 487                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 488                   [rseq_scratch2]       "m" (rseq_scratch[2])
 489                   RSEQ_INJECT_INPUT
 490                 : "memory", "cc", "r0"
 491                   RSEQ_INJECT_CLOBBER
 492                 : abort, cmpfail
 493 #ifdef RSEQ_COMPARE_TWICE
 494                   , error1, error2
 495 #endif
 496         );
 497         return 0;
 498 abort:
 499         RSEQ_INJECT_FAILED
 500         return -1;
 501 cmpfail:
 502         return 1;
 503 #ifdef RSEQ_COMPARE_TWICE
 504 error1:
 505         rseq_bug("cpu_id comparison failed");
 506 error2:
 507         rseq_bug("expected value comparison failed");
 508 #endif
 509 }
 510
 511 /* s390 is TSO. */
 512 static inline __attribute__((always_inline))
 513 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 514                                          void *dst, void *src, size_t len,
 515                                          intptr_t newv, int cpu)
 516 {
 517         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 518                                             newv, cpu);
 519 }
 520 #endif /* !RSEQ_SKIP_FASTPATH */