tools/testing/selftests/rseq/rseq-ppc-bits.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-ppc-bits.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <[email protected]>
   6  * (C) Copyright 2016-2018 - Boqun Feng <[email protected]>
   7  */
   8
   9 #include "rseq-bits-template.h"
  10
  11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  12         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
  13
  14 static inline __attribute__((always_inline))
  15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  16 {
  17         RSEQ_INJECT_C(9)
  18
  19         __asm__ __volatile__ goto (
  20                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  21                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  22 #ifdef RSEQ_COMPARE_TWICE
  23                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  24                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  25 #endif
  26                 /* Start rseq by storing table entry pointer into rseq_cs. */
  27                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  28                 /* cmp cpuid */
  29                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  30                 RSEQ_INJECT_ASM(3)
  31                 /* cmp @v equal to @expect */
  32                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
  33                 RSEQ_INJECT_ASM(4)
  34 #ifdef RSEQ_COMPARE_TWICE
  35                 /* cmp cpuid */
  36                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  37                 /* cmp @v equal to @expect */
  38                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
  39 #endif
  40                 /* final store */
  41                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
  42                 RSEQ_INJECT_ASM(5)
  43                 RSEQ_ASM_DEFINE_ABORT(4, abort)
  44                 : /* gcc asm goto does not allow outputs */
  45                 : [cpu_id]              "r" (cpu),
  46                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  47                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  48                   [v]                   "m" (*v),
  49                   [expect]              "r" (expect),
  50                   [newv]                "r" (newv)
  51                   RSEQ_INJECT_INPUT
  52                 : "memory", "cc", "r17"
  53                   RSEQ_INJECT_CLOBBER
  54                 : abort, cmpfail
  55 #ifdef RSEQ_COMPARE_TWICE
  56                   , error1, error2
  57 #endif
  58         );
  59         rseq_after_asm_goto();
  60         return 0;
  61 abort:
  62         rseq_after_asm_goto();
  63         RSEQ_INJECT_FAILED
  64         return -1;
  65 cmpfail:
  66         rseq_after_asm_goto();
  67         return 1;
  68 #ifdef RSEQ_COMPARE_TWICE
  69 error1:
  70         rseq_after_asm_goto();
  71         rseq_bug("cpu_id comparison failed");
  72 error2:
  73         rseq_after_asm_goto();
  74         rseq_bug("expected value comparison failed");
  75 #endif
  76 }
  77
  78 static inline __attribute__((always_inline))
  79 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  80                                long voffp, intptr_t *load, int cpu)
  81 {
  82         RSEQ_INJECT_C(9)
  83
  84         __asm__ __volatile__ goto (
  85                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  86                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  87 #ifdef RSEQ_COMPARE_TWICE
  88                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  89                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  90 #endif
  91                 /* Start rseq by storing table entry pointer into rseq_cs. */
  92                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  93                 /* cmp cpuid */
  94                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  95                 RSEQ_INJECT_ASM(3)
  96                 /* cmp @v not equal to @expectnot */
  97                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
  98                 RSEQ_INJECT_ASM(4)
  99 #ifdef RSEQ_COMPARE_TWICE
 100                 /* cmp cpuid */
 101                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 102                 /* cmp @v not equal to @expectnot */
 103                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 104 #endif
 105                 /* load the value of @v */
 106                 RSEQ_ASM_OP_R_LOAD(v)
 107                 /* store it in @load */
 108                 RSEQ_ASM_OP_R_STORE(load)
 109                 /* dereference voffp(v) */
 110                 RSEQ_ASM_OP_R_LOADX(voffp)
 111                 /* final store the value at voffp(v) */
 112                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 113                 RSEQ_INJECT_ASM(5)
 114                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 115                 : /* gcc asm goto does not allow outputs */
 116                 : [cpu_id]              "r" (cpu),
 117                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 118                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 119                   /* final store input */
 120                   [v]                   "m" (*v),
 121                   [expectnot]           "r" (expectnot),
 122                   [voffp]               "b" (voffp),
 123                   [load]                "m" (*load)
 124                   RSEQ_INJECT_INPUT
 125                 : "memory", "cc", "r17"
 126                   RSEQ_INJECT_CLOBBER
 127                 : abort, cmpfail
 128 #ifdef RSEQ_COMPARE_TWICE
 129                   , error1, error2
 130 #endif
 131         );
 132         rseq_after_asm_goto();
 133         return 0;
 134 abort:
 135         rseq_after_asm_goto();
 136         RSEQ_INJECT_FAILED
 137         return -1;
 138 cmpfail:
 139         rseq_after_asm_goto();
 140         return 1;
 141 #ifdef RSEQ_COMPARE_TWICE
 142 error1:
 143         rseq_after_asm_goto();
 144         rseq_bug("cpu_id comparison failed");
 145 error2:
 146         rseq_after_asm_goto();
 147         rseq_bug("expected value comparison failed");
 148 #endif
 149 }
 150
 151 static inline __attribute__((always_inline))
 152 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 153 {
 154         RSEQ_INJECT_C(9)
 155
 156         __asm__ __volatile__ goto (
 157                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 158 #ifdef RSEQ_COMPARE_TWICE
 159                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 160 #endif
 161                 /* Start rseq by storing table entry pointer into rseq_cs. */
 162                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 163                 /* cmp cpuid */
 164                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 165                 RSEQ_INJECT_ASM(3)
 166 #ifdef RSEQ_COMPARE_TWICE
 167                 /* cmp cpuid */
 168                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 169 #endif
 170                 /* load the value of @v */
 171                 RSEQ_ASM_OP_R_LOAD(v)
 172                 /* add @count to it */
 173                 RSEQ_ASM_OP_R_ADD(count)
 174                 /* final store */
 175                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 176                 RSEQ_INJECT_ASM(4)
 177                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 178                 : /* gcc asm goto does not allow outputs */
 179                 : [cpu_id]              "r" (cpu),
 180                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 181                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 182                   /* final store input */
 183                   [v]                   "m" (*v),
 184                   [count]               "r" (count)
 185                   RSEQ_INJECT_INPUT
 186                 : "memory", "cc", "r17"
 187                   RSEQ_INJECT_CLOBBER
 188                 : abort
 189 #ifdef RSEQ_COMPARE_TWICE
 190                   , error1
 191 #endif
 192         );
 193         rseq_after_asm_goto();
 194         return 0;
 195 abort:
 196         rseq_after_asm_goto();
 197         RSEQ_INJECT_FAILED
 198         return -1;
 199 #ifdef RSEQ_COMPARE_TWICE
 200 error1:
 201         rseq_after_asm_goto();
 202         rseq_bug("cpu_id comparison failed");
 203 #endif
 204 }
 205
 206 static inline __attribute__((always_inline))
 207 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 208                               intptr_t *v2, intptr_t expect2,
 209                               intptr_t newv, int cpu)
 210 {
 211         RSEQ_INJECT_C(9)
 212
 213         __asm__ __volatile__ goto (
 214                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 215                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 216 #ifdef RSEQ_COMPARE_TWICE
 217                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 218                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 219                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 220 #endif
 221                 /* Start rseq by storing table entry pointer into rseq_cs. */
 222                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 223                 /* cmp cpuid */
 224                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 225                 RSEQ_INJECT_ASM(3)
 226                 /* cmp @v equal to @expect */
 227                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 228                 RSEQ_INJECT_ASM(4)
 229                 /* cmp @v2 equal to @expct2 */
 230                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 231                 RSEQ_INJECT_ASM(5)
 232 #ifdef RSEQ_COMPARE_TWICE
 233                 /* cmp cpuid */
 234                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 235                 /* cmp @v equal to @expect */
 236                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 237                 /* cmp @v2 equal to @expct2 */
 238                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 239 #endif
 240                 /* final store */
 241                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 242                 RSEQ_INJECT_ASM(6)
 243                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 244                 : /* gcc asm goto does not allow outputs */
 245                 : [cpu_id]              "r" (cpu),
 246                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 247                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 248                   /* cmp2 input */
 249                   [v2]                  "m" (*v2),
 250                   [expect2]             "r" (expect2),
 251                   /* final store input */
 252                   [v]                   "m" (*v),
 253                   [expect]              "r" (expect),
 254                   [newv]                "r" (newv)
 255                   RSEQ_INJECT_INPUT
 256                 : "memory", "cc", "r17"
 257                   RSEQ_INJECT_CLOBBER
 258                 : abort, cmpfail
 259 #ifdef RSEQ_COMPARE_TWICE
 260                   , error1, error2, error3
 261 #endif
 262         );
 263         rseq_after_asm_goto();
 264         return 0;
 265 abort:
 266         rseq_after_asm_goto();
 267         RSEQ_INJECT_FAILED
 268         return -1;
 269 cmpfail:
 270         rseq_after_asm_goto();
 271         return 1;
 272 #ifdef RSEQ_COMPARE_TWICE
 273 error1:
 274         rseq_after_asm_goto();
 275         rseq_bug("cpu_id comparison failed");
 276 error2:
 277         rseq_after_asm_goto();
 278         rseq_bug("1st expected value comparison failed");
 279 error3:
 280         rseq_after_asm_goto();
 281         rseq_bug("2nd expected value comparison failed");
 282 #endif
 283 }
 284
 285 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 286         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 287
 288 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 289         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 290
 291 static inline __attribute__((always_inline))
 292 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 293                                  intptr_t *v2, intptr_t newv2,
 294                                  intptr_t newv, int cpu)
 295 {
 296         RSEQ_INJECT_C(9)
 297
 298         __asm__ __volatile__ goto (
 299                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 300                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 301 #ifdef RSEQ_COMPARE_TWICE
 302                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 303                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 304 #endif
 305                 /* Start rseq by storing table entry pointer into rseq_cs. */
 306                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 307                 /* cmp cpuid */
 308                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 309                 RSEQ_INJECT_ASM(3)
 310                 /* cmp @v equal to @expect */
 311                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 312                 RSEQ_INJECT_ASM(4)
 313 #ifdef RSEQ_COMPARE_TWICE
 314                 /* cmp cpuid */
 315                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 316                 /* cmp @v equal to @expect */
 317                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 318 #endif
 319                 /* try store */
 320                 RSEQ_ASM_OP_STORE(newv2, v2)
 321                 RSEQ_INJECT_ASM(5)
 322 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 323                 /* for 'release' */
 324                 "lwsync\n\t"
 325 #endif
 326                 /* final store */
 327                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 328                 RSEQ_INJECT_ASM(6)
 329                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 330                 : /* gcc asm goto does not allow outputs */
 331                 : [cpu_id]              "r" (cpu),
 332                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 333                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 334                   /* try store input */
 335                   [v2]                  "m" (*v2),
 336                   [newv2]               "r" (newv2),
 337                   /* final store input */
 338                   [v]                   "m" (*v),
 339                   [expect]              "r" (expect),
 340                   [newv]                "r" (newv)
 341                   RSEQ_INJECT_INPUT
 342                 : "memory", "cc", "r17"
 343                   RSEQ_INJECT_CLOBBER
 344                 : abort, cmpfail
 345 #ifdef RSEQ_COMPARE_TWICE
 346                   , error1, error2
 347 #endif
 348         );
 349         rseq_after_asm_goto();
 350         return 0;
 351 abort:
 352         rseq_after_asm_goto();
 353         RSEQ_INJECT_FAILED
 354         return -1;
 355 cmpfail:
 356         rseq_after_asm_goto();
 357         return 1;
 358 #ifdef RSEQ_COMPARE_TWICE
 359 error1:
 360         rseq_after_asm_goto();
 361         rseq_bug("cpu_id comparison failed");
 362 error2:
 363         rseq_after_asm_goto();
 364         rseq_bug("expected value comparison failed");
 365 #endif
 366 }
 367
 368 static inline __attribute__((always_inline))
 369 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 370                                  void *dst, void *src, size_t len,
 371                                  intptr_t newv, int cpu)
 372 {
 373         RSEQ_INJECT_C(9)
 374
 375         __asm__ __volatile__ goto (
 376                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 377                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 378 #ifdef RSEQ_COMPARE_TWICE
 379                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 380                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 381 #endif
 382                 /* setup for mempcy */
 383                 "mr %%r19, %[len]\n\t"
 384                 "mr %%r20, %[src]\n\t"
 385                 "mr %%r21, %[dst]\n\t"
 386                 /* Start rseq by storing table entry pointer into rseq_cs. */
 387                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 388                 /* cmp cpuid */
 389                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 390                 RSEQ_INJECT_ASM(3)
 391                 /* cmp @v equal to @expect */
 392                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 393                 RSEQ_INJECT_ASM(4)
 394 #ifdef RSEQ_COMPARE_TWICE
 395                 /* cmp cpuid */
 396                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 397                 /* cmp @v equal to @expect */
 398                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 399 #endif
 400                 /* try memcpy */
 401                 RSEQ_ASM_OP_R_MEMCPY()
 402                 RSEQ_INJECT_ASM(5)
 403 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 404                 /* for 'release' */
 405                 "lwsync\n\t"
 406 #endif
 407                 /* final store */
 408                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 409                 RSEQ_INJECT_ASM(6)
 410                 /* teardown */
 411                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 412                 : /* gcc asm goto does not allow outputs */
 413                 : [cpu_id]              "r" (cpu),
 414                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 415                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 416                   /* final store input */
 417                   [v]                   "m" (*v),
 418                   [expect]              "r" (expect),
 419                   [newv]                "r" (newv),
 420                   /* try memcpy input */
 421                   [dst]                 "r" (dst),
 422                   [src]                 "r" (src),
 423                   [len]                 "r" (len)
 424                   RSEQ_INJECT_INPUT
 425                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 426                   RSEQ_INJECT_CLOBBER
 427                 : abort, cmpfail
 428 #ifdef RSEQ_COMPARE_TWICE
 429                   , error1, error2
 430 #endif
 431         );
 432         rseq_after_asm_goto();
 433         return 0;
 434 abort:
 435         rseq_after_asm_goto();
 436         RSEQ_INJECT_FAILED
 437         return -1;
 438 cmpfail:
 439         rseq_after_asm_goto();
 440         return 1;
 441 #ifdef RSEQ_COMPARE_TWICE
 442 error1:
 443         rseq_after_asm_goto();
 444         rseq_bug("cpu_id comparison failed");
 445 error2:
 446         rseq_after_asm_goto();
 447         rseq_bug("expected value comparison failed");
 448 #endif
 449 }
 450
 451 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 452         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 453
 454 #include "rseq-bits-reset.h"