tools/testing/selftests/rseq/rseq-s390-bits.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2
   3 #include "rseq-bits-template.h"
   4
   5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
   6         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
   7
   8 static inline __attribute__((always_inline))
   9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  10 {
  11         RSEQ_INJECT_C(9)
  12
  13         __asm__ __volatile__ goto (
  14                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  15                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  16 #ifdef RSEQ_COMPARE_TWICE
  17                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  18                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  19 #endif
  20                 /* Start rseq by storing table entry pointer into rseq_cs. */
  21                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  22                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  23                 RSEQ_INJECT_ASM(3)
  24                 LONG_CMP " %[expect], %[v]\n\t"
  25                 "jnz %l[cmpfail]\n\t"
  26                 RSEQ_INJECT_ASM(4)
  27 #ifdef RSEQ_COMPARE_TWICE
  28                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  29                 LONG_CMP " %[expect], %[v]\n\t"
  30                 "jnz %l[error2]\n\t"
  31 #endif
  32                 /* final store */
  33                 LONG_S " %[newv], %[v]\n\t"
  34                 "2:\n\t"
  35                 RSEQ_INJECT_ASM(5)
  36                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
  37                 : /* gcc asm goto does not allow outputs */
  38                 : [cpu_id]              "r" (cpu),
  39                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  40                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  41                   [v]                   "m" (*v),
  42                   [expect]              "r" (expect),
  43                   [newv]                "r" (newv)
  44                   RSEQ_INJECT_INPUT
  45                 : "memory", "cc", "r0"
  46                   RSEQ_INJECT_CLOBBER
  47                 : abort, cmpfail
  48 #ifdef RSEQ_COMPARE_TWICE
  49                   , error1, error2
  50 #endif
  51         );
  52         rseq_after_asm_goto();
  53         return 0;
  54 abort:
  55         rseq_after_asm_goto();
  56         RSEQ_INJECT_FAILED
  57         return -1;
  58 cmpfail:
  59         rseq_after_asm_goto();
  60         return 1;
  61 #ifdef RSEQ_COMPARE_TWICE
  62 error1:
  63         rseq_after_asm_goto();
  64         rseq_bug("cpu_id comparison failed");
  65 error2:
  66         rseq_after_asm_goto();
  67         rseq_bug("expected value comparison failed");
  68 #endif
  69 }
  70
  71 /*
  72  * Compare @v against @expectnot. When it does _not_ match, load @v
  73  * into @load, and store the content of *@v + voffp into @v.
  74  */
  75 static inline __attribute__((always_inline))
  76 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  77                                long voffp, intptr_t *load, int cpu)
  78 {
  79         RSEQ_INJECT_C(9)
  80
  81         __asm__ __volatile__ goto (
  82                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  83                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  84 #ifdef RSEQ_COMPARE_TWICE
  85                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  86                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  87 #endif
  88                 /* Start rseq by storing table entry pointer into rseq_cs. */
  89                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  90                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  91                 RSEQ_INJECT_ASM(3)
  92                 LONG_L " %%r1, %[v]\n\t"
  93                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
  94                 "je %l[cmpfail]\n\t"
  95                 RSEQ_INJECT_ASM(4)
  96 #ifdef RSEQ_COMPARE_TWICE
  97                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  98                 LONG_L " %%r1, %[v]\n\t"
  99                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 100                 "je %l[error2]\n\t"
 101 #endif
 102                 LONG_S " %%r1, %[load]\n\t"
 103                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 104                 LONG_L " %%r1, 0(%%r1)\n\t"
 105                 /* final store */
 106                 LONG_S " %%r1, %[v]\n\t"
 107                 "2:\n\t"
 108                 RSEQ_INJECT_ASM(5)
 109                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 110                 : /* gcc asm goto does not allow outputs */
 111                 : [cpu_id]              "r" (cpu),
 112                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 113                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 114                   /* final store input */
 115                   [v]                   "m" (*v),
 116                   [expectnot]           "r" (expectnot),
 117                   [voffp]               "r" (voffp),
 118                   [load]                "m" (*load)
 119                   RSEQ_INJECT_INPUT
 120                 : "memory", "cc", "r0", "r1"
 121                   RSEQ_INJECT_CLOBBER
 122                 : abort, cmpfail
 123 #ifdef RSEQ_COMPARE_TWICE
 124                   , error1, error2
 125 #endif
 126         );
 127         rseq_after_asm_goto();
 128         return 0;
 129 abort:
 130         rseq_after_asm_goto();
 131         RSEQ_INJECT_FAILED
 132         return -1;
 133 cmpfail:
 134         rseq_after_asm_goto();
 135         return 1;
 136 #ifdef RSEQ_COMPARE_TWICE
 137 error1:
 138         rseq_after_asm_goto();
 139         rseq_bug("cpu_id comparison failed");
 140 error2:
 141         rseq_after_asm_goto();
 142         rseq_bug("expected value comparison failed");
 143 #endif
 144 }
 145
 146 static inline __attribute__((always_inline))
 147 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 148 {
 149         RSEQ_INJECT_C(9)
 150
 151         __asm__ __volatile__ goto (
 152                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 153 #ifdef RSEQ_COMPARE_TWICE
 154                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 155 #endif
 156                 /* Start rseq by storing table entry pointer into rseq_cs. */
 157                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 158                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 159                 RSEQ_INJECT_ASM(3)
 160 #ifdef RSEQ_COMPARE_TWICE
 161                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 162 #endif
 163                 LONG_L " %%r0, %[v]\n\t"
 164                 LONG_ADD_R " %%r0, %[count]\n\t"
 165                 /* final store */
 166                 LONG_S " %%r0, %[v]\n\t"
 167                 "2:\n\t"
 168                 RSEQ_INJECT_ASM(4)
 169                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 170                 : /* gcc asm goto does not allow outputs */
 171                 : [cpu_id]              "r" (cpu),
 172                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 173                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 174                   /* final store input */
 175                   [v]                   "m" (*v),
 176                   [count]               "r" (count)
 177                   RSEQ_INJECT_INPUT
 178                 : "memory", "cc", "r0"
 179                   RSEQ_INJECT_CLOBBER
 180                 : abort
 181 #ifdef RSEQ_COMPARE_TWICE
 182                   , error1
 183 #endif
 184         );
 185         rseq_after_asm_goto();
 186         return 0;
 187 abort:
 188         rseq_after_asm_goto();
 189         RSEQ_INJECT_FAILED
 190         return -1;
 191 #ifdef RSEQ_COMPARE_TWICE
 192 error1:
 193         rseq_after_asm_goto();
 194         rseq_bug("cpu_id comparison failed");
 195 #endif
 196 }
 197
 198 static inline __attribute__((always_inline))
 199 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 200                               intptr_t *v2, intptr_t expect2,
 201                               intptr_t newv, int cpu)
 202 {
 203         RSEQ_INJECT_C(9)
 204
 205         __asm__ __volatile__ goto (
 206                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 207                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 208 #ifdef RSEQ_COMPARE_TWICE
 209                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 210                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 211                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 212 #endif
 213                 /* Start rseq by storing table entry pointer into rseq_cs. */
 214                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 215                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 216                 RSEQ_INJECT_ASM(3)
 217                 LONG_CMP " %[expect], %[v]\n\t"
 218                 "jnz %l[cmpfail]\n\t"
 219                 RSEQ_INJECT_ASM(4)
 220                 LONG_CMP " %[expect2], %[v2]\n\t"
 221                 "jnz %l[cmpfail]\n\t"
 222                 RSEQ_INJECT_ASM(5)
 223 #ifdef RSEQ_COMPARE_TWICE
 224                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 225                 LONG_CMP " %[expect], %[v]\n\t"
 226                 "jnz %l[error2]\n\t"
 227                 LONG_CMP " %[expect2], %[v2]\n\t"
 228                 "jnz %l[error3]\n\t"
 229 #endif
 230                 /* final store */
 231                 LONG_S " %[newv], %[v]\n\t"
 232                 "2:\n\t"
 233                 RSEQ_INJECT_ASM(6)
 234                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 235                 : /* gcc asm goto does not allow outputs */
 236                 : [cpu_id]              "r" (cpu),
 237                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 238                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 239                   /* cmp2 input */
 240                   [v2]                  "m" (*v2),
 241                   [expect2]             "r" (expect2),
 242                   /* final store input */
 243                   [v]                   "m" (*v),
 244                   [expect]              "r" (expect),
 245                   [newv]                "r" (newv)
 246                   RSEQ_INJECT_INPUT
 247                 : "memory", "cc", "r0"
 248                   RSEQ_INJECT_CLOBBER
 249                 : abort, cmpfail
 250 #ifdef RSEQ_COMPARE_TWICE
 251                   , error1, error2, error3
 252 #endif
 253         );
 254         rseq_after_asm_goto();
 255         return 0;
 256 abort:
 257         rseq_after_asm_goto();
 258         RSEQ_INJECT_FAILED
 259         return -1;
 260 cmpfail:
 261         rseq_after_asm_goto();
 262         return 1;
 263 #ifdef RSEQ_COMPARE_TWICE
 264 error1:
 265         rseq_after_asm_goto();
 266         rseq_bug("cpu_id comparison failed");
 267 error2:
 268         rseq_after_asm_goto();
 269         rseq_bug("1st expected value comparison failed");
 270 error3:
 271         rseq_after_asm_goto();
 272         rseq_bug("2nd expected value comparison failed");
 273 #endif
 274 }
 275
 276 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 277         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 278
 279 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 280         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 281
 282 /* s390 is TSO. */
 283 static inline __attribute__((always_inline))
 284 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 285                                  intptr_t *v2, intptr_t newv2,
 286                                  intptr_t newv, int cpu)
 287 {
 288         RSEQ_INJECT_C(9)
 289
 290         __asm__ __volatile__ goto (
 291                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 292                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 293 #ifdef RSEQ_COMPARE_TWICE
 294                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 295                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 296 #endif
 297                 /* Start rseq by storing table entry pointer into rseq_cs. */
 298                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 299                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 300                 RSEQ_INJECT_ASM(3)
 301                 LONG_CMP " %[expect], %[v]\n\t"
 302                 "jnz %l[cmpfail]\n\t"
 303                 RSEQ_INJECT_ASM(4)
 304 #ifdef RSEQ_COMPARE_TWICE
 305                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 306                 LONG_CMP " %[expect], %[v]\n\t"
 307                 "jnz %l[error2]\n\t"
 308 #endif
 309                 /* try store */
 310                 LONG_S " %[newv2], %[v2]\n\t"
 311                 RSEQ_INJECT_ASM(5)
 312                 /* final store */
 313                 LONG_S " %[newv], %[v]\n\t"
 314                 "2:\n\t"
 315                 RSEQ_INJECT_ASM(6)
 316                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 317                 : /* gcc asm goto does not allow outputs */
 318                 : [cpu_id]              "r" (cpu),
 319                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 320                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 321                   /* try store input */
 322                   [v2]                  "m" (*v2),
 323                   [newv2]               "r" (newv2),
 324                   /* final store input */
 325                   [v]                   "m" (*v),
 326                   [expect]              "r" (expect),
 327                   [newv]                "r" (newv)
 328                   RSEQ_INJECT_INPUT
 329                 : "memory", "cc", "r0"
 330                   RSEQ_INJECT_CLOBBER
 331                 : abort, cmpfail
 332 #ifdef RSEQ_COMPARE_TWICE
 333                   , error1, error2
 334 #endif
 335         );
 336         rseq_after_asm_goto();
 337         return 0;
 338 abort:
 339         rseq_after_asm_goto();
 340         RSEQ_INJECT_FAILED
 341         return -1;
 342 cmpfail:
 343         rseq_after_asm_goto();
 344         return 1;
 345 #ifdef RSEQ_COMPARE_TWICE
 346 error1:
 347         rseq_after_asm_goto();
 348         rseq_bug("cpu_id comparison failed");
 349 error2:
 350         rseq_after_asm_goto();
 351         rseq_bug("expected value comparison failed");
 352 #endif
 353 }
 354
 355 /* s390 is TSO. */
 356 static inline __attribute__((always_inline))
 357 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 358                                  void *dst, void *src, size_t len,
 359                                  intptr_t newv, int cpu)
 360 {
 361         uint64_t rseq_scratch[3];
 362
 363         RSEQ_INJECT_C(9)
 364
 365         __asm__ __volatile__ goto (
 366                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 367                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 368 #ifdef RSEQ_COMPARE_TWICE
 369                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 370                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 371 #endif
 372                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 373                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 374                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 375                 /* Start rseq by storing table entry pointer into rseq_cs. */
 376                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 377                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 378                 RSEQ_INJECT_ASM(3)
 379                 LONG_CMP " %[expect], %[v]\n\t"
 380                 "jnz 5f\n\t"
 381                 RSEQ_INJECT_ASM(4)
 382 #ifdef RSEQ_COMPARE_TWICE
 383                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 384                 LONG_CMP " %[expect], %[v]\n\t"
 385                 "jnz 7f\n\t"
 386 #endif
 387                 /* try memcpy */
 388                 LONG_LT_R " %[len], %[len]\n\t"
 389                 "jz 333f\n\t"
 390                 "222:\n\t"
 391                 "ic %%r0,0(%[src])\n\t"
 392                 "stc %%r0,0(%[dst])\n\t"
 393                 LONG_ADDI " %[src], 1\n\t"
 394                 LONG_ADDI " %[dst], 1\n\t"
 395                 LONG_ADDI " %[len], -1\n\t"
 396                 "jnz 222b\n\t"
 397                 "333:\n\t"
 398                 RSEQ_INJECT_ASM(5)
 399                 /* final store */
 400                 LONG_S " %[newv], %[v]\n\t"
 401                 "2:\n\t"
 402                 RSEQ_INJECT_ASM(6)
 403                 /* teardown */
 404                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 405                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 406                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 407                 RSEQ_ASM_DEFINE_ABORT(4,
 408                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 409                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 410                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 411                         abort)
 412                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 413                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 414                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 415                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 416                         cmpfail)
 417 #ifdef RSEQ_COMPARE_TWICE
 418                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 419                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 420                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 421                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 422                         error1)
 423                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 424                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 425                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 426                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 427                         error2)
 428 #endif
 429                 : /* gcc asm goto does not allow outputs */
 430                 : [cpu_id]              "r" (cpu),
 431                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 432                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 433                   /* final store input */
 434                   [v]                   "m" (*v),
 435                   [expect]              "r" (expect),
 436                   [newv]                "r" (newv),
 437                   /* try memcpy input */
 438                   [dst]                 "r" (dst),
 439                   [src]                 "r" (src),
 440                   [len]                 "r" (len),
 441                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 442                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 443                   [rseq_scratch2]       "m" (rseq_scratch[2])
 444                   RSEQ_INJECT_INPUT
 445                 : "memory", "cc", "r0"
 446                   RSEQ_INJECT_CLOBBER
 447                 : abort, cmpfail
 448 #ifdef RSEQ_COMPARE_TWICE
 449                   , error1, error2
 450 #endif
 451         );
 452         rseq_after_asm_goto();
 453         return 0;
 454 abort:
 455         rseq_after_asm_goto();
 456         RSEQ_INJECT_FAILED
 457         return -1;
 458 cmpfail:
 459         rseq_after_asm_goto();
 460         return 1;
 461 #ifdef RSEQ_COMPARE_TWICE
 462 error1:
 463         rseq_after_asm_goto();
 464         rseq_bug("cpu_id comparison failed");
 465 error2:
 466         rseq_after_asm_goto();
 467         rseq_bug("expected value comparison failed");
 468 #endif
 469 }
 470
 471 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 472         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 473
 474 #include "rseq-bits-reset.h"