tools/testing/selftests/rseq/rseq-arm64-bits.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-arm64-bits.h
   4  *
   5  * (C) Copyright 2016-2022 - Mathieu Desnoyers <[email protected]>
   6  * (C) Copyright 2018 - Will Deacon <[email protected]>
   7  */
   8
   9 #include "rseq-bits-template.h"
  10
  11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  12         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
  13
  14 static inline __attribute__((always_inline))
  15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  16 {
  17         RSEQ_INJECT_C(9)
  18
  19         __asm__ __volatile__ goto (
  20                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
  21                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
  22 #ifdef RSEQ_COMPARE_TWICE
  23                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
  24                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
  25 #endif
  26                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
  27                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  28                 RSEQ_INJECT_ASM(3)
  29                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
  30                 RSEQ_INJECT_ASM(4)
  31 #ifdef RSEQ_COMPARE_TWICE
  32                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  33                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
  34 #endif
  35                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
  36                 RSEQ_INJECT_ASM(5)
  37                 RSEQ_ASM_DEFINE_ABORT(4, abort)
  38                 : /* gcc asm goto does not allow outputs */
  39                 : [cpu_id]              "r" (cpu),
  40                   [current_cpu_id]      "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  41                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  42                   [v]                   "Qo" (*v),
  43                   [expect]              "r" (expect),
  44                   [newv]                "r" (newv)
  45                   RSEQ_INJECT_INPUT
  46                 : "memory", RSEQ_ASM_TMP_REG
  47                 : abort, cmpfail
  48 #ifdef RSEQ_COMPARE_TWICE
  49                   , error1, error2
  50 #endif
  51         );
  52         rseq_after_asm_goto();
  53         return 0;
  54 abort:
  55         rseq_after_asm_goto();
  56         RSEQ_INJECT_FAILED
  57         return -1;
  58 cmpfail:
  59         rseq_after_asm_goto();
  60         return 1;
  61 #ifdef RSEQ_COMPARE_TWICE
  62 error1:
  63         rseq_after_asm_goto();
  64         rseq_bug("cpu_id comparison failed");
  65 error2:
  66         rseq_after_asm_goto();
  67         rseq_bug("expected value comparison failed");
  68 #endif
  69 }
  70
  71 static inline __attribute__((always_inline))
  72 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  73                                long voffp, intptr_t *load, int cpu)
  74 {
  75         RSEQ_INJECT_C(9)
  76
  77         __asm__ __volatile__ goto (
  78                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
  79                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
  80 #ifdef RSEQ_COMPARE_TWICE
  81                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
  82                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
  83 #endif
  84                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
  85                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  86                 RSEQ_INJECT_ASM(3)
  87                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
  88                 RSEQ_INJECT_ASM(4)
  89 #ifdef RSEQ_COMPARE_TWICE
  90                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  91                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
  92 #endif
  93                 RSEQ_ASM_OP_R_LOAD(v)
  94                 RSEQ_ASM_OP_R_STORE(load)
  95                 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
  96                 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
  97                 RSEQ_INJECT_ASM(5)
  98                 RSEQ_ASM_DEFINE_ABORT(4, abort)
  99                 : /* gcc asm goto does not allow outputs */
 100                 : [cpu_id]              "r" (cpu),
 101                   [current_cpu_id]      "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 102                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 103                   [v]                   "Qo" (*v),
 104                   [expectnot]           "r" (expectnot),
 105                   [load]                "Qo" (*load),
 106                   [voffp]               "r" (voffp)
 107                   RSEQ_INJECT_INPUT
 108                 : "memory", RSEQ_ASM_TMP_REG
 109                 : abort, cmpfail
 110 #ifdef RSEQ_COMPARE_TWICE
 111                   , error1, error2
 112 #endif
 113         );
 114         rseq_after_asm_goto();
 115         return 0;
 116 abort:
 117         rseq_after_asm_goto();
 118         RSEQ_INJECT_FAILED
 119         return -1;
 120 cmpfail:
 121         rseq_after_asm_goto();
 122         return 1;
 123 #ifdef RSEQ_COMPARE_TWICE
 124 error1:
 125         rseq_after_asm_goto();
 126         rseq_bug("cpu_id comparison failed");
 127 error2:
 128         rseq_after_asm_goto();
 129         rseq_bug("expected value comparison failed");
 130 #endif
 131 }
 132
 133 static inline __attribute__((always_inline))
 134 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 135 {
 136         RSEQ_INJECT_C(9)
 137
 138         __asm__ __volatile__ goto (
 139                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 140 #ifdef RSEQ_COMPARE_TWICE
 141                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 142 #endif
 143                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 144                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 145                 RSEQ_INJECT_ASM(3)
 146 #ifdef RSEQ_COMPARE_TWICE
 147                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 148 #endif
 149                 RSEQ_ASM_OP_R_LOAD(v)
 150                 RSEQ_ASM_OP_R_ADD(count)
 151                 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 152                 RSEQ_INJECT_ASM(4)
 153                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 154                 : /* gcc asm goto does not allow outputs */
 155                 : [cpu_id]              "r" (cpu),
 156                   [current_cpu_id]      "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 157                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 158                   [v]                   "Qo" (*v),
 159                   [count]               "r" (count)
 160                   RSEQ_INJECT_INPUT
 161                 : "memory", RSEQ_ASM_TMP_REG
 162                 : abort
 163 #ifdef RSEQ_COMPARE_TWICE
 164                   , error1
 165 #endif
 166         );
 167         rseq_after_asm_goto();
 168         return 0;
 169 abort:
 170         rseq_after_asm_goto();
 171         RSEQ_INJECT_FAILED
 172         return -1;
 173 #ifdef RSEQ_COMPARE_TWICE
 174 error1:
 175         rseq_after_asm_goto();
 176         rseq_bug("cpu_id comparison failed");
 177 #endif
 178 }
 179
 180 static inline __attribute__((always_inline))
 181 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 182                               intptr_t *v2, intptr_t expect2,
 183                               intptr_t newv, int cpu)
 184 {
 185         RSEQ_INJECT_C(9)
 186
 187         __asm__ __volatile__ goto (
 188                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 189                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 190 #ifdef RSEQ_COMPARE_TWICE
 191                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 192                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 193                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
 194 #endif
 195                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 196                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 197                 RSEQ_INJECT_ASM(3)
 198                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 199                 RSEQ_INJECT_ASM(4)
 200                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 201                 RSEQ_INJECT_ASM(5)
 202 #ifdef RSEQ_COMPARE_TWICE
 203                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 204                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 205                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 206 #endif
 207                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 208                 RSEQ_INJECT_ASM(6)
 209                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 210                 : /* gcc asm goto does not allow outputs */
 211                 : [cpu_id]              "r" (cpu),
 212                   [current_cpu_id]      "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 213                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 214                   [v]                   "Qo" (*v),
 215                   [expect]              "r" (expect),
 216                   [v2]                  "Qo" (*v2),
 217                   [expect2]             "r" (expect2),
 218                   [newv]                "r" (newv)
 219                   RSEQ_INJECT_INPUT
 220                 : "memory", RSEQ_ASM_TMP_REG
 221                 : abort, cmpfail
 222 #ifdef RSEQ_COMPARE_TWICE
 223                   , error1, error2, error3
 224 #endif
 225         );
 226         rseq_after_asm_goto();
 227         return 0;
 228 abort:
 229         rseq_after_asm_goto();
 230         RSEQ_INJECT_FAILED
 231         return -1;
 232 cmpfail:
 233         rseq_after_asm_goto();
 234         return 1;
 235 #ifdef RSEQ_COMPARE_TWICE
 236 error1:
 237         rseq_after_asm_goto();
 238         rseq_bug("cpu_id comparison failed");
 239 error2:
 240         rseq_after_asm_goto();
 241         rseq_bug("expected value comparison failed");
 242 error3:
 243         rseq_after_asm_goto();
 244         rseq_bug("2nd expected value comparison failed");
 245 #endif
 246 }
 247
 248 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 249         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 250
 251 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 252         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 253
 254 static inline __attribute__((always_inline))
 255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 256                                  intptr_t *v2, intptr_t newv2,
 257                                  intptr_t newv, int cpu)
 258 {
 259         RSEQ_INJECT_C(9)
 260
 261         __asm__ __volatile__ goto (
 262                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 263                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 264 #ifdef RSEQ_COMPARE_TWICE
 265                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 266                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 267 #endif
 268                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 269                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 270                 RSEQ_INJECT_ASM(3)
 271                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 272                 RSEQ_INJECT_ASM(4)
 273 #ifdef RSEQ_COMPARE_TWICE
 274                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 275                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 276 #endif
 277                 RSEQ_ASM_OP_STORE(newv2, v2)
 278                 RSEQ_INJECT_ASM(5)
 279 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 280                 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 281 #else
 282                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 283 #endif
 284                 RSEQ_INJECT_ASM(6)
 285                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 286                 : /* gcc asm goto does not allow outputs */
 287                 : [cpu_id]              "r" (cpu),
 288                   [current_cpu_id]      "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 289                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 290                   [expect]              "r" (expect),
 291                   [v]                   "Qo" (*v),
 292                   [newv]                "r" (newv),
 293                   [v2]                  "Qo" (*v2),
 294                   [newv2]               "r" (newv2)
 295                   RSEQ_INJECT_INPUT
 296                 : "memory", RSEQ_ASM_TMP_REG
 297                 : abort, cmpfail
 298 #ifdef RSEQ_COMPARE_TWICE
 299                   , error1, error2
 300 #endif
 301         );
 302         rseq_after_asm_goto();
 303         return 0;
 304 abort:
 305         rseq_after_asm_goto();
 306         RSEQ_INJECT_FAILED
 307         return -1;
 308 cmpfail:
 309         rseq_after_asm_goto();
 310         return 1;
 311 #ifdef RSEQ_COMPARE_TWICE
 312 error1:
 313         rseq_after_asm_goto();
 314         rseq_bug("cpu_id comparison failed");
 315 error2:
 316         rseq_after_asm_goto();
 317         rseq_bug("expected value comparison failed");
 318 #endif
 319 }
 320
 321 static inline __attribute__((always_inline))
 322 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 323                                  void *dst, void *src, size_t len,
 324                                  intptr_t newv, int cpu)
 325 {
 326         RSEQ_INJECT_C(9)
 327
 328         __asm__ __volatile__ goto (
 329                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 330                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 331 #ifdef RSEQ_COMPARE_TWICE
 332                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 333                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 334 #endif
 335                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 336                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 337                 RSEQ_INJECT_ASM(3)
 338                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 339                 RSEQ_INJECT_ASM(4)
 340 #ifdef RSEQ_COMPARE_TWICE
 341                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 342                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 343 #endif
 344                 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 345                 RSEQ_INJECT_ASM(5)
 346 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 347                 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 348 #else
 349                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 350 #endif
 351                 RSEQ_INJECT_ASM(6)
 352                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 353                 : /* gcc asm goto does not allow outputs */
 354                 : [cpu_id]              "r" (cpu),
 355                   [current_cpu_id]      "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 356                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 357                   [expect]              "r" (expect),
 358                   [v]                   "Qo" (*v),
 359                   [newv]                "r" (newv),
 360                   [dst]                 "r" (dst),
 361                   [src]                 "r" (src),
 362                   [len]                 "r" (len)
 363                   RSEQ_INJECT_INPUT
 364                 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
 365                 : abort, cmpfail
 366 #ifdef RSEQ_COMPARE_TWICE
 367                   , error1, error2
 368 #endif
 369         );
 370         rseq_after_asm_goto();
 371         return 0;
 372 abort:
 373         rseq_after_asm_goto();
 374         RSEQ_INJECT_FAILED
 375         return -1;
 376 cmpfail:
 377         rseq_after_asm_goto();
 378         return 1;
 379 #ifdef RSEQ_COMPARE_TWICE
 380 error1:
 381         rseq_after_asm_goto();
 382         rseq_bug("cpu_id comparison failed");
 383 error2:
 384         rseq_after_asm_goto();
 385         rseq_bug("expected value comparison failed");
 386 #endif
 387 }
 388
 389 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 390         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 391
 392 #include "rseq-bits-reset.h"