tools/testing/selftests/rseq/rseq-riscv-bits.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2
   3 #include "rseq-bits-template.h"
   4
   5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
   6         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
   7
   8 static inline __always_inline
   9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  10 {
  11         RSEQ_INJECT_C(9)
  12
  13         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
  14                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
  15 #ifdef RSEQ_COMPARE_TWICE
  16                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
  17                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
  18 #endif
  19                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
  20                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  21                                   RSEQ_INJECT_ASM(3)
  22                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
  23                                   RSEQ_INJECT_ASM(4)
  24 #ifdef RSEQ_COMPARE_TWICE
  25                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
  26                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
  27 #endif
  28                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
  29                                   RSEQ_INJECT_ASM(5)
  30                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
  31                                   : /* gcc asm goto does not allow outputs */
  32                                   : [cpu_id]            "r" (cpu),
  33                                     [current_cpu_id]    "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  34                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  35                                     [v]                 "m" (*v),
  36                                     [expect]            "r" (expect),
  37                                     [newv]              "r" (newv)
  38                                     RSEQ_INJECT_INPUT
  39                                   : "memory", RSEQ_ASM_TMP_REG_1
  40                                     RSEQ_INJECT_CLOBBER
  41                                   : abort, cmpfail
  42 #ifdef RSEQ_COMPARE_TWICE
  43                                     , error1, error2
  44 #endif
  45         );
  46
  47         return 0;
  48 abort:
  49         RSEQ_INJECT_FAILED
  50         return -1;
  51 cmpfail:
  52         return 1;
  53 #ifdef RSEQ_COMPARE_TWICE
  54 error1:
  55         rseq_bug("cpu_id comparison failed");
  56 error2:
  57         rseq_bug("expected value comparison failed");
  58 #endif
  59 }
  60
  61 static inline __always_inline
  62 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  63                                off_t voffp, intptr_t *load, int cpu)
  64 {
  65         RSEQ_INJECT_C(9)
  66
  67         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
  68                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
  69 #ifdef RSEQ_COMPARE_TWICE
  70                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
  71                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
  72 #endif
  73                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
  74                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  75                                   RSEQ_INJECT_ASM(3)
  76                                   RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
  77                                   RSEQ_INJECT_ASM(4)
  78 #ifdef RSEQ_COMPARE_TWICE
  79                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
  80                                   RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
  81 #endif
  82                                   RSEQ_ASM_OP_R_LOAD(v)
  83                                   RSEQ_ASM_OP_R_STORE(load)
  84                                   RSEQ_ASM_OP_R_LOAD_OFF(voffp)
  85                                   RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
  86                                   RSEQ_INJECT_ASM(5)
  87                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
  88                                   : /* gcc asm goto does not allow outputs */
  89                                   : [cpu_id]            "r" (cpu),
  90                                     [current_cpu_id]    "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  91                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  92                                     [v]                 "m" (*v),
  93                                     [expectnot]         "r" (expectnot),
  94                                     [load]              "m" (*load),
  95                                     [voffp]             "r" (voffp)
  96                                     RSEQ_INJECT_INPUT
  97                                   : "memory", RSEQ_ASM_TMP_REG_1
  98                                     RSEQ_INJECT_CLOBBER
  99                                   : abort, cmpfail
 100 #ifdef RSEQ_COMPARE_TWICE
 101                                     , error1, error2
 102 #endif
 103         );
 104         return 0;
 105 abort:
 106         RSEQ_INJECT_FAILED
 107         return -1;
 108 cmpfail:
 109         return 1;
 110 #ifdef RSEQ_COMPARE_TWICE
 111 error1:
 112         rseq_bug("cpu_id comparison failed");
 113 error2:
 114         rseq_bug("expected value comparison failed");
 115 #endif
 116 }
 117
 118 static inline __always_inline
 119 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 120 {
 121         RSEQ_INJECT_C(9)
 122
 123         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 124 #ifdef RSEQ_COMPARE_TWICE
 125                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 126 #endif
 127                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 128                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 129                                   RSEQ_INJECT_ASM(3)
 130 #ifdef RSEQ_COMPARE_TWICE
 131                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 132 #endif
 133                                   RSEQ_ASM_OP_R_LOAD(v)
 134                                   RSEQ_ASM_OP_R_ADD(count)
 135                                   RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 136                                   RSEQ_INJECT_ASM(4)
 137                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 138                                   : /* gcc asm goto does not allow outputs */
 139                                   : [cpu_id]            "r" (cpu),
 140                                     [current_cpu_id]    "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 141                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 142                                     [v]                 "m" (*v),
 143                                     [count]             "r" (count)
 144                                     RSEQ_INJECT_INPUT
 145                                   : "memory", RSEQ_ASM_TMP_REG_1
 146                                     RSEQ_INJECT_CLOBBER
 147                                   : abort
 148 #ifdef RSEQ_COMPARE_TWICE
 149                                     , error1
 150 #endif
 151         );
 152         return 0;
 153 abort:
 154         RSEQ_INJECT_FAILED
 155         return -1;
 156 #ifdef RSEQ_COMPARE_TWICE
 157 error1:
 158         rseq_bug("cpu_id comparison failed");
 159 #endif
 160 }
 161
 162 static inline __always_inline
 163 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 164                               intptr_t *v2, intptr_t expect2,
 165                               intptr_t newv, int cpu)
 166 {
 167         RSEQ_INJECT_C(9)
 168
 169         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 170                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 171 #ifdef RSEQ_COMPARE_TWICE
 172                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 173                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 174                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
 175 #endif
 176                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 177                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 178                                   RSEQ_INJECT_ASM(3)
 179                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 180                                   RSEQ_INJECT_ASM(4)
 181                                   RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
 182                                   RSEQ_INJECT_ASM(5)
 183 #ifdef RSEQ_COMPARE_TWICE
 184                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 185                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 186                                   RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
 187 #endif
 188                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 189                                   RSEQ_INJECT_ASM(6)
 190                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 191                                   : /* gcc asm goto does not allow outputs */
 192                                   : [cpu_id]            "r" (cpu),
 193                                     [current_cpu_id]    "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 194                                     [rseq_cs]           "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 195                                     [v]                 "m" (*v),
 196                                     [expect]            "r" (expect),
 197                                     [v2]                        "m" (*v2),
 198                                     [expect2]           "r" (expect2),
 199                                     [newv]              "r" (newv)
 200                                     RSEQ_INJECT_INPUT
 201                                   : "memory", RSEQ_ASM_TMP_REG_1
 202                                     RSEQ_INJECT_CLOBBER
 203                                   : abort, cmpfail
 204 #ifdef RSEQ_COMPARE_TWICE
 205                                     , error1, error2, error3
 206 #endif
 207         );
 208
 209         return 0;
 210 abort:
 211         RSEQ_INJECT_FAILED
 212         return -1;
 213 cmpfail:
 214         return 1;
 215 #ifdef RSEQ_COMPARE_TWICE
 216 error1:
 217         rseq_bug("cpu_id comparison failed");
 218 error2:
 219         rseq_bug("expected value comparison failed");
 220 error3:
 221         rseq_bug("2nd expected value comparison failed");
 222 #endif
 223 }
 224
 225 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
 226
 227 /*
 228  *   pval = *(ptr+off)
 229  *  *pval += inc;
 230  */
 231 static inline __always_inline
 232 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
 233 {
 234         RSEQ_INJECT_C(9)
 235
 236         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 237 #ifdef RSEQ_COMPARE_TWICE
 238                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 239 #endif
 240                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 241                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 242                                   RSEQ_INJECT_ASM(3)
 243 #ifdef RSEQ_COMPARE_TWICE
 244                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 245 #endif
 246                                   RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
 247                                   RSEQ_INJECT_ASM(4)
 248                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 249                                   : /* gcc asm goto does not allow outputs */
 250                                   : [cpu_id]                    "r" (cpu),
 251                                     [current_cpu_id]            "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 252                                     [rseq_cs]                   "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 253                                     [ptr]                       "r" (ptr),
 254                                     [off]                       "er" (off),
 255                                     [inc]                       "er" (inc)
 256                                     RSEQ_INJECT_INPUT
 257                                   : "memory", RSEQ_ASM_TMP_REG_1
 258                                     RSEQ_INJECT_CLOBBER
 259                                   : abort
 260 #ifdef RSEQ_COMPARE_TWICE
 261                                     , error1
 262 #endif
 263         );
 264         return 0;
 265 abort:
 266         RSEQ_INJECT_FAILED
 267         return -1;
 268 #ifdef RSEQ_COMPARE_TWICE
 269 error1:
 270         rseq_bug("cpu_id comparison failed");
 271 #endif
 272 }
 273
 274 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 275         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 276
 277 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 278         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 279
 280 static inline __always_inline
 281 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 282                                  intptr_t *v2, intptr_t newv2,
 283                                  intptr_t newv, int cpu)
 284 {
 285         RSEQ_INJECT_C(9)
 286
 287         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 288                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 289 #ifdef RSEQ_COMPARE_TWICE
 290                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 291                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 292 #endif
 293                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 294                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 295                                   RSEQ_INJECT_ASM(3)
 296                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 297                                   RSEQ_INJECT_ASM(4)
 298 #ifdef RSEQ_COMPARE_TWICE
 299                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 300                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 301 #endif
 302                                   RSEQ_ASM_OP_STORE(newv2, v2)
 303                                   RSEQ_INJECT_ASM(5)
 304 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 305                                   RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 306 #else
 307                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 308 #endif
 309                                   RSEQ_INJECT_ASM(6)
 310                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 311                                   : /* gcc asm goto does not allow outputs */
 312                                   : [cpu_id]                    "r" (cpu),
 313                                     [current_cpu_id]            "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 314                                     [rseq_cs]                   "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 315                                     [expect]                    "r" (expect),
 316                                     [v]                         "m" (*v),
 317                                     [newv]                      "r" (newv),
 318                                     [v2]                        "m" (*v2),
 319                                     [newv2]                     "r" (newv2)
 320                                     RSEQ_INJECT_INPUT
 321                                   : "memory", RSEQ_ASM_TMP_REG_1
 322                                     RSEQ_INJECT_CLOBBER
 323                                   : abort, cmpfail
 324 #ifdef RSEQ_COMPARE_TWICE
 325                                     , error1, error2
 326 #endif
 327         );
 328
 329         return 0;
 330 abort:
 331         RSEQ_INJECT_FAILED
 332         return -1;
 333 cmpfail:
 334         return 1;
 335 #ifdef RSEQ_COMPARE_TWICE
 336 error1:
 337         rseq_bug("cpu_id comparison failed");
 338 error2:
 339         rseq_bug("expected value comparison failed");
 340 #endif
 341 }
 342
 343 static inline __always_inline
 344 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 345                                  void *dst, void *src, size_t len,
 346                                  intptr_t newv, int cpu)
 347 {
 348         RSEQ_INJECT_C(9)
 349         __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 350                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
 351 #ifdef RSEQ_COMPARE_TWICE
 352                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
 353                                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
 354 #endif
 355                                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 356                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 357                                   RSEQ_INJECT_ASM(3)
 358                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
 359                                   RSEQ_INJECT_ASM(4)
 360 #ifdef RSEQ_COMPARE_TWICE
 361                                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
 362                                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
 363 #endif
 364                                   RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 365                                   RSEQ_INJECT_ASM(5)
 366 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 367                                   RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 368 #else
 369                                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 370 #endif
 371                                   RSEQ_INJECT_ASM(6)
 372                                   RSEQ_ASM_DEFINE_ABORT(4, abort)
 373                                   : /* gcc asm goto does not allow outputs */
 374                                   : [cpu_id]                    "r" (cpu),
 375                                     [current_cpu_id]            "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 376                                     [rseq_cs]                   "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 377                                     [expect]                    "r" (expect),
 378                                     [v]                         "m" (*v),
 379                                     [newv]                      "r" (newv),
 380                                     [dst]                       "r" (dst),
 381                                     [src]                       "r" (src),
 382                                     [len]                       "r" (len)
 383                                     RSEQ_INJECT_INPUT
 384                                   : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
 385                                     RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
 386                                     RSEQ_INJECT_CLOBBER
 387                                   : abort, cmpfail
 388 #ifdef RSEQ_COMPARE_TWICE
 389                                     , error1, error2
 390 #endif
 391         );
 392
 393         return 0;
 394 abort:
 395         RSEQ_INJECT_FAILED
 396         return -1;
 397 cmpfail:
 398         return 1;
 399 #ifdef RSEQ_COMPARE_TWICE
 400 error1:
 401         rseq_bug("cpu_id comparison failed");
 402 error2:
 403         rseq_bug("expected value comparison failed");
 404 #endif
 405 }
 406
 407 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 408         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 409
 410 #include "rseq-bits-reset.h"