arch/arc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   4  */
   5
   6 #ifndef _ASM_ARC_ATOMIC_H
   7 #define _ASM_ARC_ATOMIC_H
   8
   9 #ifndef __ASSEMBLY__
  10
  11 #include <linux/types.h>
  12 #include <linux/compiler.h>
  13 #include <asm/cmpxchg.h>
  14 #include <asm/barrier.h>
  15 #include <asm/smp.h>
  16
  17 #ifndef CONFIG_ARC_PLAT_EZNPS
  18
  19 #define atomic_read(v)  READ_ONCE((v)->counter)
  20
  21 #ifdef CONFIG_ARC_HAS_LLSC
  22
  23 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  24
  25 #define ATOMIC_OP(op, c_op, asm_op)                                     \
  26 static inline void atomic_##op(int i, atomic_t *v)                      \
  27 {                                                                       \
  28         unsigned int val;                                               \
  29                                                                         \
  30         __asm__ __volatile__(                                           \
  31         "1:     llock   %[val], [%[ctr]]                \n"             \
  32         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  33         "       scond   %[val], [%[ctr]]                \n"             \
  34         "       bnz     1b                              \n"             \
  35         : [val] "=&r"   (val) /* Early clobber to prevent reg reuse */  \
  36         : [ctr] "r"     (&v->counter), /* Not "m": llock only supports reg direct addr mode */  \
  37           [i]   "ir"    (i)                                             \
  38         : "cc");                                                        \
  39 }                                                                       \
  40
  41 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
  42 static inline int atomic_##op##_return(int i, atomic_t *v)              \
  43 {                                                                       \
  44         unsigned int val;                                               \
  45                                                                         \
  46         /*                                                              \
  47          * Explicit full memory barrier needed before/after as          \
  48          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  49          */                                                             \
  50         smp_mb();                                                       \
  51                                                                         \
  52         __asm__ __volatile__(                                           \
  53         "1:     llock   %[val], [%[ctr]]                \n"             \
  54         "       " #asm_op " %[val], %[val], %[i]        \n"             \
  55         "       scond   %[val], [%[ctr]]                \n"             \
  56         "       bnz     1b                              \n"             \
  57         : [val] "=&r"   (val)                                           \
  58         : [ctr] "r"     (&v->counter),                                  \
  59           [i]   "ir"    (i)                                             \
  60         : "cc");                                                        \
  61                                                                         \
  62         smp_mb();                                                       \
  63                                                                         \
  64         return val;                                                     \
  65 }
  66
  67 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
  68 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
  69 {                                                                       \
  70         unsigned int val, orig;                                         \
  71                                                                         \
  72         /*                                                              \
  73          * Explicit full memory barrier needed before/after as          \
  74          * LLOCK/SCOND thmeselves don't provide any such semantics      \
  75          */                                                             \
  76         smp_mb();                                                       \
  77                                                                         \
  78         __asm__ __volatile__(                                           \
  79         "1:     llock   %[orig], [%[ctr]]               \n"             \
  80         "       " #asm_op " %[val], %[orig], %[i]       \n"             \
  81         "       scond   %[val], [%[ctr]]                \n"             \
  82         "       bnz     1b                              \n"             \
  83         : [val] "=&r"   (val),                                          \
  84           [orig] "=&r" (orig)                                           \
  85         : [ctr] "r"     (&v->counter),                                  \
  86           [i]   "ir"    (i)                                             \
  87         : "cc");                                                        \
  88                                                                         \
  89         smp_mb();                                                       \
  90                                                                         \
  91         return orig;                                                    \
  92 }
  93
  94 #else   /* !CONFIG_ARC_HAS_LLSC */
  95
  96 #ifndef CONFIG_SMP
  97
  98  /* violating atomic_xxx API locking protocol in UP for optimization sake */
  99 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
 100
 101 #else
 102
 103 static inline void atomic_set(atomic_t *v, int i)
 104 {
 105         /*
 106          * Independent of hardware support, all of the atomic_xxx() APIs need
 107          * to follow the same locking rules to make sure that a "hardware"
 108          * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
 109          * sequence
 110          *
 111          * Thus atomic_set() despite being 1 insn (and seemingly atomic)
 112          * requires the locking.
 113          */
 114         unsigned long flags;
 115
 116         atomic_ops_lock(flags);
 117         WRITE_ONCE(v->counter, i);
 118         atomic_ops_unlock(flags);
 119 }
 120
 121 #define atomic_set_release(v, i)        atomic_set((v), (i))
 122
 123 #endif
 124
 125 /*
 126  * Non hardware assisted Atomic-R-M-W
 127  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 128  */
 129
 130 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 131 static inline void atomic_##op(int i, atomic_t *v)                      \
 132 {                                                                       \
 133         unsigned long flags;                                            \
 134                                                                         \
 135         atomic_ops_lock(flags);                                         \
 136         v->counter c_op i;                                              \
 137         atomic_ops_unlock(flags);                                       \
 138 }
 139
 140 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 141 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 142 {                                                                       \
 143         unsigned long flags;                                            \
 144         unsigned long temp;                                             \
 145                                                                         \
 146         /*                                                              \
 147          * spin lock/unlock provides the needed smp_mb() before/after   \
 148          */                                                             \
 149         atomic_ops_lock(flags);                                         \
 150         temp = v->counter;                                              \
 151         temp c_op i;                                                    \
 152         v->counter = temp;                                              \
 153         atomic_ops_unlock(flags);                                       \
 154                                                                         \
 155         return temp;                                                    \
 156 }
 157
 158 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 159 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 160 {                                                                       \
 161         unsigned long flags;                                            \
 162         unsigned long orig;                                             \
 163                                                                         \
 164         /*                                                              \
 165          * spin lock/unlock provides the needed smp_mb() before/after   \
 166          */                                                             \
 167         atomic_ops_lock(flags);                                         \
 168         orig = v->counter;                                              \
 169         v->counter c_op i;                                              \
 170         atomic_ops_unlock(flags);                                       \
 171                                                                         \
 172         return orig;                                                    \
 173 }
 174
 175 #endif /* !CONFIG_ARC_HAS_LLSC */
 176
 177 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 178         ATOMIC_OP(op, c_op, asm_op)                                     \
 179         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 180         ATOMIC_FETCH_OP(op, c_op, asm_op)
 181
 182 ATOMIC_OPS(add, +=, add)
 183 ATOMIC_OPS(sub, -=, sub)
 184
 185 #define atomic_andnot           atomic_andnot
 186 #define atomic_fetch_andnot     atomic_fetch_andnot
 187
 188 #undef ATOMIC_OPS
 189 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 190         ATOMIC_OP(op, c_op, asm_op)                                     \
 191         ATOMIC_FETCH_OP(op, c_op, asm_op)
 192
 193 ATOMIC_OPS(and, &=, and)
 194 ATOMIC_OPS(andnot, &= ~, bic)
 195 ATOMIC_OPS(or, |=, or)
 196 ATOMIC_OPS(xor, ^=, xor)
 197
 198 #else /* CONFIG_ARC_PLAT_EZNPS */
 199
 200 static inline int atomic_read(const atomic_t *v)
 201 {
 202         int temp;
 203
 204         __asm__ __volatile__(
 205         "       ld.di %0, [%1]"
 206         : "=r"(temp)
 207         : "r"(&v->counter)
 208         : "memory");
 209         return temp;
 210 }
 211
 212 static inline void atomic_set(atomic_t *v, int i)
 213 {
 214         __asm__ __volatile__(
 215         "       st.di %0,[%1]"
 216         :
 217         : "r"(i), "r"(&v->counter)
 218         : "memory");
 219 }
 220
 221 #define ATOMIC_OP(op, c_op, asm_op)                                     \
 222 static inline void atomic_##op(int i, atomic_t *v)                      \
 223 {                                                                       \
 224         __asm__ __volatile__(                                           \
 225         "       mov r2, %0\n"                                           \
 226         "       mov r3, %1\n"                                           \
 227         "       .word %2\n"                                             \
 228         :                                                               \
 229         : "r"(i), "r"(&v->counter), "i"(asm_op)                         \
 230         : "r2", "r3", "memory");                                        \
 231 }                                                                       \
 232
 233 #define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 234 static inline int atomic_##op##_return(int i, atomic_t *v)              \
 235 {                                                                       \
 236         unsigned int temp = i;                                          \
 237                                                                         \
 238         /* Explicit full memory barrier needed before/after */          \
 239         smp_mb();                                                       \
 240                                                                         \
 241         __asm__ __volatile__(                                           \
 242         "       mov r2, %0\n"                                           \
 243         "       mov r3, %1\n"                                           \
 244         "       .word %2\n"                                             \
 245         "       mov %0, r2"                                             \
 246         : "+r"(temp)                                                    \
 247         : "r"(&v->counter), "i"(asm_op)                                 \
 248         : "r2", "r3", "memory");                                        \
 249                                                                         \
 250         smp_mb();                                                       \
 251                                                                         \
 252         temp c_op i;                                                    \
 253                                                                         \
 254         return temp;                                                    \
 255 }
 256
 257 #define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 258 static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 259 {                                                                       \
 260         unsigned int temp = i;                                          \
 261                                                                         \
 262         /* Explicit full memory barrier needed before/after */          \
 263         smp_mb();                                                       \
 264                                                                         \
 265         __asm__ __volatile__(                                           \
 266         "       mov r2, %0\n"                                           \
 267         "       mov r3, %1\n"                                           \
 268         "       .word %2\n"                                             \
 269         "       mov %0, r2"                                             \
 270         : "+r"(temp)                                                    \
 271         : "r"(&v->counter), "i"(asm_op)                                 \
 272         : "r2", "r3", "memory");                                        \
 273                                                                         \
 274         smp_mb();                                                       \
 275                                                                         \
 276         return temp;                                                    \
 277 }
 278
 279 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 280         ATOMIC_OP(op, c_op, asm_op)                                     \
 281         ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 282         ATOMIC_FETCH_OP(op, c_op, asm_op)
 283
 284 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
 285 #define atomic_sub(i, v) atomic_add(-(i), (v))
 286 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
 287 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
 288
 289 #undef ATOMIC_OPS
 290 #define ATOMIC_OPS(op, c_op, asm_op)                                    \
 291         ATOMIC_OP(op, c_op, asm_op)                                     \
 292         ATOMIC_FETCH_OP(op, c_op, asm_op)
 293
 294 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
 295 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
 296 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
 297
 298 #endif /* CONFIG_ARC_PLAT_EZNPS */
 299
 300 #undef ATOMIC_OPS
 301 #undef ATOMIC_FETCH_OP
 302 #undef ATOMIC_OP_RETURN
 303 #undef ATOMIC_OP
 304
 305 #ifdef CONFIG_GENERIC_ATOMIC64
 306
 307 #include <asm-generic/atomic64.h>
 308
 309 #else   /* Kconfig ensures this is only enabled with needed h/w assist */
 310
 311 /*
 312  * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 313  *  - The address HAS to be 64-bit aligned
 314  *  - There are 2 semantics involved here:
 315  *    = exclusive implies no interim update between load/store to same addr
 316  *    = both words are observed/updated together: this is guaranteed even
 317  *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
 318  *      is NOT required to use LLOCKD+SCONDD, STD suffices
 319  */
 320
 321 typedef struct {
 322         s64 __aligned(8) counter;
 323 } atomic64_t;
 324
 325 #define ATOMIC64_INIT(a) { (a) }
 326
 327 static inline s64 atomic64_read(const atomic64_t *v)
 328 {
 329         s64 val;
 330
 331         __asm__ __volatile__(
 332         "       ldd   %0, [%1]  \n"
 333         : "=r"(val)
 334         : "r"(&v->counter));
 335
 336         return val;
 337 }
 338
 339 static inline void atomic64_set(atomic64_t *v, s64 a)
 340 {
 341         /*
 342          * This could have been a simple assignment in "C" but would need
 343          * explicit volatile. Otherwise gcc optimizers could elide the store
 344          * which borked atomic64 self-test
 345          * In the inline asm version, memory clobber needed for exact same
 346          * reason, to tell gcc about the store.
 347          *
 348          * This however is not needed for sibling atomic64_add() etc since both
 349          * load/store are explicitly done in inline asm. As long as API is used
 350          * for each access, gcc has no way to optimize away any load/store
 351          */
 352         __asm__ __volatile__(
 353         "       std   %0, [%1]  \n"
 354         :
 355         : "r"(a), "r"(&v->counter)
 356         : "memory");
 357 }
 358
 359 #define ATOMIC64_OP(op, op1, op2)                                       \
 360 static inline void atomic64_##op(s64 a, atomic64_t *v)                  \
 361 {                                                                       \
 362         s64 val;                                                        \
 363                                                                         \
 364         __asm__ __volatile__(                                           \
 365         "1:                             \n"                             \
 366         "       llockd  %0, [%1]        \n"                             \
 367         "       " #op1 " %L0, %L0, %L2  \n"                             \
 368         "       " #op2 " %H0, %H0, %H2  \n"                             \
 369         "       scondd   %0, [%1]       \n"                             \
 370         "       bnz     1b              \n"                             \
 371         : "=&r"(val)                                                    \
 372         : "r"(&v->counter), "ir"(a)                                     \
 373         : "cc");                                                        \
 374 }                                                                       \
 375
 376 #define ATOMIC64_OP_RETURN(op, op1, op2)                                \
 377 static inline s64 atomic64_##op##_return(s64 a, atomic64_t *v)          \
 378 {                                                                       \
 379         s64 val;                                                        \
 380                                                                         \
 381         smp_mb();                                                       \
 382                                                                         \
 383         __asm__ __volatile__(                                           \
 384         "1:                             \n"                             \
 385         "       llockd   %0, [%1]       \n"                             \
 386         "       " #op1 " %L0, %L0, %L2  \n"                             \
 387         "       " #op2 " %H0, %H0, %H2  \n"                             \
 388         "       scondd   %0, [%1]       \n"                             \
 389         "       bnz     1b              \n"                             \
 390         : [val] "=&r"(val)                                              \
 391         : "r"(&v->counter), "ir"(a)                                     \
 392         : "cc");        /* memory clobber comes from smp_mb() */        \
 393                                                                         \
 394         smp_mb();                                                       \
 395                                                                         \
 396         return val;                                                     \
 397 }
 398
 399 #define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
 400 static inline s64 atomic64_fetch_##op(s64 a, atomic64_t *v)             \
 401 {                                                                       \
 402         s64 val, orig;                                                  \
 403                                                                         \
 404         smp_mb();                                                       \
 405                                                                         \
 406         __asm__ __volatile__(                                           \
 407         "1:                             \n"                             \
 408         "       llockd   %0, [%2]       \n"                             \
 409         "       " #op1 " %L1, %L0, %L3  \n"                             \
 410         "       " #op2 " %H1, %H0, %H3  \n"                             \
 411         "       scondd   %1, [%2]       \n"                             \
 412         "       bnz     1b              \n"                             \
 413         : "=&r"(orig), "=&r"(val)                                       \
 414         : "r"(&v->counter), "ir"(a)                                     \
 415         : "cc");        /* memory clobber comes from smp_mb() */        \
 416                                                                         \
 417         smp_mb();                                                       \
 418                                                                         \
 419         return orig;                                                    \
 420 }
 421
 422 #define ATOMIC64_OPS(op, op1, op2)                                      \
 423         ATOMIC64_OP(op, op1, op2)                                       \
 424         ATOMIC64_OP_RETURN(op, op1, op2)                                \
 425         ATOMIC64_FETCH_OP(op, op1, op2)
 426
 427 #define atomic64_andnot         atomic64_andnot
 428 #define atomic64_fetch_andnot   atomic64_fetch_andnot
 429
 430 ATOMIC64_OPS(add, add.f, adc)
 431 ATOMIC64_OPS(sub, sub.f, sbc)
 432 ATOMIC64_OPS(and, and, and)
 433 ATOMIC64_OPS(andnot, bic, bic)
 434 ATOMIC64_OPS(or, or, or)
 435 ATOMIC64_OPS(xor, xor, xor)
 436
 437 #undef ATOMIC64_OPS
 438 #undef ATOMIC64_FETCH_OP
 439 #undef ATOMIC64_OP_RETURN
 440 #undef ATOMIC64_OP
 441
 442 static inline s64
 443 atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
 444 {
 445         s64 prev;
 446
 447         smp_mb();
 448
 449         __asm__ __volatile__(
 450         "1:     llockd  %0, [%1]        \n"
 451         "       brne    %L0, %L2, 2f    \n"
 452         "       brne    %H0, %H2, 2f    \n"
 453         "       scondd  %3, [%1]        \n"
 454         "       bnz     1b              \n"
 455         "2:                             \n"
 456         : "=&r"(prev)
 457         : "r"(ptr), "ir"(expected), "r"(new)
 458         : "cc");        /* memory clobber comes from smp_mb() */
 459
 460         smp_mb();
 461
 462         return prev;
 463 }
 464
 465 static inline s64 atomic64_xchg(atomic64_t *ptr, s64 new)
 466 {
 467         s64 prev;
 468
 469         smp_mb();
 470
 471         __asm__ __volatile__(
 472         "1:     llockd  %0, [%1]        \n"
 473         "       scondd  %2, [%1]        \n"
 474         "       bnz     1b              \n"
 475         "2:                             \n"
 476         : "=&r"(prev)
 477         : "r"(ptr), "r"(new)
 478         : "cc");        /* memory clobber comes from smp_mb() */
 479
 480         smp_mb();
 481
 482         return prev;
 483 }
 484
 485 /**
 486  * atomic64_dec_if_positive - decrement by 1 if old value positive
 487  * @v: pointer of type atomic64_t
 488  *
 489  * The function returns the old value of *v minus 1, even if
 490  * the atomic variable, v, was not decremented.
 491  */
 492
 493 static inline s64 atomic64_dec_if_positive(atomic64_t *v)
 494 {
 495         s64 val;
 496
 497         smp_mb();
 498
 499         __asm__ __volatile__(
 500         "1:     llockd  %0, [%1]        \n"
 501         "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 502         "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 503         "       brlt    %H0, 0, 2f      \n"
 504         "       scondd  %0, [%1]        \n"
 505         "       bnz     1b              \n"
 506         "2:                             \n"
 507         : "=&r"(val)
 508         : "r"(&v->counter)
 509         : "cc");        /* memory clobber comes from smp_mb() */
 510
 511         smp_mb();
 512
 513         return val;
 514 }
 515 #define atomic64_dec_if_positive atomic64_dec_if_positive
 516
 517 /**
 518  * atomic64_fetch_add_unless - add unless the number is a given value
 519  * @v: pointer of type atomic64_t
 520  * @a: the amount to add to v...
 521  * @u: ...unless v is equal to u.
 522  *
 523  * Atomically adds @a to @v, if it was not @u.
 524  * Returns the old value of @v
 525  */
 526 static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 527 {
 528         s64 old, temp;
 529
 530         smp_mb();
 531
 532         __asm__ __volatile__(
 533         "1:     llockd  %0, [%2]        \n"
 534         "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 535         "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 536         "2:                             \n"
 537         "       add.f   %L1, %L0, %L3   \n"
 538         "       adc     %H1, %H0, %H3   \n"
 539         "       scondd  %1, [%2]        \n"
 540         "       bnz     1b              \n"
 541         "3:                             \n"
 542         : "=&r"(old), "=&r" (temp)
 543         : "r"(&v->counter), "r"(a), "r"(u)
 544         : "cc");        /* memory clobber comes from smp_mb() */
 545
 546         smp_mb();
 547
 548         return old;
 549 }
 550 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
 551
 552 #endif  /* !CONFIG_GENERIC_ATOMIC64 */
 553
 554 #endif  /* !__ASSEMBLY__ */
 555
 556 #endif