arch/powerpc/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_POWERPC_ATOMIC_H_
   3 #define _ASM_POWERPC_ATOMIC_H_
   4
   5 /*
   6  * PowerPC atomic operations
   7  */
   8
   9 #ifdef __KERNEL__
  10 #include <linux/types.h>
  11 #include <asm/cmpxchg.h>
  12 #include <asm/barrier.h>
  13
  14 /*
  15  * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
  16  * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
  17  * on the platform without lwsync.
  18  */
  19 #define __atomic_acquire_fence()                                        \
  20         __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory")
  21
  22 #define __atomic_release_fence()                                        \
  23         __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory")
  24
  25 static __inline__ int atomic_read(const atomic_t *v)
  26 {
  27         int t;
  28
  29         __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
  30
  31         return t;
  32 }
  33
  34 static __inline__ void atomic_set(atomic_t *v, int i)
  35 {
  36         __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
  37 }
  38
  39 #define ATOMIC_OP(op, asm_op)                                           \
  40 static __inline__ void atomic_##op(int a, atomic_t *v)                  \
  41 {                                                                       \
  42         int t;                                                          \
  43                                                                         \
  44         __asm__ __volatile__(                                           \
  45 "1:     lwarx   %0,0,%3         # atomic_" #op "\n"                     \
  46         #asm_op " %0,%2,%0\n"                                           \
  47 "       stwcx.  %0,0,%3 \n"                                             \
  48 "       bne-    1b\n"                                                   \
  49         : "=&r" (t), "+m" (v->counter)                                  \
  50         : "r" (a), "r" (&v->counter)                                    \
  51         : "cc");                                                        \
  52 }                                                                       \
  53
  54 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op)                            \
  55 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v)      \
  56 {                                                                       \
  57         int t;                                                          \
  58                                                                         \
  59         __asm__ __volatile__(                                           \
  60 "1:     lwarx   %0,0,%3         # atomic_" #op "_return_relaxed\n"      \
  61         #asm_op " %0,%2,%0\n"                                           \
  62 "       stwcx.  %0,0,%3\n"                                              \
  63 "       bne-    1b\n"                                                   \
  64         : "=&r" (t), "+m" (v->counter)                                  \
  65         : "r" (a), "r" (&v->counter)                                    \
  66         : "cc");                                                        \
  67                                                                         \
  68         return t;                                                       \
  69 }
  70
  71 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op)                             \
  72 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v)       \
  73 {                                                                       \
  74         int res, t;                                                     \
  75                                                                         \
  76         __asm__ __volatile__(                                           \
  77 "1:     lwarx   %0,0,%4         # atomic_fetch_" #op "_relaxed\n"       \
  78         #asm_op " %1,%3,%0\n"                                           \
  79 "       stwcx.  %1,0,%4\n"                                              \
  80 "       bne-    1b\n"                                                   \
  81         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
  82         : "r" (a), "r" (&v->counter)                                    \
  83         : "cc");                                                        \
  84                                                                         \
  85         return res;                                                     \
  86 }
  87
  88 #define ATOMIC_OPS(op, asm_op)                                          \
  89         ATOMIC_OP(op, asm_op)                                           \
  90         ATOMIC_OP_RETURN_RELAXED(op, asm_op)                            \
  91         ATOMIC_FETCH_OP_RELAXED(op, asm_op)
  92
  93 ATOMIC_OPS(add, add)
  94 ATOMIC_OPS(sub, subf)
  95
  96 #define atomic_add_return_relaxed atomic_add_return_relaxed
  97 #define atomic_sub_return_relaxed atomic_sub_return_relaxed
  98
  99 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
 100 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
 101
 102 #undef ATOMIC_OPS
 103 #define ATOMIC_OPS(op, asm_op)                                          \
 104         ATOMIC_OP(op, asm_op)                                           \
 105         ATOMIC_FETCH_OP_RELAXED(op, asm_op)
 106
 107 ATOMIC_OPS(and, and)
 108 ATOMIC_OPS(or, or)
 109 ATOMIC_OPS(xor, xor)
 110
 111 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
 112 #define atomic_fetch_or_relaxed  atomic_fetch_or_relaxed
 113 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
 114
 115 #undef ATOMIC_OPS
 116 #undef ATOMIC_FETCH_OP_RELAXED
 117 #undef ATOMIC_OP_RETURN_RELAXED
 118 #undef ATOMIC_OP
 119
 120 static __inline__ void atomic_inc(atomic_t *v)
 121 {
 122         int t;
 123
 124         __asm__ __volatile__(
 125 "1:     lwarx   %0,0,%2         # atomic_inc\n\
 126         addic   %0,%0,1\n"
 127 "       stwcx.  %0,0,%2 \n\
 128         bne-    1b"
 129         : "=&r" (t), "+m" (v->counter)
 130         : "r" (&v->counter)
 131         : "cc", "xer");
 132 }
 133 #define atomic_inc atomic_inc
 134
 135 static __inline__ int atomic_inc_return_relaxed(atomic_t *v)
 136 {
 137         int t;
 138
 139         __asm__ __volatile__(
 140 "1:     lwarx   %0,0,%2         # atomic_inc_return_relaxed\n"
 141 "       addic   %0,%0,1\n"
 142 "       stwcx.  %0,0,%2\n"
 143 "       bne-    1b"
 144         : "=&r" (t), "+m" (v->counter)
 145         : "r" (&v->counter)
 146         : "cc", "xer");
 147
 148         return t;
 149 }
 150
 151 static __inline__ void atomic_dec(atomic_t *v)
 152 {
 153         int t;
 154
 155         __asm__ __volatile__(
 156 "1:     lwarx   %0,0,%2         # atomic_dec\n\
 157         addic   %0,%0,-1\n"
 158 "       stwcx.  %0,0,%2\n\
 159         bne-    1b"
 160         : "=&r" (t), "+m" (v->counter)
 161         : "r" (&v->counter)
 162         : "cc", "xer");
 163 }
 164 #define atomic_dec atomic_dec
 165
 166 static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
 167 {
 168         int t;
 169
 170         __asm__ __volatile__(
 171 "1:     lwarx   %0,0,%2         # atomic_dec_return_relaxed\n"
 172 "       addic   %0,%0,-1\n"
 173 "       stwcx.  %0,0,%2\n"
 174 "       bne-    1b"
 175         : "=&r" (t), "+m" (v->counter)
 176         : "r" (&v->counter)
 177         : "cc", "xer");
 178
 179         return t;
 180 }
 181
 182 #define atomic_inc_return_relaxed atomic_inc_return_relaxed
 183 #define atomic_dec_return_relaxed atomic_dec_return_relaxed
 184
 185 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 186 #define atomic_cmpxchg_relaxed(v, o, n) \
 187         cmpxchg_relaxed(&((v)->counter), (o), (n))
 188 #define atomic_cmpxchg_acquire(v, o, n) \
 189         cmpxchg_acquire(&((v)->counter), (o), (n))
 190
 191 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
 192 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
 193
 194 /*
 195  * Don't want to override the generic atomic_try_cmpxchg_acquire, because
 196  * we add a lock hint to the lwarx, which may not be wanted for the
 197  * _acquire case (and is not used by the other _acquire variants so it
 198  * would be a surprise).
 199  */
 200 static __always_inline bool
 201 atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new)
 202 {
 203         int r, o = *old;
 204
 205         __asm__ __volatile__ (
 206 "1:\t"  PPC_LWARX(%0,0,%2,1) "  # atomic_try_cmpxchg_acquire    \n"
 207 "       cmpw    0,%0,%3                                                 \n"
 208 "       bne-    2f                                                      \n"
 209 "       stwcx.  %4,0,%2                                                 \n"
 210 "       bne-    1b                                                      \n"
 211 "\t"    PPC_ACQUIRE_BARRIER "                                           \n"
 212 "2:                                                                     \n"
 213         : "=&r" (r), "+m" (v->counter)
 214         : "r" (&v->counter), "r" (o), "r" (new)
 215         : "cr0", "memory");
 216
 217         if (unlikely(r != o))
 218                 *old = r;
 219         return likely(r == o);
 220 }
 221
 222 /**
 223  * atomic_fetch_add_unless - add unless the number is a given value
 224  * @v: pointer of type atomic_t
 225  * @a: the amount to add to v...
 226  * @u: ...unless v is equal to u.
 227  *
 228  * Atomically adds @a to @v, so long as it was not @u.
 229  * Returns the old value of @v.
 230  */
 231 static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u)
 232 {
 233         int t;
 234
 235         __asm__ __volatile__ (
 236         PPC_ATOMIC_ENTRY_BARRIER
 237 "1:     lwarx   %0,0,%1         # atomic_fetch_add_unless\n\
 238         cmpw    0,%0,%3 \n\
 239         beq     2f \n\
 240         add     %0,%2,%0 \n"
 241 "       stwcx.  %0,0,%1 \n\
 242         bne-    1b \n"
 243         PPC_ATOMIC_EXIT_BARRIER
 244 "       subf    %0,%2,%0 \n\
 245 2:"
 246         : "=&r" (t)
 247         : "r" (&v->counter), "r" (a), "r" (u)
 248         : "cc", "memory");
 249
 250         return t;
 251 }
 252 #define atomic_fetch_add_unless atomic_fetch_add_unless
 253
 254 /**
 255  * atomic_inc_not_zero - increment unless the number is zero
 256  * @v: pointer of type atomic_t
 257  *
 258  * Atomically increments @v by 1, so long as @v is non-zero.
 259  * Returns non-zero if @v was non-zero, and zero otherwise.
 260  */
 261 static __inline__ int atomic_inc_not_zero(atomic_t *v)
 262 {
 263         int t1, t2;
 264
 265         __asm__ __volatile__ (
 266         PPC_ATOMIC_ENTRY_BARRIER
 267 "1:     lwarx   %0,0,%2         # atomic_inc_not_zero\n\
 268         cmpwi   0,%0,0\n\
 269         beq-    2f\n\
 270         addic   %1,%0,1\n"
 271 "       stwcx.  %1,0,%2\n\
 272         bne-    1b\n"
 273         PPC_ATOMIC_EXIT_BARRIER
 274         "\n\
 275 2:"
 276         : "=&r" (t1), "=&r" (t2)
 277         : "r" (&v->counter)
 278         : "cc", "xer", "memory");
 279
 280         return t1;
 281 }
 282 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v))
 283
 284 /*
 285  * Atomically test *v and decrement if it is greater than 0.
 286  * The function returns the old value of *v minus 1, even if
 287  * the atomic variable, v, was not decremented.
 288  */
 289 static __inline__ int atomic_dec_if_positive(atomic_t *v)
 290 {
 291         int t;
 292
 293         __asm__ __volatile__(
 294         PPC_ATOMIC_ENTRY_BARRIER
 295 "1:     lwarx   %0,0,%1         # atomic_dec_if_positive\n\
 296         cmpwi   %0,1\n\
 297         addi    %0,%0,-1\n\
 298         blt-    2f\n"
 299 "       stwcx.  %0,0,%1\n\
 300         bne-    1b"
 301         PPC_ATOMIC_EXIT_BARRIER
 302         "\n\
 303 2:"     : "=&b" (t)
 304         : "r" (&v->counter)
 305         : "cc", "memory");
 306
 307         return t;
 308 }
 309 #define atomic_dec_if_positive atomic_dec_if_positive
 310
 311 #ifdef __powerpc64__
 312
 313 #define ATOMIC64_INIT(i)        { (i) }
 314
 315 static __inline__ s64 atomic64_read(const atomic64_t *v)
 316 {
 317         s64 t;
 318
 319         __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
 320
 321         return t;
 322 }
 323
 324 static __inline__ void atomic64_set(atomic64_t *v, s64 i)
 325 {
 326         __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
 327 }
 328
 329 #define ATOMIC64_OP(op, asm_op)                                         \
 330 static __inline__ void atomic64_##op(s64 a, atomic64_t *v)              \
 331 {                                                                       \
 332         s64 t;                                                          \
 333                                                                         \
 334         __asm__ __volatile__(                                           \
 335 "1:     ldarx   %0,0,%3         # atomic64_" #op "\n"                   \
 336         #asm_op " %0,%2,%0\n"                                           \
 337 "       stdcx.  %0,0,%3 \n"                                             \
 338 "       bne-    1b\n"                                                   \
 339         : "=&r" (t), "+m" (v->counter)                                  \
 340         : "r" (a), "r" (&v->counter)                                    \
 341         : "cc");                                                        \
 342 }
 343
 344 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 345 static inline s64                                                       \
 346 atomic64_##op##_return_relaxed(s64 a, atomic64_t *v)                    \
 347 {                                                                       \
 348         s64 t;                                                          \
 349                                                                         \
 350         __asm__ __volatile__(                                           \
 351 "1:     ldarx   %0,0,%3         # atomic64_" #op "_return_relaxed\n"    \
 352         #asm_op " %0,%2,%0\n"                                           \
 353 "       stdcx.  %0,0,%3\n"                                              \
 354 "       bne-    1b\n"                                                   \
 355         : "=&r" (t), "+m" (v->counter)                                  \
 356         : "r" (a), "r" (&v->counter)                                    \
 357         : "cc");                                                        \
 358                                                                         \
 359         return t;                                                       \
 360 }
 361
 362 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op)                           \
 363 static inline s64                                                       \
 364 atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v)                     \
 365 {                                                                       \
 366         s64 res, t;                                                     \
 367                                                                         \
 368         __asm__ __volatile__(                                           \
 369 "1:     ldarx   %0,0,%4         # atomic64_fetch_" #op "_relaxed\n"     \
 370         #asm_op " %1,%3,%0\n"                                           \
 371 "       stdcx.  %1,0,%4\n"                                              \
 372 "       bne-    1b\n"                                                   \
 373         : "=&r" (res), "=&r" (t), "+m" (v->counter)                     \
 374         : "r" (a), "r" (&v->counter)                                    \
 375         : "cc");                                                        \
 376                                                                         \
 377         return res;                                                     \
 378 }
 379
 380 #define ATOMIC64_OPS(op, asm_op)                                        \
 381         ATOMIC64_OP(op, asm_op)                                         \
 382         ATOMIC64_OP_RETURN_RELAXED(op, asm_op)                          \
 383         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 384
 385 ATOMIC64_OPS(add, add)
 386 ATOMIC64_OPS(sub, subf)
 387
 388 #define atomic64_add_return_relaxed atomic64_add_return_relaxed
 389 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
 390
 391 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
 392 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
 393
 394 #undef ATOMIC64_OPS
 395 #define ATOMIC64_OPS(op, asm_op)                                        \
 396         ATOMIC64_OP(op, asm_op)                                         \
 397         ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
 398
 399 ATOMIC64_OPS(and, and)
 400 ATOMIC64_OPS(or, or)
 401 ATOMIC64_OPS(xor, xor)
 402
 403 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
 404 #define atomic64_fetch_or_relaxed  atomic64_fetch_or_relaxed
 405 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
 406
 407 #undef ATOPIC64_OPS
 408 #undef ATOMIC64_FETCH_OP_RELAXED
 409 #undef ATOMIC64_OP_RETURN_RELAXED
 410 #undef ATOMIC64_OP
 411
 412 static __inline__ void atomic64_inc(atomic64_t *v)
 413 {
 414         s64 t;
 415
 416         __asm__ __volatile__(
 417 "1:     ldarx   %0,0,%2         # atomic64_inc\n\
 418         addic   %0,%0,1\n\
 419         stdcx.  %0,0,%2 \n\
 420         bne-    1b"
 421         : "=&r" (t), "+m" (v->counter)
 422         : "r" (&v->counter)
 423         : "cc", "xer");
 424 }
 425 #define atomic64_inc atomic64_inc
 426
 427 static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v)
 428 {
 429         s64 t;
 430
 431         __asm__ __volatile__(
 432 "1:     ldarx   %0,0,%2         # atomic64_inc_return_relaxed\n"
 433 "       addic   %0,%0,1\n"
 434 "       stdcx.  %0,0,%2\n"
 435 "       bne-    1b"
 436         : "=&r" (t), "+m" (v->counter)
 437         : "r" (&v->counter)
 438         : "cc", "xer");
 439
 440         return t;
 441 }
 442
 443 static __inline__ void atomic64_dec(atomic64_t *v)
 444 {
 445         s64 t;
 446
 447         __asm__ __volatile__(
 448 "1:     ldarx   %0,0,%2         # atomic64_dec\n\
 449         addic   %0,%0,-1\n\
 450         stdcx.  %0,0,%2\n\
 451         bne-    1b"
 452         : "=&r" (t), "+m" (v->counter)
 453         : "r" (&v->counter)
 454         : "cc", "xer");
 455 }
 456 #define atomic64_dec atomic64_dec
 457
 458 static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v)
 459 {
 460         s64 t;
 461
 462         __asm__ __volatile__(
 463 "1:     ldarx   %0,0,%2         # atomic64_dec_return_relaxed\n"
 464 "       addic   %0,%0,-1\n"
 465 "       stdcx.  %0,0,%2\n"
 466 "       bne-    1b"
 467         : "=&r" (t), "+m" (v->counter)
 468         : "r" (&v->counter)
 469         : "cc", "xer");
 470
 471         return t;
 472 }
 473
 474 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
 475 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
 476
 477 /*
 478  * Atomically test *v and decrement if it is greater than 0.
 479  * The function returns the old value of *v minus 1.
 480  */
 481 static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v)
 482 {
 483         s64 t;
 484
 485         __asm__ __volatile__(
 486         PPC_ATOMIC_ENTRY_BARRIER
 487 "1:     ldarx   %0,0,%1         # atomic64_dec_if_positive\n\
 488         addic.  %0,%0,-1\n\
 489         blt-    2f\n\
 490         stdcx.  %0,0,%1\n\
 491         bne-    1b"
 492         PPC_ATOMIC_EXIT_BARRIER
 493         "\n\
 494 2:"     : "=&r" (t)
 495         : "r" (&v->counter)
 496         : "cc", "xer", "memory");
 497
 498         return t;
 499 }
 500 #define atomic64_dec_if_positive atomic64_dec_if_positive
 501
 502 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 503 #define atomic64_cmpxchg_relaxed(v, o, n) \
 504         cmpxchg_relaxed(&((v)->counter), (o), (n))
 505 #define atomic64_cmpxchg_acquire(v, o, n) \
 506         cmpxchg_acquire(&((v)->counter), (o), (n))
 507
 508 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 509 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
 510
 511 /**
 512  * atomic64_fetch_add_unless - add unless the number is a given value
 513  * @v: pointer of type atomic64_t
 514  * @a: the amount to add to v...
 515  * @u: ...unless v is equal to u.
 516  *
 517  * Atomically adds @a to @v, so long as it was not @u.
 518  * Returns the old value of @v.
 519  */
 520 static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
 521 {
 522         s64 t;
 523
 524         __asm__ __volatile__ (
 525         PPC_ATOMIC_ENTRY_BARRIER
 526 "1:     ldarx   %0,0,%1         # atomic64_fetch_add_unless\n\
 527         cmpd    0,%0,%3 \n\
 528         beq     2f \n\
 529         add     %0,%2,%0 \n"
 530 "       stdcx.  %0,0,%1 \n\
 531         bne-    1b \n"
 532         PPC_ATOMIC_EXIT_BARRIER
 533 "       subf    %0,%2,%0 \n\
 534 2:"
 535         : "=&r" (t)
 536         : "r" (&v->counter), "r" (a), "r" (u)
 537         : "cc", "memory");
 538
 539         return t;
 540 }
 541 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
 542
 543 /**
 544  * atomic_inc64_not_zero - increment unless the number is zero
 545  * @v: pointer of type atomic64_t
 546  *
 547  * Atomically increments @v by 1, so long as @v is non-zero.
 548  * Returns non-zero if @v was non-zero, and zero otherwise.
 549  */
 550 static __inline__ int atomic64_inc_not_zero(atomic64_t *v)
 551 {
 552         s64 t1, t2;
 553
 554         __asm__ __volatile__ (
 555         PPC_ATOMIC_ENTRY_BARRIER
 556 "1:     ldarx   %0,0,%2         # atomic64_inc_not_zero\n\
 557         cmpdi   0,%0,0\n\
 558         beq-    2f\n\
 559         addic   %1,%0,1\n\
 560         stdcx.  %1,0,%2\n\
 561         bne-    1b\n"
 562         PPC_ATOMIC_EXIT_BARRIER
 563         "\n\
 564 2:"
 565         : "=&r" (t1), "=&r" (t2)
 566         : "r" (&v->counter)
 567         : "cc", "xer", "memory");
 568
 569         return t1 != 0;
 570 }
 571 #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v))
 572
 573 #endif /* __powerpc64__ */
 574
 575 #endif /* __KERNEL__ */
 576 #endif /* _ASM_POWERPC_ATOMIC_H_ */