include/linux/atomic.h

   1 /* Atomic operations usable in machine independent code */
   2 #ifndef _LINUX_ATOMIC_H
   3 #define _LINUX_ATOMIC_H
   4 #include <asm/atomic.h>
   5 #include <asm/barrier.h>
   6
   7 /*
   8  * Relaxed variants of xchg, cmpxchg and some atomic operations.
   9  *
  10  * We support four variants:
  11  *
  12  * - Fully ordered: The default implementation, no suffix required.
  13  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  14  * - Release: Provides RELEASE semantics, _release suffix.
  15  * - Relaxed: No ordering guarantees, _relaxed suffix.
  16  *
  17  * For compound atomics performing both a load and a store, ACQUIRE
  18  * semantics apply only to the load and RELEASE semantics only to the
  19  * store portion of the operation. Note that a failed cmpxchg_acquire
  20  * does -not- imply any memory ordering constraints.
  21  *
  22  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  23  */
  24
  25 #ifndef atomic_read_acquire
  26 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  27 #endif
  28
  29 #ifndef atomic_set_release
  30 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  31 #endif
  32
  33 /*
  34  * The idea here is to build acquire/release variants by adding explicit
  35  * barriers on top of the relaxed variant. In the case where the relaxed
  36  * variant is already fully ordered, no additional barriers are needed.
  37  *
  38  * Besides, if an arch has a special barrier for acquire/release, it could
  39  * implement its own __atomic_op_* and use the same framework for building
  40  * variants
  41  */
  42 #ifndef __atomic_op_acquire
  43 #define __atomic_op_acquire(op, args...)                                \
  44 ({                                                                      \
  45         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  46         smp_mb__after_atomic();                                         \
  47         __ret;                                                          \
  48 })
  49 #endif
  50
  51 #ifndef __atomic_op_release
  52 #define __atomic_op_release(op, args...)                                \
  53 ({                                                                      \
  54         smp_mb__before_atomic();                                        \
  55         op##_relaxed(args);                                             \
  56 })
  57 #endif
  58
  59 #ifndef __atomic_op_fence
  60 #define __atomic_op_fence(op, args...)                                  \
  61 ({                                                                      \
  62         typeof(op##_relaxed(args)) __ret;                               \
  63         smp_mb__before_atomic();                                        \
  64         __ret = op##_relaxed(args);                                     \
  65         smp_mb__after_atomic();                                         \
  66         __ret;                                                          \
  67 })
  68 #endif
  69
  70 /* atomic_add_return_relaxed */
  71 #ifndef atomic_add_return_relaxed
  72 #define  atomic_add_return_relaxed      atomic_add_return
  73 #define  atomic_add_return_acquire      atomic_add_return
  74 #define  atomic_add_return_release      atomic_add_return
  75
  76 #else /* atomic_add_return_relaxed */
  77
  78 #ifndef atomic_add_return_acquire
  79 #define  atomic_add_return_acquire(...)                                 \
  80         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  81 #endif
  82
  83 #ifndef atomic_add_return_release
  84 #define  atomic_add_return_release(...)                                 \
  85         __atomic_op_release(atomic_add_return, __VA_ARGS__)
  86 #endif
  87
  88 #ifndef atomic_add_return
  89 #define  atomic_add_return(...)                                         \
  90         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  91 #endif
  92 #endif /* atomic_add_return_relaxed */
  93
  94 /* atomic_inc_return_relaxed */
  95 #ifndef atomic_inc_return_relaxed
  96 #define  atomic_inc_return_relaxed      atomic_inc_return
  97 #define  atomic_inc_return_acquire      atomic_inc_return
  98 #define  atomic_inc_return_release      atomic_inc_return
  99
 100 #else /* atomic_inc_return_relaxed */
 101
 102 #ifndef atomic_inc_return_acquire
 103 #define  atomic_inc_return_acquire(...)                                 \
 104         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
 105 #endif
 106
 107 #ifndef atomic_inc_return_release
 108 #define  atomic_inc_return_release(...)                                 \
 109         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 110 #endif
 111
 112 #ifndef atomic_inc_return
 113 #define  atomic_inc_return(...)                                         \
 114         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 115 #endif
 116 #endif /* atomic_inc_return_relaxed */
 117
 118 /* atomic_sub_return_relaxed */
 119 #ifndef atomic_sub_return_relaxed
 120 #define  atomic_sub_return_relaxed      atomic_sub_return
 121 #define  atomic_sub_return_acquire      atomic_sub_return
 122 #define  atomic_sub_return_release      atomic_sub_return
 123
 124 #else /* atomic_sub_return_relaxed */
 125
 126 #ifndef atomic_sub_return_acquire
 127 #define  atomic_sub_return_acquire(...)                                 \
 128         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 129 #endif
 130
 131 #ifndef atomic_sub_return_release
 132 #define  atomic_sub_return_release(...)                                 \
 133         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 134 #endif
 135
 136 #ifndef atomic_sub_return
 137 #define  atomic_sub_return(...)                                         \
 138         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 139 #endif
 140 #endif /* atomic_sub_return_relaxed */
 141
 142 /* atomic_dec_return_relaxed */
 143 #ifndef atomic_dec_return_relaxed
 144 #define  atomic_dec_return_relaxed      atomic_dec_return
 145 #define  atomic_dec_return_acquire      atomic_dec_return
 146 #define  atomic_dec_return_release      atomic_dec_return
 147
 148 #else /* atomic_dec_return_relaxed */
 149
 150 #ifndef atomic_dec_return_acquire
 151 #define  atomic_dec_return_acquire(...)                                 \
 152         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 153 #endif
 154
 155 #ifndef atomic_dec_return_release
 156 #define  atomic_dec_return_release(...)                                 \
 157         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 158 #endif
 159
 160 #ifndef atomic_dec_return
 161 #define  atomic_dec_return(...)                                         \
 162         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 163 #endif
 164 #endif /* atomic_dec_return_relaxed */
 165
 166 /* atomic_xchg_relaxed */
 167 #ifndef atomic_xchg_relaxed
 168 #define  atomic_xchg_relaxed            atomic_xchg
 169 #define  atomic_xchg_acquire            atomic_xchg
 170 #define  atomic_xchg_release            atomic_xchg
 171
 172 #else /* atomic_xchg_relaxed */
 173
 174 #ifndef atomic_xchg_acquire
 175 #define  atomic_xchg_acquire(...)                                       \
 176         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 177 #endif
 178
 179 #ifndef atomic_xchg_release
 180 #define  atomic_xchg_release(...)                                       \
 181         __atomic_op_release(atomic_xchg, __VA_ARGS__)
 182 #endif
 183
 184 #ifndef atomic_xchg
 185 #define  atomic_xchg(...)                                               \
 186         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 187 #endif
 188 #endif /* atomic_xchg_relaxed */
 189
 190 /* atomic_cmpxchg_relaxed */
 191 #ifndef atomic_cmpxchg_relaxed
 192 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 193 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
 194 #define  atomic_cmpxchg_release         atomic_cmpxchg
 195
 196 #else /* atomic_cmpxchg_relaxed */
 197
 198 #ifndef atomic_cmpxchg_acquire
 199 #define  atomic_cmpxchg_acquire(...)                                    \
 200         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 201 #endif
 202
 203 #ifndef atomic_cmpxchg_release
 204 #define  atomic_cmpxchg_release(...)                                    \
 205         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 206 #endif
 207
 208 #ifndef atomic_cmpxchg
 209 #define  atomic_cmpxchg(...)                                            \
 210         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 211 #endif
 212 #endif /* atomic_cmpxchg_relaxed */
 213
 214 #ifndef atomic64_read_acquire
 215 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 216 #endif
 217
 218 #ifndef atomic64_set_release
 219 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 220 #endif
 221
 222 /* atomic64_add_return_relaxed */
 223 #ifndef atomic64_add_return_relaxed
 224 #define  atomic64_add_return_relaxed    atomic64_add_return
 225 #define  atomic64_add_return_acquire    atomic64_add_return
 226 #define  atomic64_add_return_release    atomic64_add_return
 227
 228 #else /* atomic64_add_return_relaxed */
 229
 230 #ifndef atomic64_add_return_acquire
 231 #define  atomic64_add_return_acquire(...)                               \
 232         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 233 #endif
 234
 235 #ifndef atomic64_add_return_release
 236 #define  atomic64_add_return_release(...)                               \
 237         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 238 #endif
 239
 240 #ifndef atomic64_add_return
 241 #define  atomic64_add_return(...)                                       \
 242         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 243 #endif
 244 #endif /* atomic64_add_return_relaxed */
 245
 246 /* atomic64_inc_return_relaxed */
 247 #ifndef atomic64_inc_return_relaxed
 248 #define  atomic64_inc_return_relaxed    atomic64_inc_return
 249 #define  atomic64_inc_return_acquire    atomic64_inc_return
 250 #define  atomic64_inc_return_release    atomic64_inc_return
 251
 252 #else /* atomic64_inc_return_relaxed */
 253
 254 #ifndef atomic64_inc_return_acquire
 255 #define  atomic64_inc_return_acquire(...)                               \
 256         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 257 #endif
 258
 259 #ifndef atomic64_inc_return_release
 260 #define  atomic64_inc_return_release(...)                               \
 261         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 262 #endif
 263
 264 #ifndef atomic64_inc_return
 265 #define  atomic64_inc_return(...)                                       \
 266         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 267 #endif
 268 #endif /* atomic64_inc_return_relaxed */
 269
 270
 271 /* atomic64_sub_return_relaxed */
 272 #ifndef atomic64_sub_return_relaxed
 273 #define  atomic64_sub_return_relaxed    atomic64_sub_return
 274 #define  atomic64_sub_return_acquire    atomic64_sub_return
 275 #define  atomic64_sub_return_release    atomic64_sub_return
 276
 277 #else /* atomic64_sub_return_relaxed */
 278
 279 #ifndef atomic64_sub_return_acquire
 280 #define  atomic64_sub_return_acquire(...)                               \
 281         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 282 #endif
 283
 284 #ifndef atomic64_sub_return_release
 285 #define  atomic64_sub_return_release(...)                               \
 286         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 287 #endif
 288
 289 #ifndef atomic64_sub_return
 290 #define  atomic64_sub_return(...)                                       \
 291         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 292 #endif
 293 #endif /* atomic64_sub_return_relaxed */
 294
 295 /* atomic64_dec_return_relaxed */
 296 #ifndef atomic64_dec_return_relaxed
 297 #define  atomic64_dec_return_relaxed    atomic64_dec_return
 298 #define  atomic64_dec_return_acquire    atomic64_dec_return
 299 #define  atomic64_dec_return_release    atomic64_dec_return
 300
 301 #else /* atomic64_dec_return_relaxed */
 302
 303 #ifndef atomic64_dec_return_acquire
 304 #define  atomic64_dec_return_acquire(...)                               \
 305         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 306 #endif
 307
 308 #ifndef atomic64_dec_return_release
 309 #define  atomic64_dec_return_release(...)                               \
 310         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 311 #endif
 312
 313 #ifndef atomic64_dec_return
 314 #define  atomic64_dec_return(...)                                       \
 315         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 316 #endif
 317 #endif /* atomic64_dec_return_relaxed */
 318
 319 /* atomic64_xchg_relaxed */
 320 #ifndef atomic64_xchg_relaxed
 321 #define  atomic64_xchg_relaxed          atomic64_xchg
 322 #define  atomic64_xchg_acquire          atomic64_xchg
 323 #define  atomic64_xchg_release          atomic64_xchg
 324
 325 #else /* atomic64_xchg_relaxed */
 326
 327 #ifndef atomic64_xchg_acquire
 328 #define  atomic64_xchg_acquire(...)                                     \
 329         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 330 #endif
 331
 332 #ifndef atomic64_xchg_release
 333 #define  atomic64_xchg_release(...)                                     \
 334         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 335 #endif
 336
 337 #ifndef atomic64_xchg
 338 #define  atomic64_xchg(...)                                             \
 339         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
 340 #endif
 341 #endif /* atomic64_xchg_relaxed */
 342
 343 /* atomic64_cmpxchg_relaxed */
 344 #ifndef atomic64_cmpxchg_relaxed
 345 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
 346 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
 347 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
 348
 349 #else /* atomic64_cmpxchg_relaxed */
 350
 351 #ifndef atomic64_cmpxchg_acquire
 352 #define  atomic64_cmpxchg_acquire(...)                                  \
 353         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
 354 #endif
 355
 356 #ifndef atomic64_cmpxchg_release
 357 #define  atomic64_cmpxchg_release(...)                                  \
 358         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
 359 #endif
 360
 361 #ifndef atomic64_cmpxchg
 362 #define  atomic64_cmpxchg(...)                                          \
 363         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
 364 #endif
 365 #endif /* atomic64_cmpxchg_relaxed */
 366
 367 /* cmpxchg_relaxed */
 368 #ifndef cmpxchg_relaxed
 369 #define  cmpxchg_relaxed                cmpxchg
 370 #define  cmpxchg_acquire                cmpxchg
 371 #define  cmpxchg_release                cmpxchg
 372
 373 #else /* cmpxchg_relaxed */
 374
 375 #ifndef cmpxchg_acquire
 376 #define  cmpxchg_acquire(...)                                           \
 377         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 378 #endif
 379
 380 #ifndef cmpxchg_release
 381 #define  cmpxchg_release(...)                                           \
 382         __atomic_op_release(cmpxchg, __VA_ARGS__)
 383 #endif
 384
 385 #ifndef cmpxchg
 386 #define  cmpxchg(...)                                                   \
 387         __atomic_op_fence(cmpxchg, __VA_ARGS__)
 388 #endif
 389 #endif /* cmpxchg_relaxed */
 390
 391 /* cmpxchg64_relaxed */
 392 #ifndef cmpxchg64_relaxed
 393 #define  cmpxchg64_relaxed              cmpxchg64
 394 #define  cmpxchg64_acquire              cmpxchg64
 395 #define  cmpxchg64_release              cmpxchg64
 396
 397 #else /* cmpxchg64_relaxed */
 398
 399 #ifndef cmpxchg64_acquire
 400 #define  cmpxchg64_acquire(...)                                         \
 401         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 402 #endif
 403
 404 #ifndef cmpxchg64_release
 405 #define  cmpxchg64_release(...)                                         \
 406         __atomic_op_release(cmpxchg64, __VA_ARGS__)
 407 #endif
 408
 409 #ifndef cmpxchg64
 410 #define  cmpxchg64(...)                                                 \
 411         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 412 #endif
 413 #endif /* cmpxchg64_relaxed */
 414
 415 /* xchg_relaxed */
 416 #ifndef xchg_relaxed
 417 #define  xchg_relaxed                   xchg
 418 #define  xchg_acquire                   xchg
 419 #define  xchg_release                   xchg
 420
 421 #else /* xchg_relaxed */
 422
 423 #ifndef xchg_acquire
 424 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 425 #endif
 426
 427 #ifndef xchg_release
 428 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 429 #endif
 430
 431 #ifndef xchg
 432 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 433 #endif
 434 #endif /* xchg_relaxed */
 435
 436 /**
 437  * atomic_add_unless - add unless the number is already a given value
 438  * @v: pointer of type atomic_t
 439  * @a: the amount to add to v...
 440  * @u: ...unless v is equal to u.
 441  *
 442  * Atomically adds @a to @v, so long as @v was not already @u.
 443  * Returns non-zero if @v was not @u, and zero otherwise.
 444  */
 445 static inline int atomic_add_unless(atomic_t *v, int a, int u)
 446 {
 447         return __atomic_add_unless(v, a, u) != u;
 448 }
 449
 450 /**
 451  * atomic_inc_not_zero - increment unless the number is zero
 452  * @v: pointer of type atomic_t
 453  *
 454  * Atomically increments @v by 1, so long as @v is non-zero.
 455  * Returns non-zero if @v was non-zero, and zero otherwise.
 456  */
 457 #ifndef atomic_inc_not_zero
 458 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 459 #endif
 460
 461 #ifndef atomic_andnot
 462 static inline void atomic_andnot(int i, atomic_t *v)
 463 {
 464         atomic_and(~i, v);
 465 }
 466 #endif
 467
 468 static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
 469 {
 470         atomic_andnot(mask, v);
 471 }
 472
 473 static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
 474 {
 475         atomic_or(mask, v);
 476 }
 477
 478 /**
 479  * atomic_inc_not_zero_hint - increment if not null
 480  * @v: pointer of type atomic_t
 481  * @hint: probable value of the atomic before the increment
 482  *
 483  * This version of atomic_inc_not_zero() gives a hint of probable
 484  * value of the atomic. This helps processor to not read the memory
 485  * before doing the atomic read/modify/write cycle, lowering
 486  * number of bus transactions on some arches.
 487  *
 488  * Returns: 0 if increment was not done, 1 otherwise.
 489  */
 490 #ifndef atomic_inc_not_zero_hint
 491 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 492 {
 493         int val, c = hint;
 494
 495         /* sanity test, should be removed by compiler if hint is a constant */
 496         if (!hint)
 497                 return atomic_inc_not_zero(v);
 498
 499         do {
 500                 val = atomic_cmpxchg(v, c, c + 1);
 501                 if (val == c)
 502                         return 1;
 503                 c = val;
 504         } while (c);
 505
 506         return 0;
 507 }
 508 #endif
 509
 510 #ifndef atomic_inc_unless_negative
 511 static inline int atomic_inc_unless_negative(atomic_t *p)
 512 {
 513         int v, v1;
 514         for (v = 0; v >= 0; v = v1) {
 515                 v1 = atomic_cmpxchg(p, v, v + 1);
 516                 if (likely(v1 == v))
 517                         return 1;
 518         }
 519         return 0;
 520 }
 521 #endif
 522
 523 #ifndef atomic_dec_unless_positive
 524 static inline int atomic_dec_unless_positive(atomic_t *p)
 525 {
 526         int v, v1;
 527         for (v = 0; v <= 0; v = v1) {
 528                 v1 = atomic_cmpxchg(p, v, v - 1);
 529                 if (likely(v1 == v))
 530                         return 1;
 531         }
 532         return 0;
 533 }
 534 #endif
 535
 536 /*
 537  * atomic_dec_if_positive - decrement by 1 if old value positive
 538  * @v: pointer of type atomic_t
 539  *
 540  * The function returns the old value of *v minus 1, even if
 541  * the atomic variable, v, was not decremented.
 542  */
 543 #ifndef atomic_dec_if_positive
 544 static inline int atomic_dec_if_positive(atomic_t *v)
 545 {
 546         int c, old, dec;
 547         c = atomic_read(v);
 548         for (;;) {
 549                 dec = c - 1;
 550                 if (unlikely(dec < 0))
 551                         break;
 552                 old = atomic_cmpxchg((v), c, dec);
 553                 if (likely(old == c))
 554                         break;
 555                 c = old;
 556         }
 557         return dec;
 558 }
 559 #endif
 560
 561 /**
 562  * atomic_fetch_or - perform *p |= mask and return old value of *p
 563  * @p: pointer to atomic_t
 564  * @mask: mask to OR on the atomic_t
 565  */
 566 #ifndef atomic_fetch_or
 567 static inline int atomic_fetch_or(atomic_t *p, int mask)
 568 {
 569         int old, val = atomic_read(p);
 570
 571         for (;;) {
 572                 old = atomic_cmpxchg(p, val, val | mask);
 573                 if (old == val)
 574                         break;
 575                 val = old;
 576         }
 577
 578         return old;
 579 }
 580 #endif
 581
 582 /**
 583  * fetch_or - perform *ptr |= mask and return old value of *ptr
 584  * @ptr: pointer to value
 585  * @mask: mask to OR on the value
 586  *
 587  * cmpxchg based fetch_or, macro so it works for different integer types
 588  */
 589 #ifndef fetch_or
 590 #define fetch_or(ptr, mask)                                             \
 591 ({      typeof(*(ptr)) __old, __val = *(ptr);                           \
 592         for (;;) {                                                      \
 593                 __old = cmpxchg((ptr), __val, __val | (mask));          \
 594                 if (__old == __val)                                     \
 595                         break;                                          \
 596                 __val = __old;                                          \
 597         }                                                               \
 598         __old;                                                          \
 599 })
 600 #endif
 601
 602
 603 #ifdef CONFIG_GENERIC_ATOMIC64
 604 #include <asm-generic/atomic64.h>
 605 #endif
 606
 607 #ifndef atomic64_andnot
 608 static inline void atomic64_andnot(long long i, atomic64_t *v)
 609 {
 610         atomic64_and(~i, v);
 611 }
 612 #endif
 613
 614 #include <asm-generic/atomic-long.h>
 615
 616 #endif /* _LINUX_ATOMIC_H */