arch/loongarch/include/asm/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Atomic operations.
   4  *
   5  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   6  */
   7 #ifndef _ASM_ATOMIC_H
   8 #define _ASM_ATOMIC_H
   9
  10 #include <linux/types.h>
  11 #include <asm/barrier.h>
  12 #include <asm/cmpxchg.h>
  13
  14 #if __SIZEOF_LONG__ == 4
  15 #define __LL            "ll.w   "
  16 #define __SC            "sc.w   "
  17 #define __AMADD         "amadd.w        "
  18 #define __AMOR          "amor.w         "
  19 #define __AMAND_DB      "amand_db.w     "
  20 #define __AMOR_DB       "amor_db.w      "
  21 #define __AMXOR_DB      "amxor_db.w     "
  22 #elif __SIZEOF_LONG__ == 8
  23 #define __LL            "ll.d   "
  24 #define __SC            "sc.d   "
  25 #define __AMADD         "amadd.d        "
  26 #define __AMOR          "amor.d         "
  27 #define __AMAND_DB      "amand_db.d     "
  28 #define __AMOR_DB       "amor_db.d      "
  29 #define __AMXOR_DB      "amxor_db.d     "
  30 #endif
  31
  32 #define ATOMIC_INIT(i)    { (i) }
  33
  34 #define arch_atomic_read(v)     READ_ONCE((v)->counter)
  35 #define arch_atomic_set(v, i)   WRITE_ONCE((v)->counter, (i))
  36
  37 #define ATOMIC_OP(op, I, asm_op)                                        \
  38 static inline void arch_atomic_##op(int i, atomic_t *v)                 \
  39 {                                                                       \
  40         __asm__ __volatile__(                                           \
  41         "am"#asm_op".w" " $zero, %1, %0 \n"                             \
  42         : "+ZB" (v->counter)                                            \
  43         : "r" (I)                                                       \
  44         : "memory");                                                    \
  45 }
  46
  47 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix)               \
  48 static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
  49 {                                                                       \
  50         int result;                                                     \
  51                                                                         \
  52         __asm__ __volatile__(                                           \
  53         "am"#asm_op#mb".w" " %1, %2, %0         \n"                     \
  54         : "+ZB" (v->counter), "=&r" (result)                            \
  55         : "r" (I)                                                       \
  56         : "memory");                                                    \
  57                                                                         \
  58         return result c_op I;                                           \
  59 }
  60
  61 #define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix)                      \
  62 static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v)    \
  63 {                                                                       \
  64         int result;                                                     \
  65                                                                         \
  66         __asm__ __volatile__(                                           \
  67         "am"#asm_op#mb".w" " %1, %2, %0         \n"                     \
  68         : "+ZB" (v->counter), "=&r" (result)                            \
  69         : "r" (I)                                                       \
  70         : "memory");                                                    \
  71                                                                         \
  72         return result;                                                  \
  73 }
  74
  75 #define ATOMIC_OPS(op, I, asm_op, c_op)                                 \
  76         ATOMIC_OP(op, I, asm_op)                                        \
  77         ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db,         )            \
  78         ATOMIC_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)            \
  79         ATOMIC_FETCH_OP(op, I, asm_op, _db,         )                   \
  80         ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)
  81
  82 ATOMIC_OPS(add, i, add, +)
  83 ATOMIC_OPS(sub, -i, add, +)
  84
  85 #define arch_atomic_add_return          arch_atomic_add_return
  86 #define arch_atomic_add_return_acquire  arch_atomic_add_return
  87 #define arch_atomic_add_return_release  arch_atomic_add_return
  88 #define arch_atomic_add_return_relaxed  arch_atomic_add_return_relaxed
  89 #define arch_atomic_sub_return          arch_atomic_sub_return
  90 #define arch_atomic_sub_return_acquire  arch_atomic_sub_return
  91 #define arch_atomic_sub_return_release  arch_atomic_sub_return
  92 #define arch_atomic_sub_return_relaxed  arch_atomic_sub_return_relaxed
  93 #define arch_atomic_fetch_add           arch_atomic_fetch_add
  94 #define arch_atomic_fetch_add_acquire   arch_atomic_fetch_add
  95 #define arch_atomic_fetch_add_release   arch_atomic_fetch_add
  96 #define arch_atomic_fetch_add_relaxed   arch_atomic_fetch_add_relaxed
  97 #define arch_atomic_fetch_sub           arch_atomic_fetch_sub
  98 #define arch_atomic_fetch_sub_acquire   arch_atomic_fetch_sub
  99 #define arch_atomic_fetch_sub_release   arch_atomic_fetch_sub
 100 #define arch_atomic_fetch_sub_relaxed   arch_atomic_fetch_sub_relaxed
 101
 102 #undef ATOMIC_OPS
 103
 104 #define ATOMIC_OPS(op, I, asm_op)                                       \
 105         ATOMIC_OP(op, I, asm_op)                                        \
 106         ATOMIC_FETCH_OP(op, I, asm_op, _db,         )                   \
 107         ATOMIC_FETCH_OP(op, I, asm_op,    , _relaxed)
 108
 109 ATOMIC_OPS(and, i, and)
 110 ATOMIC_OPS(or, i, or)
 111 ATOMIC_OPS(xor, i, xor)
 112
 113 #define arch_atomic_fetch_and           arch_atomic_fetch_and
 114 #define arch_atomic_fetch_and_acquire   arch_atomic_fetch_and
 115 #define arch_atomic_fetch_and_release   arch_atomic_fetch_and
 116 #define arch_atomic_fetch_and_relaxed   arch_atomic_fetch_and_relaxed
 117 #define arch_atomic_fetch_or            arch_atomic_fetch_or
 118 #define arch_atomic_fetch_or_acquire    arch_atomic_fetch_or
 119 #define arch_atomic_fetch_or_release    arch_atomic_fetch_or
 120 #define arch_atomic_fetch_or_relaxed    arch_atomic_fetch_or_relaxed
 121 #define arch_atomic_fetch_xor           arch_atomic_fetch_xor
 122 #define arch_atomic_fetch_xor_acquire   arch_atomic_fetch_xor
 123 #define arch_atomic_fetch_xor_release   arch_atomic_fetch_xor
 124 #define arch_atomic_fetch_xor_relaxed   arch_atomic_fetch_xor_relaxed
 125
 126 #undef ATOMIC_OPS
 127 #undef ATOMIC_FETCH_OP
 128 #undef ATOMIC_OP_RETURN
 129 #undef ATOMIC_OP
 130
 131 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
 132 {
 133        int prev, rc;
 134
 135         __asm__ __volatile__ (
 136                 "0:     ll.w    %[p],  %[c]\n"
 137                 "       beq     %[p],  %[u], 1f\n"
 138                 "       add.w   %[rc], %[p], %[a]\n"
 139                 "       sc.w    %[rc], %[c]\n"
 140                 "       beqz    %[rc], 0b\n"
 141                 "       b       2f\n"
 142                 "1:\n"
 143                 __WEAK_LLSC_MB
 144                 "2:\n"
 145                 : [p]"=&r" (prev), [rc]"=&r" (rc),
 146                   [c]"=ZB" (v->counter)
 147                 : [a]"r" (a), [u]"r" (u)
 148                 : "memory");
 149
 150         return prev;
 151 }
 152 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
 153
 154 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
 155 {
 156         int result;
 157         int temp;
 158
 159         if (__builtin_constant_p(i)) {
 160                 __asm__ __volatile__(
 161                 "1:     ll.w    %1, %2          # atomic_sub_if_positive\n"
 162                 "       addi.w  %0, %1, %3                              \n"
 163                 "       move    %1, %0                                  \n"
 164                 "       bltz    %0, 2f                                  \n"
 165                 "       sc.w    %1, %2                                  \n"
 166                 "       beqz    %1, 1b                                  \n"
 167                 "2:                                                     \n"
 168                 __WEAK_LLSC_MB
 169                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 170                 : "I" (-i));
 171         } else {
 172                 __asm__ __volatile__(
 173                 "1:     ll.w    %1, %2          # atomic_sub_if_positive\n"
 174                 "       sub.w   %0, %1, %3                              \n"
 175                 "       move    %1, %0                                  \n"
 176                 "       bltz    %0, 2f                                  \n"
 177                 "       sc.w    %1, %2                                  \n"
 178                 "       beqz    %1, 1b                                  \n"
 179                 "2:                                                     \n"
 180                 __WEAK_LLSC_MB
 181                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 182                 : "r" (i));
 183         }
 184
 185         return result;
 186 }
 187
 188 #define arch_atomic_dec_if_positive(v)  arch_atomic_sub_if_positive(1, v)
 189
 190 #ifdef CONFIG_64BIT
 191
 192 #define ATOMIC64_INIT(i)    { (i) }
 193
 194 #define arch_atomic64_read(v)   READ_ONCE((v)->counter)
 195 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
 196
 197 #define ATOMIC64_OP(op, I, asm_op)                                      \
 198 static inline void arch_atomic64_##op(long i, atomic64_t *v)            \
 199 {                                                                       \
 200         __asm__ __volatile__(                                           \
 201         "am"#asm_op".d " " $zero, %1, %0        \n"                     \
 202         : "+ZB" (v->counter)                                            \
 203         : "r" (I)                                                       \
 204         : "memory");                                                    \
 205 }
 206
 207 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix)                     \
 208 static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v)   \
 209 {                                                                               \
 210         long result;                                                            \
 211         __asm__ __volatile__(                                                   \
 212         "am"#asm_op#mb".d " " %1, %2, %0                \n"                     \
 213         : "+ZB" (v->counter), "=&r" (result)                                    \
 214         : "r" (I)                                                               \
 215         : "memory");                                                            \
 216                                                                                 \
 217         return result c_op I;                                                   \
 218 }
 219
 220 #define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix)                            \
 221 static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v)      \
 222 {                                                                               \
 223         long result;                                                            \
 224                                                                                 \
 225         __asm__ __volatile__(                                                   \
 226         "am"#asm_op#mb".d " " %1, %2, %0                \n"                     \
 227         : "+ZB" (v->counter), "=&r" (result)                                    \
 228         : "r" (I)                                                               \
 229         : "memory");                                                            \
 230                                                                                 \
 231         return result;                                                          \
 232 }
 233
 234 #define ATOMIC64_OPS(op, I, asm_op, c_op)                                     \
 235         ATOMIC64_OP(op, I, asm_op)                                            \
 236         ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db,         )                \
 237         ATOMIC64_OP_RETURN(op, I, asm_op, c_op,    , _relaxed)                \
 238         ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )                       \
 239         ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)
 240
 241 ATOMIC64_OPS(add, i, add, +)
 242 ATOMIC64_OPS(sub, -i, add, +)
 243
 244 #define arch_atomic64_add_return                arch_atomic64_add_return
 245 #define arch_atomic64_add_return_acquire        arch_atomic64_add_return
 246 #define arch_atomic64_add_return_release        arch_atomic64_add_return
 247 #define arch_atomic64_add_return_relaxed        arch_atomic64_add_return_relaxed
 248 #define arch_atomic64_sub_return                arch_atomic64_sub_return
 249 #define arch_atomic64_sub_return_acquire        arch_atomic64_sub_return
 250 #define arch_atomic64_sub_return_release        arch_atomic64_sub_return
 251 #define arch_atomic64_sub_return_relaxed        arch_atomic64_sub_return_relaxed
 252 #define arch_atomic64_fetch_add                 arch_atomic64_fetch_add
 253 #define arch_atomic64_fetch_add_acquire         arch_atomic64_fetch_add
 254 #define arch_atomic64_fetch_add_release         arch_atomic64_fetch_add
 255 #define arch_atomic64_fetch_add_relaxed         arch_atomic64_fetch_add_relaxed
 256 #define arch_atomic64_fetch_sub                 arch_atomic64_fetch_sub
 257 #define arch_atomic64_fetch_sub_acquire         arch_atomic64_fetch_sub
 258 #define arch_atomic64_fetch_sub_release         arch_atomic64_fetch_sub
 259 #define arch_atomic64_fetch_sub_relaxed         arch_atomic64_fetch_sub_relaxed
 260
 261 #undef ATOMIC64_OPS
 262
 263 #define ATOMIC64_OPS(op, I, asm_op)                                           \
 264         ATOMIC64_OP(op, I, asm_op)                                            \
 265         ATOMIC64_FETCH_OP(op, I, asm_op, _db,         )                       \
 266         ATOMIC64_FETCH_OP(op, I, asm_op,    , _relaxed)
 267
 268 ATOMIC64_OPS(and, i, and)
 269 ATOMIC64_OPS(or, i, or)
 270 ATOMIC64_OPS(xor, i, xor)
 271
 272 #define arch_atomic64_fetch_and         arch_atomic64_fetch_and
 273 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
 274 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
 275 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
 276 #define arch_atomic64_fetch_or          arch_atomic64_fetch_or
 277 #define arch_atomic64_fetch_or_acquire  arch_atomic64_fetch_or
 278 #define arch_atomic64_fetch_or_release  arch_atomic64_fetch_or
 279 #define arch_atomic64_fetch_or_relaxed  arch_atomic64_fetch_or_relaxed
 280 #define arch_atomic64_fetch_xor         arch_atomic64_fetch_xor
 281 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
 282 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
 283 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
 284
 285 #undef ATOMIC64_OPS
 286 #undef ATOMIC64_FETCH_OP
 287 #undef ATOMIC64_OP_RETURN
 288 #undef ATOMIC64_OP
 289
 290 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
 291 {
 292        long prev, rc;
 293
 294         __asm__ __volatile__ (
 295                 "0:     ll.d    %[p],  %[c]\n"
 296                 "       beq     %[p],  %[u], 1f\n"
 297                 "       add.d   %[rc], %[p], %[a]\n"
 298                 "       sc.d    %[rc], %[c]\n"
 299                 "       beqz    %[rc], 0b\n"
 300                 "       b       2f\n"
 301                 "1:\n"
 302                 __WEAK_LLSC_MB
 303                 "2:\n"
 304                 : [p]"=&r" (prev), [rc]"=&r" (rc),
 305                   [c] "=ZB" (v->counter)
 306                 : [a]"r" (a), [u]"r" (u)
 307                 : "memory");
 308
 309         return prev;
 310 }
 311 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
 312
 313 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
 314 {
 315         long result;
 316         long temp;
 317
 318         if (__builtin_constant_p(i)) {
 319                 __asm__ __volatile__(
 320                 "1:     ll.d    %1, %2  # atomic64_sub_if_positive      \n"
 321                 "       addi.d  %0, %1, %3                              \n"
 322                 "       move    %1, %0                                  \n"
 323                 "       bltz    %0, 2f                                  \n"
 324                 "       sc.d    %1, %2                                  \n"
 325                 "       beqz    %1, 1b                                  \n"
 326                 "2:                                                     \n"
 327                 __WEAK_LLSC_MB
 328                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 329                 : "I" (-i));
 330         } else {
 331                 __asm__ __volatile__(
 332                 "1:     ll.d    %1, %2  # atomic64_sub_if_positive      \n"
 333                 "       sub.d   %0, %1, %3                              \n"
 334                 "       move    %1, %0                                  \n"
 335                 "       bltz    %0, 2f                                  \n"
 336                 "       sc.d    %1, %2                                  \n"
 337                 "       beqz    %1, 1b                                  \n"
 338                 "2:                                                     \n"
 339                 __WEAK_LLSC_MB
 340                 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
 341                 : "r" (i));
 342         }
 343
 344         return result;
 345 }
 346
 347 #define arch_atomic64_dec_if_positive(v)        arch_atomic64_sub_if_positive(1, v)
 348
 349 #endif /* CONFIG_64BIT */
 350
 351 #endif /* _ASM_ATOMIC_H */