1 /* SPDX-License-Identifier: GPL-2.0 */
5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
10 #include <linux/types.h>
11 #include <asm/barrier.h>
12 #include <asm/cmpxchg.h>
14 #if __SIZEOF_LONG__ == 4
17 #define __AMADD "amadd.w "
18 #define __AMAND_DB "amand_db.w "
19 #define __AMOR_DB "amor_db.w "
20 #define __AMXOR_DB "amxor_db.w "
21 #elif __SIZEOF_LONG__ == 8
24 #define __AMADD "amadd.d "
25 #define __AMAND_DB "amand_db.d "
26 #define __AMOR_DB "amor_db.d "
27 #define __AMXOR_DB "amxor_db.d "
30 #define ATOMIC_INIT(i) { (i) }
33 * arch_atomic_read - read atomic variable
34 * @v: pointer of type atomic_t
36 * Atomically reads the value of @v.
38 #define arch_atomic_read(v) READ_ONCE((v)->counter)
41 * arch_atomic_set - set atomic variable
42 * @v: pointer of type atomic_t
45 * Atomically sets the value of @v to @i.
47 #define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))
49 #define ATOMIC_OP(op, I, asm_op) \
50 static inline void arch_atomic_##op(int i, atomic_t *v) \
52 __asm__ __volatile__( \
53 "am"#asm_op"_db.w" " $zero, %1, %0 \n" \
54 : "+ZB" (v->counter) \
59 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op) \
60 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
64 __asm__ __volatile__( \
65 "am"#asm_op"_db.w" " %1, %2, %0 \n" \
66 : "+ZB" (v->counter), "=&r" (result) \
70 return result c_op I; \
73 #define ATOMIC_FETCH_OP(op, I, asm_op) \
74 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
78 __asm__ __volatile__( \
79 "am"#asm_op"_db.w" " %1, %2, %0 \n" \
80 : "+ZB" (v->counter), "=&r" (result) \
87 #define ATOMIC_OPS(op, I, asm_op, c_op) \
88 ATOMIC_OP(op, I, asm_op) \
89 ATOMIC_OP_RETURN(op, I, asm_op, c_op) \
90 ATOMIC_FETCH_OP(op, I, asm_op)
92 ATOMIC_OPS(add, i, add, +)
93 ATOMIC_OPS(sub, -i, add, +)
95 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
96 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
97 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
98 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
102 #define ATOMIC_OPS(op, I, asm_op) \
103 ATOMIC_OP(op, I, asm_op) \
104 ATOMIC_FETCH_OP(op, I, asm_op)
106 ATOMIC_OPS(and, i, and)
107 ATOMIC_OPS(or, i, or)
108 ATOMIC_OPS(xor, i, xor)
110 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
111 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
112 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
115 #undef ATOMIC_FETCH_OP
116 #undef ATOMIC_OP_RETURN
119 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
123 __asm__ __volatile__ (
124 "0: ll.w %[p], %[c]\n"
125 " beq %[p], %[u], 1f\n"
126 " add.w %[rc], %[p], %[a]\n"
127 " sc.w %[rc], %[c]\n"
133 : [p]"=&r" (prev), [rc]"=&r" (rc),
134 [c]"=ZB" (v->counter)
135 : [a]"r" (a), [u]"r" (u)
140 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
143 * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable
144 * @i: integer value to subtract
145 * @v: pointer of type atomic_t
147 * Atomically test @v and subtract @i if @v is greater or equal than @i.
148 * The function returns the old value of @v minus @i.
150 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
155 if (__builtin_constant_p(i)) {
156 __asm__ __volatile__(
157 "1: ll.w %1, %2 # atomic_sub_if_positive\n"
158 " addi.w %0, %1, %3 \n"
165 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
168 __asm__ __volatile__(
169 "1: ll.w %1, %2 # atomic_sub_if_positive\n"
170 " sub.w %0, %1, %3 \n"
177 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
184 #define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
185 #define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
188 * arch_atomic_dec_if_positive - decrement by 1 if old value positive
189 * @v: pointer of type atomic_t
191 #define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v)
195 #define ATOMIC64_INIT(i) { (i) }
198 * arch_atomic64_read - read atomic variable
199 * @v: pointer of type atomic64_t
202 #define arch_atomic64_read(v) READ_ONCE((v)->counter)
205 * arch_atomic64_set - set atomic variable
206 * @v: pointer of type atomic64_t
209 #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
211 #define ATOMIC64_OP(op, I, asm_op) \
212 static inline void arch_atomic64_##op(long i, atomic64_t *v) \
214 __asm__ __volatile__( \
215 "am"#asm_op"_db.d " " $zero, %1, %0 \n" \
216 : "+ZB" (v->counter) \
221 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op) \
222 static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v) \
225 __asm__ __volatile__( \
226 "am"#asm_op"_db.d " " %1, %2, %0 \n" \
227 : "+ZB" (v->counter), "=&r" (result) \
231 return result c_op I; \
234 #define ATOMIC64_FETCH_OP(op, I, asm_op) \
235 static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \
239 __asm__ __volatile__( \
240 "am"#asm_op"_db.d " " %1, %2, %0 \n" \
241 : "+ZB" (v->counter), "=&r" (result) \
248 #define ATOMIC64_OPS(op, I, asm_op, c_op) \
249 ATOMIC64_OP(op, I, asm_op) \
250 ATOMIC64_OP_RETURN(op, I, asm_op, c_op) \
251 ATOMIC64_FETCH_OP(op, I, asm_op)
253 ATOMIC64_OPS(add, i, add, +)
254 ATOMIC64_OPS(sub, -i, add, +)
256 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
257 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
258 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
259 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
263 #define ATOMIC64_OPS(op, I, asm_op) \
264 ATOMIC64_OP(op, I, asm_op) \
265 ATOMIC64_FETCH_OP(op, I, asm_op)
267 ATOMIC64_OPS(and, i, and)
268 ATOMIC64_OPS(or, i, or)
269 ATOMIC64_OPS(xor, i, xor)
271 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
272 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
273 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
276 #undef ATOMIC64_FETCH_OP
277 #undef ATOMIC64_OP_RETURN
280 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
284 __asm__ __volatile__ (
285 "0: ll.d %[p], %[c]\n"
286 " beq %[p], %[u], 1f\n"
287 " add.d %[rc], %[p], %[a]\n"
288 " sc.d %[rc], %[c]\n"
294 : [p]"=&r" (prev), [rc]"=&r" (rc),
295 [c] "=ZB" (v->counter)
296 : [a]"r" (a), [u]"r" (u)
301 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
304 * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable
305 * @i: integer value to subtract
306 * @v: pointer of type atomic64_t
308 * Atomically test @v and subtract @i if @v is greater or equal than @i.
309 * The function returns the old value of @v minus @i.
311 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
316 if (__builtin_constant_p(i)) {
317 __asm__ __volatile__(
318 "1: ll.d %1, %2 # atomic64_sub_if_positive \n"
319 " addi.d %0, %1, %3 \n"
326 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
329 __asm__ __volatile__(
330 "1: ll.d %1, %2 # atomic64_sub_if_positive \n"
331 " sub.d %0, %1, %3 \n"
338 : "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
345 #define arch_atomic64_cmpxchg(v, o, n) \
346 ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
347 #define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
350 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
351 * @v: pointer of type atomic64_t
353 #define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v)
355 #endif /* CONFIG_64BIT */
357 #endif /* _ASM_ATOMIC_H */