arch/x86/include/asm/atomic64_32.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_ATOMIC64_32_H
   3 #define _ASM_X86_ATOMIC64_32_H
   4
   5 #include <linux/compiler.h>
   6 #include <linux/types.h>
   7 //#include <asm/cmpxchg.h>
   8
   9 /* An 64bit atomic type */
  10
  11 typedef struct {
  12         s64 __aligned(8) counter;
  13 } atomic64_t;
  14
  15 #define ATOMIC64_INIT(val)      { (val) }
  16
  17 /*
  18  * Read an atomic64_t non-atomically.
  19  *
  20  * This is intended to be used in cases where a subsequent atomic operation
  21  * will handle the torn value, and can be used to prime the first iteration
  22  * of unconditional try_cmpxchg() loops, e.g.:
  23  *
  24  *      s64 val = arch_atomic64_read_nonatomic(v);
  25  *      do { } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i);
  26  *
  27  * This is NOT safe to use where the value is not always checked by a
  28  * subsequent atomic operation, such as in conditional try_cmpxchg() loops
  29  * that can break before the atomic operation, e.g.:
  30  *
  31  *      s64 val = arch_atomic64_read_nonatomic(v);
  32  *      do {
  33  *              if (condition(val))
  34  *                      break;
  35  *      } while (!arch_atomic64_try_cmpxchg(v, &val, val OP i);
  36  */
  37 static __always_inline s64 arch_atomic64_read_nonatomic(const atomic64_t *v)
  38 {
  39         /* See comment in arch_atomic_read(). */
  40         return __READ_ONCE(v->counter);
  41 }
  42
  43 #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
  44 #ifndef ATOMIC64_EXPORT
  45 #define ATOMIC64_DECL_ONE __ATOMIC64_DECL
  46 #else
  47 #define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \
  48         ATOMIC64_EXPORT(atomic64_##sym)
  49 #endif
  50
  51 #ifdef CONFIG_X86_CMPXCHG64
  52 #define __alternative_atomic64(f, g, out, in...) \
  53         asm volatile("call %c[func]" \
  54                      : ALT_OUTPUT_SP(out) \
  55                      : [func] "i" (atomic64_##g##_cx8), ## in)
  56
  57 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
  58 #else
  59 #define __alternative_atomic64(f, g, out, in...) \
  60         alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \
  61                          X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in)
  62
  63 #define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
  64         ATOMIC64_DECL_ONE(sym##_386)
  65
  66 ATOMIC64_DECL_ONE(add_386);
  67 ATOMIC64_DECL_ONE(sub_386);
  68 ATOMIC64_DECL_ONE(inc_386);
  69 ATOMIC64_DECL_ONE(dec_386);
  70 #endif
  71
  72 #define alternative_atomic64(f, out, in...) \
  73         __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
  74
  75 ATOMIC64_DECL(read);
  76 ATOMIC64_DECL(set);
  77 ATOMIC64_DECL(xchg);
  78 ATOMIC64_DECL(add_return);
  79 ATOMIC64_DECL(sub_return);
  80 ATOMIC64_DECL(inc_return);
  81 ATOMIC64_DECL(dec_return);
  82 ATOMIC64_DECL(dec_if_positive);
  83 ATOMIC64_DECL(inc_not_zero);
  84 ATOMIC64_DECL(add_unless);
  85
  86 #undef ATOMIC64_DECL
  87 #undef ATOMIC64_DECL_ONE
  88 #undef __ATOMIC64_DECL
  89 #undef ATOMIC64_EXPORT
  90
  91 static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
  92 {
  93         return arch_cmpxchg64(&v->counter, old, new);
  94 }
  95 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
  96
  97 static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
  98 {
  99         return arch_try_cmpxchg64(&v->counter, old, new);
 100 }
 101 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
 102
 103 static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)
 104 {
 105         s64 o;
 106         unsigned high = (unsigned)(n >> 32);
 107         unsigned low = (unsigned)n;
 108         alternative_atomic64(xchg, "=&A" (o),
 109                              "S" (v), "b" (low), "c" (high)
 110                              : "memory");
 111         return o;
 112 }
 113 #define arch_atomic64_xchg arch_atomic64_xchg
 114
 115 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
 116 {
 117         unsigned high = (unsigned)(i >> 32);
 118         unsigned low = (unsigned)i;
 119         alternative_atomic64(set, /* no output */,
 120                              "S" (v), "b" (low), "c" (high)
 121                              : "eax", "edx", "memory");
 122 }
 123
 124 static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
 125 {
 126         s64 r;
 127         alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
 128         return r;
 129 }
 130
 131 static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
 132 {
 133         alternative_atomic64(add_return,
 134                              ASM_OUTPUT2("+A" (i), "+c" (v)),
 135                              ASM_NO_INPUT_CLOBBER("memory"));
 136         return i;
 137 }
 138 #define arch_atomic64_add_return arch_atomic64_add_return
 139
 140 static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
 141 {
 142         alternative_atomic64(sub_return,
 143                              ASM_OUTPUT2("+A" (i), "+c" (v)),
 144                              ASM_NO_INPUT_CLOBBER("memory"));
 145         return i;
 146 }
 147 #define arch_atomic64_sub_return arch_atomic64_sub_return
 148
 149 static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v)
 150 {
 151         s64 a;
 152         alternative_atomic64(inc_return, "=&A" (a),
 153                              "S" (v) : "memory", "ecx");
 154         return a;
 155 }
 156 #define arch_atomic64_inc_return arch_atomic64_inc_return
 157
 158 static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v)
 159 {
 160         s64 a;
 161         alternative_atomic64(dec_return, "=&A" (a),
 162                              "S" (v) : "memory", "ecx");
 163         return a;
 164 }
 165 #define arch_atomic64_dec_return arch_atomic64_dec_return
 166
 167 static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
 168 {
 169         __alternative_atomic64(add, add_return,
 170                                ASM_OUTPUT2("+A" (i), "+c" (v)),
 171                                ASM_NO_INPUT_CLOBBER("memory"));
 172 }
 173
 174 static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
 175 {
 176         __alternative_atomic64(sub, sub_return,
 177                                ASM_OUTPUT2("+A" (i), "+c" (v)),
 178                                ASM_NO_INPUT_CLOBBER("memory"));
 179 }
 180
 181 static __always_inline void arch_atomic64_inc(atomic64_t *v)
 182 {
 183         __alternative_atomic64(inc, inc_return, /* no output */,
 184                                "S" (v) : "memory", "eax", "ecx", "edx");
 185 }
 186 #define arch_atomic64_inc arch_atomic64_inc
 187
 188 static __always_inline void arch_atomic64_dec(atomic64_t *v)
 189 {
 190         __alternative_atomic64(dec, dec_return, /* no output */,
 191                                "S" (v) : "memory", "eax", "ecx", "edx");
 192 }
 193 #define arch_atomic64_dec arch_atomic64_dec
 194
 195 static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
 196 {
 197         unsigned low = (unsigned)u;
 198         unsigned high = (unsigned)(u >> 32);
 199         alternative_atomic64(add_unless,
 200                              ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)),
 201                              "S" (v) : "memory");
 202         return (int)a;
 203 }
 204 #define arch_atomic64_add_unless arch_atomic64_add_unless
 205
 206 static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v)
 207 {
 208         int r;
 209         alternative_atomic64(inc_not_zero, "=&a" (r),
 210                              "S" (v) : "ecx", "edx", "memory");
 211         return r;
 212 }
 213 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
 214
 215 static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
 216 {
 217         s64 r;
 218         alternative_atomic64(dec_if_positive, "=&A" (r),
 219                              "S" (v) : "ecx", "memory");
 220         return r;
 221 }
 222 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
 223
 224 #undef alternative_atomic64
 225 #undef __alternative_atomic64
 226
 227 static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)
 228 {
 229         s64 val = arch_atomic64_read_nonatomic(v);
 230
 231         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
 232 }
 233
 234 static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
 235 {
 236         s64 val = arch_atomic64_read_nonatomic(v);
 237
 238         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
 239
 240         return val;
 241 }
 242 #define arch_atomic64_fetch_and arch_atomic64_fetch_and
 243
 244 static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)
 245 {
 246         s64 val = arch_atomic64_read_nonatomic(v);
 247
 248         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
 249 }
 250
 251 static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
 252 {
 253         s64 val = arch_atomic64_read_nonatomic(v);
 254
 255         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
 256
 257         return val;
 258 }
 259 #define arch_atomic64_fetch_or arch_atomic64_fetch_or
 260
 261 static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)
 262 {
 263         s64 val = arch_atomic64_read_nonatomic(v);
 264
 265         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
 266 }
 267
 268 static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
 269 {
 270         s64 val = arch_atomic64_read_nonatomic(v);
 271
 272         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
 273
 274         return val;
 275 }
 276 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
 277
 278 static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
 279 {
 280         s64 val = arch_atomic64_read_nonatomic(v);
 281
 282         do { } while (!arch_atomic64_try_cmpxchg(v, &val, val + i));
 283
 284         return val;
 285 }
 286 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
 287
 288 #define arch_atomic64_fetch_sub(i, v)   arch_atomic64_fetch_add(-(i), (v))
 289
 290 #endif /* _ASM_X86_ATOMIC64_32_H */