arch/arm/include/asm/cmpxchg.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_ARM_CMPXCHG_H
   3 #define __ASM_ARM_CMPXCHG_H
   4
   5 #include <linux/irqflags.h>
   6 #include <linux/prefetch.h>
   7 #include <asm/barrier.h>
   8 #include <linux/cmpxchg-emu.h>
   9
  10 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
  11 /*
  12  * On the StrongARM, "swp" is terminally broken since it bypasses the
  13  * cache totally.  This means that the cache becomes inconsistent, and,
  14  * since we use normal loads/stores as well, this is really bad.
  15  * Typically, this causes oopsen in filp_close, but could have other,
  16  * more disastrous effects.  There are two work-arounds:
  17  *  1. Disable interrupts and emulate the atomic swap
  18  *  2. Clean the cache, perform atomic swap, flush the cache
  19  *
  20  * We choose (1) since its the "easiest" to achieve here and is not
  21  * dependent on the processor type.
  22  *
  23  * NOTE that this solution won't work on an SMP system, so explcitly
  24  * forbid it here.
  25  */
  26 #define swp_is_buggy
  27 #endif
  28
  29 static inline unsigned long
  30 __arch_xchg(unsigned long x, volatile void *ptr, int size)
  31 {
  32         extern void __bad_xchg(volatile void *, int);
  33         unsigned long ret;
  34 #ifdef swp_is_buggy
  35         unsigned long flags;
  36 #endif
  37 #if __LINUX_ARM_ARCH__ >= 6
  38         unsigned int tmp;
  39 #endif
  40
  41         prefetchw((const void *)ptr);
  42
  43         switch (size) {
  44 #if __LINUX_ARM_ARCH__ >= 6
  45 #ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */
  46         case 1:
  47                 asm volatile("@ __xchg1\n"
  48                 "1:     ldrexb  %0, [%3]\n"
  49                 "       strexb  %1, %2, [%3]\n"
  50                 "       teq     %1, #0\n"
  51                 "       bne     1b"
  52                         : "=&r" (ret), "=&r" (tmp)
  53                         : "r" (x), "r" (ptr)
  54                         : "memory", "cc");
  55                 break;
  56         case 2:
  57                 asm volatile("@ __xchg2\n"
  58                 "1:     ldrexh  %0, [%3]\n"
  59                 "       strexh  %1, %2, [%3]\n"
  60                 "       teq     %1, #0\n"
  61                 "       bne     1b"
  62                         : "=&r" (ret), "=&r" (tmp)
  63                         : "r" (x), "r" (ptr)
  64                         : "memory", "cc");
  65                 break;
  66 #endif
  67         case 4:
  68                 asm volatile("@ __xchg4\n"
  69                 "1:     ldrex   %0, [%3]\n"
  70                 "       strex   %1, %2, [%3]\n"
  71                 "       teq     %1, #0\n"
  72                 "       bne     1b"
  73                         : "=&r" (ret), "=&r" (tmp)
  74                         : "r" (x), "r" (ptr)
  75                         : "memory", "cc");
  76                 break;
  77 #elif defined(swp_is_buggy)
  78 #ifdef CONFIG_SMP
  79 #error SMP is not supported on this platform
  80 #endif
  81         case 1:
  82                 raw_local_irq_save(flags);
  83                 ret = *(volatile unsigned char *)ptr;
  84                 *(volatile unsigned char *)ptr = x;
  85                 raw_local_irq_restore(flags);
  86                 break;
  87
  88         case 4:
  89                 raw_local_irq_save(flags);
  90                 ret = *(volatile unsigned long *)ptr;
  91                 *(volatile unsigned long *)ptr = x;
  92                 raw_local_irq_restore(flags);
  93                 break;
  94 #else
  95         case 1:
  96                 asm volatile("@ __xchg1\n"
  97                 "       swpb    %0, %1, [%2]"
  98                         : "=&r" (ret)
  99                         : "r" (x), "r" (ptr)
 100                         : "memory", "cc");
 101                 break;
 102         case 4:
 103                 asm volatile("@ __xchg4\n"
 104                 "       swp     %0, %1, [%2]"
 105                         : "=&r" (ret)
 106                         : "r" (x), "r" (ptr)
 107                         : "memory", "cc");
 108                 break;
 109 #endif
 110         default:
 111                 /* Cause a link-time error, the xchg() size is not supported */
 112                 __bad_xchg(ptr, size), ret = 0;
 113                 break;
 114         }
 115
 116         return ret;
 117 }
 118
 119 #define arch_xchg_relaxed(ptr, x) ({                                    \
 120         (__typeof__(*(ptr)))__arch_xchg((unsigned long)(x), (ptr),      \
 121                                         sizeof(*(ptr)));                \
 122 })
 123
 124 #include <asm-generic/cmpxchg-local.h>
 125
 126 #if __LINUX_ARM_ARCH__ < 6
 127 /* min ARCH < ARMv6 */
 128
 129 #ifdef CONFIG_SMP
 130 #error "SMP is not supported on this platform"
 131 #endif
 132
 133 #define arch_xchg arch_xchg_relaxed
 134
 135 /*
 136  * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
 137  * them available.
 138  */
 139 #define arch_cmpxchg_local(ptr, o, n) ({                                \
 140         (__typeof(*ptr))__generic_cmpxchg_local((ptr),                  \
 141                                                 (unsigned long)(o),     \
 142                                                 (unsigned long)(n),     \
 143                                                 sizeof(*(ptr)));        \
 144 })
 145
 146 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
 147
 148 #include <asm-generic/cmpxchg.h>
 149
 150 #else   /* min ARCH >= ARMv6 */
 151
 152 extern void __bad_cmpxchg(volatile void *ptr, int size);
 153
 154 /*
 155  * cmpxchg only support 32-bits operands on ARMv6.
 156  */
 157
 158 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 159                                       unsigned long new, int size)
 160 {
 161         unsigned long oldval, res;
 162
 163         prefetchw((const void *)ptr);
 164
 165         switch (size) {
 166 #ifdef CONFIG_CPU_V6    /* ARCH == ARMv6 */
 167         case 1:
 168                 oldval = cmpxchg_emu_u8((volatile u8 *)ptr, old, new);
 169                 break;
 170 #else /* min ARCH > ARMv6 */
 171         case 1:
 172                 do {
 173                         asm volatile("@ __cmpxchg1\n"
 174                         "       ldrexb  %1, [%2]\n"
 175                         "       mov     %0, #0\n"
 176                         "       teq     %1, %3\n"
 177                         "       strexbeq %0, %4, [%2]\n"
 178                                 : "=&r" (res), "=&r" (oldval)
 179                                 : "r" (ptr), "Ir" (old), "r" (new)
 180                                 : "memory", "cc");
 181                 } while (res);
 182                 break;
 183         case 2:
 184                 do {
 185                         asm volatile("@ __cmpxchg1\n"
 186                         "       ldrexh  %1, [%2]\n"
 187                         "       mov     %0, #0\n"
 188                         "       teq     %1, %3\n"
 189                         "       strexheq %0, %4, [%2]\n"
 190                                 : "=&r" (res), "=&r" (oldval)
 191                                 : "r" (ptr), "Ir" (old), "r" (new)
 192                                 : "memory", "cc");
 193                 } while (res);
 194                 break;
 195 #endif
 196         case 4:
 197                 do {
 198                         asm volatile("@ __cmpxchg4\n"
 199                         "       ldrex   %1, [%2]\n"
 200                         "       mov     %0, #0\n"
 201                         "       teq     %1, %3\n"
 202                         "       strexeq %0, %4, [%2]\n"
 203                                 : "=&r" (res), "=&r" (oldval)
 204                                 : "r" (ptr), "Ir" (old), "r" (new)
 205                                 : "memory", "cc");
 206                 } while (res);
 207                 break;
 208         default:
 209                 __bad_cmpxchg(ptr, size);
 210                 oldval = 0;
 211         }
 212
 213         return oldval;
 214 }
 215
 216 #define arch_cmpxchg_relaxed(ptr,o,n) ({                                \
 217         (__typeof__(*(ptr)))__cmpxchg((ptr),                            \
 218                                       (unsigned long)(o),               \
 219                                       (unsigned long)(n),               \
 220                                       sizeof(*(ptr)));                  \
 221 })
 222
 223 static inline unsigned long __cmpxchg_local(volatile void *ptr,
 224                                             unsigned long old,
 225                                             unsigned long new, int size)
 226 {
 227         unsigned long ret;
 228
 229         switch (size) {
 230 #ifdef CONFIG_CPU_V6    /* min ARCH == ARMv6 */
 231         case 1:
 232         case 2:
 233                 ret = __generic_cmpxchg_local(ptr, old, new, size);
 234                 break;
 235 #endif
 236         default:
 237                 ret = __cmpxchg(ptr, old, new, size);
 238         }
 239
 240         return ret;
 241 }
 242
 243 #define arch_cmpxchg_local(ptr, o, n) ({                                \
 244         (__typeof(*ptr))__cmpxchg_local((ptr),                          \
 245                                         (unsigned long)(o),             \
 246                                         (unsigned long)(n),             \
 247                                         sizeof(*(ptr)));                \
 248 })
 249
 250 static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
 251                                              unsigned long long old,
 252                                              unsigned long long new)
 253 {
 254         unsigned long long oldval;
 255         unsigned long res;
 256
 257         prefetchw(ptr);
 258
 259         __asm__ __volatile__(
 260 "1:     ldrexd          %1, %H1, [%3]\n"
 261 "       teq             %1, %4\n"
 262 "       teqeq           %H1, %H4\n"
 263 "       bne             2f\n"
 264 "       strexd          %0, %5, %H5, [%3]\n"
 265 "       teq             %0, #0\n"
 266 "       bne             1b\n"
 267 "2:"
 268         : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
 269         : "r" (ptr), "r" (old), "r" (new)
 270         : "cc");
 271
 272         return oldval;
 273 }
 274
 275 #define arch_cmpxchg64_relaxed(ptr, o, n) ({                            \
 276         (__typeof__(*(ptr)))__cmpxchg64((ptr),                          \
 277                                         (unsigned long long)(o),        \
 278                                         (unsigned long long)(n));       \
 279 })
 280
 281 #define arch_cmpxchg64_local(ptr, o, n) arch_cmpxchg64_relaxed((ptr), (o), (n))
 282
 283 #endif  /* __LINUX_ARM_ARCH__ >= 6 */
 284
 285 #endif /* __ASM_ARM_CMPXCHG_H */