arch/s390/lib/spinlock.c

   1 /*
   2  *    Out of line spinlock code.
   3  *
   4  *    Copyright IBM Corp. 2004, 2006
   5  *    Author(s): Martin Schwidefsky ([email protected])
   6  */
   7
   8 #include <linux/types.h>
   9 #include <linux/export.h>
  10 #include <linux/spinlock.h>
  11 #include <linux/init.h>
  12 #include <linux/smp.h>
  13 #include <asm/io.h>
  14
  15 int spin_retry = -1;
  16
  17 static int __init spin_retry_init(void)
  18 {
  19         if (spin_retry < 0)
  20                 spin_retry = 1000;
  21         return 0;
  22 }
  23 early_initcall(spin_retry_init);
  24
  25 /**
  26  * spin_retry= parameter
  27  */
  28 static int __init spin_retry_setup(char *str)
  29 {
  30         spin_retry = simple_strtoul(str, &str, 0);
  31         return 1;
  32 }
  33 __setup("spin_retry=", spin_retry_setup);
  34
  35 static inline int arch_load_niai4(int *lock)
  36 {
  37         int owner;
  38
  39         asm volatile(
  40 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
  41                 "       .long   0xb2fa0040\n"   /* NIAI 4 */
  42 #endif
  43                 "       l       %0,%1\n"
  44                 : "=d" (owner) : "Q" (*lock) : "memory");
  45        return owner;
  46 }
  47
  48 static inline int arch_cmpxchg_niai8(int *lock, int old, int new)
  49 {
  50         int expected = old;
  51
  52         asm volatile(
  53 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
  54                 "       .long   0xb2fa0080\n"   /* NIAI 8 */
  55 #endif
  56                 "       cs      %0,%3,%1\n"
  57                 : "=d" (old), "=Q" (*lock)
  58                 : "0" (old), "d" (new), "Q" (*lock)
  59                 : "cc", "memory");
  60         return expected == old;
  61 }
  62
  63 void arch_spin_lock_wait(arch_spinlock_t *lp)
  64 {
  65         int cpu = SPINLOCK_LOCKVAL;
  66         int owner, count;
  67
  68         /* Pass the virtual CPU to the lock holder if it is not running */
  69         owner = arch_load_niai4(&lp->lock);
  70         if (owner && arch_vcpu_is_preempted(~owner))
  71                 smp_yield_cpu(~owner);
  72
  73         count = spin_retry;
  74         while (1) {
  75                 owner = arch_load_niai4(&lp->lock);
  76                 /* Try to get the lock if it is free. */
  77                 if (!owner) {
  78                         if (arch_cmpxchg_niai8(&lp->lock, 0, cpu))
  79                                 return;
  80                         continue;
  81                 }
  82                 if (count-- >= 0)
  83                         continue;
  84                 count = spin_retry;
  85                 /*
  86                  * For multiple layers of hypervisors, e.g. z/VM + LPAR
  87                  * yield the CPU unconditionally. For LPAR rely on the
  88                  * sense running status.
  89                  */
  90                 if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner))
  91                         smp_yield_cpu(~owner);
  92         }
  93 }
  94 EXPORT_SYMBOL(arch_spin_lock_wait);
  95
  96 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
  97 {
  98         int cpu = SPINLOCK_LOCKVAL;
  99         int owner, count;
 100
 101         local_irq_restore(flags);
 102
 103         /* Pass the virtual CPU to the lock holder if it is not running */
 104         owner = arch_load_niai4(&lp->lock);
 105         if (owner && arch_vcpu_is_preempted(~owner))
 106                 smp_yield_cpu(~owner);
 107
 108         count = spin_retry;
 109         while (1) {
 110                 owner = arch_load_niai4(&lp->lock);
 111                 /* Try to get the lock if it is free. */
 112                 if (!owner) {
 113                         local_irq_disable();
 114                         if (arch_cmpxchg_niai8(&lp->lock, 0, cpu))
 115                                 return;
 116                         local_irq_restore(flags);
 117                         continue;
 118                 }
 119                 if (count-- >= 0)
 120                         continue;
 121                 count = spin_retry;
 122                 /*
 123                  * For multiple layers of hypervisors, e.g. z/VM + LPAR
 124                  * yield the CPU unconditionally. For LPAR rely on the
 125                  * sense running status.
 126                  */
 127                 if (!MACHINE_IS_LPAR || arch_vcpu_is_preempted(~owner))
 128                         smp_yield_cpu(~owner);
 129         }
 130 }
 131 EXPORT_SYMBOL(arch_spin_lock_wait_flags);
 132
 133 int arch_spin_trylock_retry(arch_spinlock_t *lp)
 134 {
 135         int cpu = SPINLOCK_LOCKVAL;
 136         int owner, count;
 137
 138         for (count = spin_retry; count > 0; count--) {
 139                 owner = READ_ONCE(lp->lock);
 140                 /* Try to get the lock if it is free. */
 141                 if (!owner) {
 142                         if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
 143                                 return 1;
 144                 }
 145         }
 146         return 0;
 147 }
 148 EXPORT_SYMBOL(arch_spin_trylock_retry);
 149
 150 void _raw_read_lock_wait(arch_rwlock_t *rw)
 151 {
 152         int count = spin_retry;
 153         int owner, old;
 154
 155 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 156         __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
 157 #endif
 158         owner = 0;
 159         while (1) {
 160                 if (count-- <= 0) {
 161                         if (owner && arch_vcpu_is_preempted(~owner))
 162                                 smp_yield_cpu(~owner);
 163                         count = spin_retry;
 164                 }
 165                 old = ACCESS_ONCE(rw->lock);
 166                 owner = ACCESS_ONCE(rw->owner);
 167                 if (old < 0)
 168                         continue;
 169                 if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
 170                         return;
 171         }
 172 }
 173 EXPORT_SYMBOL(_raw_read_lock_wait);
 174
 175 int _raw_read_trylock_retry(arch_rwlock_t *rw)
 176 {
 177         int count = spin_retry;
 178         int old;
 179
 180         while (count-- > 0) {
 181                 old = ACCESS_ONCE(rw->lock);
 182                 if (old < 0)
 183                         continue;
 184                 if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
 185                         return 1;
 186         }
 187         return 0;
 188 }
 189 EXPORT_SYMBOL(_raw_read_trylock_retry);
 190
 191 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 192
 193 void _raw_write_lock_wait(arch_rwlock_t *rw, int prev)
 194 {
 195         int count = spin_retry;
 196         int owner, old;
 197
 198         owner = 0;
 199         while (1) {
 200                 if (count-- <= 0) {
 201                         if (owner && arch_vcpu_is_preempted(~owner))
 202                                 smp_yield_cpu(~owner);
 203                         count = spin_retry;
 204                 }
 205                 old = ACCESS_ONCE(rw->lock);
 206                 owner = ACCESS_ONCE(rw->owner);
 207                 smp_mb();
 208                 if (old >= 0) {
 209                         prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
 210                         old = prev;
 211                 }
 212                 if ((old & 0x7fffffff) == 0 && prev >= 0)
 213                         break;
 214         }
 215 }
 216 EXPORT_SYMBOL(_raw_write_lock_wait);
 217
 218 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 219
 220 void _raw_write_lock_wait(arch_rwlock_t *rw)
 221 {
 222         int count = spin_retry;
 223         int owner, old, prev;
 224
 225         prev = 0x80000000;
 226         owner = 0;
 227         while (1) {
 228                 if (count-- <= 0) {
 229                         if (owner && arch_vcpu_is_preempted(~owner))
 230                                 smp_yield_cpu(~owner);
 231                         count = spin_retry;
 232                 }
 233                 old = ACCESS_ONCE(rw->lock);
 234                 owner = ACCESS_ONCE(rw->owner);
 235                 if (old >= 0 &&
 236                     __atomic_cmpxchg_bool(&rw->lock, old, old | 0x80000000))
 237                         prev = old;
 238                 else
 239                         smp_mb();
 240                 if ((old & 0x7fffffff) == 0 && prev >= 0)
 241                         break;
 242         }
 243 }
 244 EXPORT_SYMBOL(_raw_write_lock_wait);
 245
 246 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 247
 248 int _raw_write_trylock_retry(arch_rwlock_t *rw)
 249 {
 250         int count = spin_retry;
 251         int old;
 252
 253         while (count-- > 0) {
 254                 old = ACCESS_ONCE(rw->lock);
 255                 if (old)
 256                         continue;
 257                 if (__atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000))
 258                         return 1;
 259         }
 260         return 0;
 261 }
 262 EXPORT_SYMBOL(_raw_write_trylock_retry);
 263
 264 void arch_lock_relax(int cpu)
 265 {
 266         if (!cpu)
 267                 return;
 268         if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(~cpu))
 269                 return;
 270         smp_yield_cpu(~cpu);
 271 }
 272 EXPORT_SYMBOL(arch_lock_relax);