kernel/locking/spinlock_rt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * PREEMPT_RT substitution for spin/rw_locks
   4  *
   5  * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
   6  * resemble the non RT semantics:
   7  *
   8  * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
   9  *   preserving. The task state is saved before blocking on the underlying
  10  *   rtmutex, and restored when the lock has been acquired. Regular wakeups
  11  *   during that time are redirected to the saved state so no wake up is
  12  *   missed.
  13  *
  14  * - Non RT spin/rwlocks disable preemption and eventually interrupts.
  15  *   Disabling preemption has the side effect of disabling migration and
  16  *   preventing RCU grace periods.
  17  *
  18  *   The RT substitutions explicitly disable migration and take
  19  *   rcu_read_lock() across the lock held section.
  20  */
  21 #include <linux/spinlock.h>
  22 #include <linux/export.h>
  23
  24 #define RT_MUTEX_BUILD_SPINLOCKS
  25 #include "rtmutex.c"
  26
  27 /*
  28  * __might_resched() skips the state check as rtlocks are state
  29  * preserving. Take RCU nesting into account as spin/read/write_lock() can
  30  * legitimately nest into an RCU read side critical section.
  31  */
  32 #define RTLOCK_RESCHED_OFFSETS                                          \
  33         (rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
  34
  35 #define rtlock_might_resched()                                          \
  36         __might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
  37
  38 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
  39 {
  40         lockdep_assert(!current->pi_blocked_on);
  41
  42         if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
  43                 rtlock_slowlock(rtm);
  44 }
  45
  46 static __always_inline void __rt_spin_lock(spinlock_t *lock)
  47 {
  48         rtlock_might_resched();
  49         rtlock_lock(&lock->lock);
  50         rcu_read_lock();
  51         migrate_disable();
  52 }
  53
  54 void __sched rt_spin_lock(spinlock_t *lock) __acquires(RCU)
  55 {
  56         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  57         __rt_spin_lock(lock);
  58 }
  59 EXPORT_SYMBOL(rt_spin_lock);
  60
  61 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  62 void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
  63 {
  64         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  65         __rt_spin_lock(lock);
  66 }
  67 EXPORT_SYMBOL(rt_spin_lock_nested);
  68
  69 void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
  70                                     struct lockdep_map *nest_lock)
  71 {
  72         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
  73         __rt_spin_lock(lock);
  74 }
  75 EXPORT_SYMBOL(rt_spin_lock_nest_lock);
  76 #endif
  77
  78 void __sched rt_spin_unlock(spinlock_t *lock) __releases(RCU)
  79 {
  80         spin_release(&lock->dep_map, _RET_IP_);
  81         migrate_enable();
  82         rcu_read_unlock();
  83
  84         if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
  85                 rt_mutex_slowunlock(&lock->lock);
  86 }
  87 EXPORT_SYMBOL(rt_spin_unlock);
  88
  89 /*
  90  * Wait for the lock to get unlocked: instead of polling for an unlock
  91  * (like raw spinlocks do), lock and unlock, to force the kernel to
  92  * schedule if there's contention:
  93  */
  94 void __sched rt_spin_lock_unlock(spinlock_t *lock)
  95 {
  96         spin_lock(lock);
  97         spin_unlock(lock);
  98 }
  99 EXPORT_SYMBOL(rt_spin_lock_unlock);
 100
 101 static __always_inline int __rt_spin_trylock(spinlock_t *lock)
 102 {
 103         int ret = 1;
 104
 105         if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
 106                 ret = rt_mutex_slowtrylock(&lock->lock);
 107
 108         if (ret) {
 109                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
 110                 rcu_read_lock();
 111                 migrate_disable();
 112         }
 113         return ret;
 114 }
 115
 116 int __sched rt_spin_trylock(spinlock_t *lock)
 117 {
 118         return __rt_spin_trylock(lock);
 119 }
 120 EXPORT_SYMBOL(rt_spin_trylock);
 121
 122 int __sched rt_spin_trylock_bh(spinlock_t *lock)
 123 {
 124         int ret;
 125
 126         local_bh_disable();
 127         ret = __rt_spin_trylock(lock);
 128         if (!ret)
 129                 local_bh_enable();
 130         return ret;
 131 }
 132 EXPORT_SYMBOL(rt_spin_trylock_bh);
 133
 134 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 135 void __rt_spin_lock_init(spinlock_t *lock, const char *name,
 136                          struct lock_class_key *key, bool percpu)
 137 {
 138         u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
 139
 140         debug_check_no_locks_freed((void *)lock, sizeof(*lock));
 141         lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG,
 142                               LD_WAIT_INV, type);
 143 }
 144 EXPORT_SYMBOL(__rt_spin_lock_init);
 145 #endif
 146
 147 /*
 148  * RT-specific reader/writer locks
 149  */
 150 #define rwbase_set_and_save_current_state(state)        \
 151         current_save_and_set_rtlock_wait_state()
 152
 153 #define rwbase_restore_current_state()                  \
 154         current_restore_rtlock_saved_state()
 155
 156 static __always_inline int
 157 rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
 158 {
 159         if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
 160                 rtlock_slowlock(rtm);
 161         return 0;
 162 }
 163
 164 static __always_inline int
 165 rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state,
 166                                struct wake_q_head *wake_q)
 167 {
 168         rtlock_slowlock_locked(rtm, wake_q);
 169         return 0;
 170 }
 171
 172 static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
 173 {
 174         if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
 175                 return;
 176
 177         rt_mutex_slowunlock(rtm);
 178 }
 179
 180 static __always_inline int  rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
 181 {
 182         if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
 183                 return 1;
 184
 185         return rt_mutex_slowtrylock(rtm);
 186 }
 187
 188 #define rwbase_signal_pending_state(state, current)     (0)
 189
 190 #define rwbase_pre_schedule()
 191
 192 #define rwbase_schedule()                               \
 193         schedule_rtlock()
 194
 195 #define rwbase_post_schedule()
 196
 197 #include "rwbase_rt.c"
 198 /*
 199  * The common functions which get wrapped into the rwlock API.
 200  */
 201 int __sched rt_read_trylock(rwlock_t *rwlock)
 202 {
 203         int ret;
 204
 205         ret = rwbase_read_trylock(&rwlock->rwbase);
 206         if (ret) {
 207                 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
 208                 rcu_read_lock();
 209                 migrate_disable();
 210         }
 211         return ret;
 212 }
 213 EXPORT_SYMBOL(rt_read_trylock);
 214
 215 int __sched rt_write_trylock(rwlock_t *rwlock)
 216 {
 217         int ret;
 218
 219         ret = rwbase_write_trylock(&rwlock->rwbase);
 220         if (ret) {
 221                 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
 222                 rcu_read_lock();
 223                 migrate_disable();
 224         }
 225         return ret;
 226 }
 227 EXPORT_SYMBOL(rt_write_trylock);
 228
 229 void __sched rt_read_lock(rwlock_t *rwlock) __acquires(RCU)
 230 {
 231         rtlock_might_resched();
 232         rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
 233         rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
 234         rcu_read_lock();
 235         migrate_disable();
 236 }
 237 EXPORT_SYMBOL(rt_read_lock);
 238
 239 void __sched rt_write_lock(rwlock_t *rwlock) __acquires(RCU)
 240 {
 241         rtlock_might_resched();
 242         rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
 243         rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
 244         rcu_read_lock();
 245         migrate_disable();
 246 }
 247 EXPORT_SYMBOL(rt_write_lock);
 248
 249 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 250 void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(RCU)
 251 {
 252         rtlock_might_resched();
 253         rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_);
 254         rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
 255         rcu_read_lock();
 256         migrate_disable();
 257 }
 258 EXPORT_SYMBOL(rt_write_lock_nested);
 259 #endif
 260
 261 void __sched rt_read_unlock(rwlock_t *rwlock) __releases(RCU)
 262 {
 263         rwlock_release(&rwlock->dep_map, _RET_IP_);
 264         migrate_enable();
 265         rcu_read_unlock();
 266         rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
 267 }
 268 EXPORT_SYMBOL(rt_read_unlock);
 269
 270 void __sched rt_write_unlock(rwlock_t *rwlock) __releases(RCU)
 271 {
 272         rwlock_release(&rwlock->dep_map, _RET_IP_);
 273         rcu_read_unlock();
 274         migrate_enable();
 275         rwbase_write_unlock(&rwlock->rwbase);
 276 }
 277 EXPORT_SYMBOL(rt_write_unlock);
 278
 279 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 280 void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
 281                       struct lock_class_key *key)
 282 {
 283         debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
 284         lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
 285 }
 286 EXPORT_SYMBOL(__rt_rwlock_init);
 287 #endif