projects / linux.git / blame
| Commit | Line | Data |
|---|---|---|
| 7c7e3d31 SL |
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | /* Copyright (c) 2021 Facebook | |
| 3 | */ | |
| 4 | ||
| 5 | #ifndef __MMAP_UNLOCK_WORK_H__ | |
| 6 | #define __MMAP_UNLOCK_WORK_H__ | |
| 7 | #include <linux/irq_work.h> | |
| 8 | ||
| 9 | /* irq_work to run mmap_read_unlock() in irq_work */ | |
| 10 | struct mmap_unlock_irq_work { | |
| 11 | struct irq_work irq_work; | |
| 12 | struct mm_struct *mm; | |
| 13 | }; | |
| 14 | ||
| 15 | DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); | |
| 16 | ||
| 17 | /* | |
| 18 | * We cannot do mmap_read_unlock() when the irq is disabled, because of | |
| 19 | * risk to deadlock with rq_lock. To look up vma when the irqs are | |
| 20 | * disabled, we need to run mmap_read_unlock() in irq_work. We use a | |
| 21 | * percpu variable to do the irq_work. If the irq_work is already used | |
| 22 | * by another lookup, we fall over. | |
| 23 | */ | |
| 24 | static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) | |
| 25 | { | |
| 26 | struct mmap_unlock_irq_work *work = NULL; | |
| 27 | bool irq_work_busy = false; | |
| 28 | ||
| 29 | if (irqs_disabled()) { | |
| 30 | if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { | |
| 31 | work = this_cpu_ptr(&mmap_unlock_work); | |
| 32 | if (irq_work_is_busy(&work->irq_work)) { | |
| 33 | /* cannot queue more up_read, fallback */ | |
| 34 | irq_work_busy = true; | |
| 35 | } | |
| 36 | } else { | |
| 37 | /* | |
| 38 | * PREEMPT_RT does not allow to trylock mmap sem in | |
| 39 | * interrupt disabled context. Force the fallback code. | |
| 40 | */ | |
| 41 | irq_work_busy = true; | |
| 42 | } | |
| 43 | } | |
| 44 | ||
| 45 | *work_ptr = work; | |
| 46 | return irq_work_busy; | |
| 47 | } | |
| 48 | ||
| 49 | static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) | |
| 50 | { | |
| 51 | if (!work) { | |
| 52 | mmap_read_unlock(mm); | |
| 53 | } else { | |
| 54 | work->mm = mm; | |
| 55 | ||
| 56 | /* The lock will be released once we're out of interrupt | |
| 57 | * context. Tell lockdep that we've released it now so | |
| 58 | * it doesn't complain that we forgot to release it. | |
| 59 | */ | |
| 60 | rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_); | |
| 61 | irq_work_queue(&work->irq_work); | |
| 62 | } | |
| 63 | } | |
| 64 | ||
| 65 | #endif /* __MMAP_UNLOCK_WORK_H__ */ |