kernel/rcu/tiny.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
   4  *
   5  * Copyright IBM Corporation, 2008
   6  *
   7  * Author: Paul E. McKenney <[email protected]>
   8  *
   9  * For detailed explanation of Read-Copy Update mechanism see -
  10  *              Documentation/RCU
  11  */
  12 #include <linux/completion.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/notifier.h>
  15 #include <linux/rcupdate_wait.h>
  16 #include <linux/kernel.h>
  17 #include <linux/export.h>
  18 #include <linux/mutex.h>
  19 #include <linux/sched.h>
  20 #include <linux/types.h>
  21 #include <linux/init.h>
  22 #include <linux/time.h>
  23 #include <linux/cpu.h>
  24 #include <linux/prefetch.h>
  25 #include <linux/slab.h>
  26 #include <linux/mm.h>
  27
  28 #include "rcu.h"
  29
  30 /* Global control variables for rcupdate callback mechanism. */
  31 struct rcu_ctrlblk {
  32         struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */
  33         struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */
  34         struct rcu_head **curtail;      /* ->next pointer of last CB. */
  35 };
  36
  37 /* Definition for rcupdate control block. */
  38 static struct rcu_ctrlblk rcu_ctrlblk = {
  39         .donetail       = &rcu_ctrlblk.rcucblist,
  40         .curtail        = &rcu_ctrlblk.rcucblist,
  41 };
  42
  43 void rcu_barrier(void)
  44 {
  45         wait_rcu_gp(call_rcu);
  46 }
  47 EXPORT_SYMBOL(rcu_barrier);
  48
  49 /* Record an rcu quiescent state.  */
  50 void rcu_qs(void)
  51 {
  52         unsigned long flags;
  53
  54         local_irq_save(flags);
  55         if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  56                 rcu_ctrlblk.donetail = rcu_ctrlblk.curtail;
  57                 raise_softirq_irqoff(RCU_SOFTIRQ);
  58         }
  59         local_irq_restore(flags);
  60 }
  61
  62 /*
  63  * Check to see if the scheduling-clock interrupt came from an extended
  64  * quiescent state, and, if so, tell RCU about it.  This function must
  65  * be called from hardirq context.  It is normally called from the
  66  * scheduling-clock interrupt.
  67  */
  68 void rcu_sched_clock_irq(int user)
  69 {
  70         if (user) {
  71                 rcu_qs();
  72         } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  73                 set_tsk_need_resched(current);
  74                 set_preempt_need_resched();
  75         }
  76 }
  77
  78 /*
  79  * Reclaim the specified callback, either by invoking it for non-kfree cases or
  80  * freeing it directly (for kfree). Return true if kfreeing, false otherwise.
  81  */
  82 static inline bool rcu_reclaim_tiny(struct rcu_head *head)
  83 {
  84         rcu_callback_t f;
  85         unsigned long offset = (unsigned long)head->func;
  86
  87         rcu_lock_acquire(&rcu_callback_map);
  88         if (__is_kvfree_rcu_offset(offset)) {
  89                 trace_rcu_invoke_kvfree_callback("", head, offset);
  90                 kvfree((void *)head - offset);
  91                 rcu_lock_release(&rcu_callback_map);
  92                 return true;
  93         }
  94
  95         trace_rcu_invoke_callback("", head);
  96         f = head->func;
  97         WRITE_ONCE(head->func, (rcu_callback_t)0L);
  98         f(head);
  99         rcu_lock_release(&rcu_callback_map);
 100         return false;
 101 }
 102
 103 /* Invoke the RCU callbacks whose grace period has elapsed.  */
 104 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
 105 {
 106         struct rcu_head *next, *list;
 107         unsigned long flags;
 108
 109         /* Move the ready-to-invoke callbacks to a local list. */
 110         local_irq_save(flags);
 111         if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) {
 112                 /* No callbacks ready, so just leave. */
 113                 local_irq_restore(flags);
 114                 return;
 115         }
 116         list = rcu_ctrlblk.rcucblist;
 117         rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail;
 118         *rcu_ctrlblk.donetail = NULL;
 119         if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail)
 120                 rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist;
 121         rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist;
 122         local_irq_restore(flags);
 123
 124         /* Invoke the callbacks on the local list. */
 125         while (list) {
 126                 next = list->next;
 127                 prefetch(next);
 128                 debug_rcu_head_unqueue(list);
 129                 local_bh_disable();
 130                 rcu_reclaim_tiny(list);
 131                 local_bh_enable();
 132                 list = next;
 133         }
 134 }
 135
 136 /*
 137  * Wait for a grace period to elapse.  But it is illegal to invoke
 138  * synchronize_rcu() from within an RCU read-side critical section.
 139  * Therefore, any legal call to synchronize_rcu() is a quiescent
 140  * state, and so on a UP system, synchronize_rcu() need do nothing.
 141  * (But Lai Jiangshan points out the benefits of doing might_sleep()
 142  * to reduce latency.)
 143  *
 144  * Cool, huh?  (Due to Josh Triplett.)
 145  */
 146 void synchronize_rcu(void)
 147 {
 148         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
 149                          lock_is_held(&rcu_lock_map) ||
 150                          lock_is_held(&rcu_sched_lock_map),
 151                          "Illegal synchronize_rcu() in RCU read-side critical section");
 152 }
 153 EXPORT_SYMBOL_GPL(synchronize_rcu);
 154
 155 /*
 156  * Post an RCU callback to be invoked after the end of an RCU grace
 157  * period.  But since we have but one CPU, that would be after any
 158  * quiescent state.
 159  */
 160 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 161 {
 162         unsigned long flags;
 163
 164         debug_rcu_head_queue(head);
 165         head->func = func;
 166         head->next = NULL;
 167
 168         local_irq_save(flags);
 169         *rcu_ctrlblk.curtail = head;
 170         rcu_ctrlblk.curtail = &head->next;
 171         local_irq_restore(flags);
 172
 173         if (unlikely(is_idle_task(current))) {
 174                 /* force scheduling for rcu_qs() */
 175                 resched_cpu(0);
 176         }
 177 }
 178 EXPORT_SYMBOL_GPL(call_rcu);
 179
 180 void __init rcu_init(void)
 181 {
 182         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 183         rcu_early_boot_tests();
 184         srcu_init();
 185 }