bool (*poll_gp_state_exp)(unsigned long oldstate);
void (*cond_sync_exp)(unsigned long oldstate);
void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp);
++++ + unsigned long (*get_comp_state)(void);
++++ + void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp);
++++ + bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2);
++++ + bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2);
unsigned long (*get_gp_state)(void);
void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);
unsigned long (*get_gp_completed)(void);
static void rcu_torture_deferred_free(struct rcu_torture *p)
{
-- --- call_rcu(&p->rtort_rcu, rcu_torture_cb);
++ +++ call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb);
}
static void rcu_sync_torture_init(void)
.deferred_free = rcu_torture_deferred_free,
.sync = synchronize_rcu,
.exp_sync = synchronize_rcu_expedited,
++++ + .same_gp_state = same_state_synchronize_rcu,
++++ + .same_gp_state_full = same_state_synchronize_rcu_full,
++++ + .get_comp_state = get_completed_synchronize_rcu,
++++ + .get_comp_state_full = get_completed_synchronize_rcu_full,
.get_gp_state = get_state_synchronize_rcu,
.get_gp_state_full = get_state_synchronize_rcu_full,
.get_gp_completed = get_completed_synchronize_rcu,
.start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full,
.poll_gp_state_exp = poll_state_synchronize_rcu,
.cond_sync_exp = cond_synchronize_rcu_expedited,
-- --- .call = call_rcu,
++ +++ .call = call_rcu_hurry,
.cb_barrier = rcu_barrier,
.fqs = rcu_force_quiescent_state,
.stats = NULL,
DEFINE_STATIC_SRCU(srcu_ctl);
static struct srcu_struct srcu_ctld;
static struct srcu_struct *srcu_ctlp = &srcu_ctl;
+++ ++static struct rcu_torture_ops srcud_ops;
static int srcu_torture_read_lock(void) __acquires(srcu_ctlp)
{
--- -- return srcu_read_lock(srcu_ctlp);
+++ ++ if (cur_ops == &srcud_ops)
+++ ++ return srcu_read_lock_nmisafe(srcu_ctlp);
+++ ++ else
+++ ++ return srcu_read_lock(srcu_ctlp);
}
static void
static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp)
{
--- -- srcu_read_unlock(srcu_ctlp, idx);
+++ ++ if (cur_ops == &srcud_ops)
+++ ++ srcu_read_unlock_nmisafe(srcu_ctlp, idx);
+++ ++ else
+++ ++ srcu_read_unlock(srcu_ctlp, idx);
}
static int torture_srcu_read_lock_held(void)
static void synchronize_rcu_mult_test(void)
{
-- --- synchronize_rcu_mult(call_rcu_tasks, call_rcu);
++ +++ synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry);
}
static struct rcu_torture_ops tasks_ops = {
} else if (gp_normal && !cur_ops->deferred_free) {
pr_alert("%s: gp_normal without primitives.\n", __func__);
}
---- - if (gp_poll1 && cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
++++ + if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state &&
++++ + cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
synctype[nsynctypes++] = RTWS_POLL_GET;
pr_info("%s: Testing polling GPs.\n", __func__);
} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
pr_alert("%s: gp_poll without primitives.\n", __func__);
}
---- - if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
++++ + if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full
++++ + && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
synctype[nsynctypes++] = RTWS_POLL_GET_FULL;
pr_info("%s: Testing polling full-state GPs.\n", __func__);
} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
struct rcu_gp_oldstate cookie_full;
int expediting = 0;
unsigned long gp_snap;
++++ + unsigned long gp_snap1;
struct rcu_gp_oldstate gp_snap_full;
++++ + struct rcu_gp_oldstate gp_snap1_full;
int i;
int idx;
int oldnice = task_nice(current);
++++ + struct rcu_gp_oldstate rgo[NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE];
struct rcu_torture *rp;
struct rcu_torture *old_rp;
static DEFINE_TORTURE_RANDOM(rand);
bool stutter_waited;
++++ + unsigned long ulo[NUM_ACTIVE_RCU_POLL_OLDSTATE];
VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
if (!can_expedite)
break;
case RTWS_POLL_GET:
rcu_torture_writer_state = RTWS_POLL_GET;
++++ + for (i = 0; i < ARRAY_SIZE(ulo); i++)
++++ + ulo[i] = cur_ops->get_comp_state();
gp_snap = cur_ops->start_gp_poll();
rcu_torture_writer_state = RTWS_POLL_WAIT;
---- - while (!cur_ops->poll_gp_state(gp_snap))
++++ + while (!cur_ops->poll_gp_state(gp_snap)) {
++++ + gp_snap1 = cur_ops->get_gp_state();
++++ + for (i = 0; i < ARRAY_SIZE(ulo); i++)
++++ + if (cur_ops->poll_gp_state(ulo[i]) ||
++++ + cur_ops->same_gp_state(ulo[i], gp_snap1)) {
++++ + ulo[i] = gp_snap1;
++++ + break;
++++ + }
++++ + WARN_ON_ONCE(i >= ARRAY_SIZE(ulo));
torture_hrtimeout_jiffies(torture_random(&rand) % 16,
&rand);
++++ + }
rcu_torture_pipe_update(old_rp);
break;
case RTWS_POLL_GET_FULL:
rcu_torture_writer_state = RTWS_POLL_GET_FULL;
++++ + for (i = 0; i < ARRAY_SIZE(rgo); i++)
++++ + cur_ops->get_comp_state_full(&rgo[i]);
cur_ops->start_gp_poll_full(&gp_snap_full);
rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
---- - while (!cur_ops->poll_gp_state_full(&gp_snap_full))
++++ + while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
++++ + cur_ops->get_gp_state_full(&gp_snap1_full);
++++ + for (i = 0; i < ARRAY_SIZE(rgo); i++)
++++ + if (cur_ops->poll_gp_state_full(&rgo[i]) ||
++++ + cur_ops->same_gp_state_full(&rgo[i],
++++ + &gp_snap1_full)) {
++++ + rgo[i] = gp_snap1_full;
++++ + break;
++++ + }
++++ + WARN_ON_ONCE(i >= ARRAY_SIZE(rgo));
torture_hrtimeout_jiffies(torture_random(&rand) % 16,
&rand);
++++ + }
rcu_torture_pipe_update(old_rp);
break;
case RTWS_POLL_GET_EXP:
/* Try to queue the rh2 pair of callbacks for the same grace period. */
preempt_disable(); /* Prevent preemption from interrupting test. */
rcu_read_lock(); /* Make it impossible to finish a grace period. */
-- --- call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */
++ +++ call_rcu_hurry(&rh1, rcu_torture_leak_cb); /* Start grace period. */
local_irq_disable(); /* Make it harder to start a new grace period. */
-- --- call_rcu(&rh2, rcu_torture_leak_cb);
-- --- call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
++ +++ call_rcu_hurry(&rh2, rcu_torture_leak_cb);
++ +++ call_rcu_hurry(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
if (rhp) {
-- --- call_rcu(rhp, rcu_torture_leak_cb);
-- --- call_rcu(rhp, rcu_torture_err_cb); /* Another duplicate callback. */
++ +++ call_rcu_hurry(rhp, rcu_torture_leak_cb);
++ +++ call_rcu_hurry(rhp, rcu_torture_err_cb); /* Another duplicate callback. */
}
local_irq_enable();
rcu_read_unlock();
return !(snap & RCU_DYNTICKS_IDX);
}
- ----/* Return true if the specified CPU is currently idle from an RCU viewpoint. */
- ----bool rcu_is_idle_cpu(int cpu)
- ----{
- ---- return rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu));
- ----}
- ----
/*
* Return true if the CPU corresponding to the specified rcu_data
* structure has spent some time in an extended quiescent state since
if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
return 0;
- ---- blkd = !!(rnp->qsmask & rdp->grpmask);
+ ++++ blkd = !!(READ_ONCE(rnp->qsmask) & rdp->grpmask);
trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
blkd ? TPS("cpuofl-bgp") : TPS("cpuofl"));
return 0;
struct rcu_node *rnp_old = NULL;
/* Funnel through hierarchy to reduce memory contention. */
- ---- rnp = __this_cpu_read(rcu_data.mynode);
+ ++++ rnp = raw_cpu_read(rcu_data.mynode);
for (; rnp != NULL; rnp = rnp->parent) {
ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
!raw_spin_trylock(&rnp->fqslock);
raw_spin_unlock_rcu_node(rnp);
}
-- ---/**
-- --- * call_rcu() - Queue an RCU callback for invocation after a grace period.
-- --- * @head: structure to be used for queueing the RCU updates.
-- --- * @func: actual callback function to be invoked after the grace period
-- --- *
-- --- * The callback function will be invoked some time after a full grace
-- --- * period elapses, in other words after all pre-existing RCU read-side
-- --- * critical sections have completed. However, the callback function
-- --- * might well execute concurrently with RCU read-side critical sections
-- --- * that started after call_rcu() was invoked.
-- --- *
-- --- * RCU read-side critical sections are delimited by rcu_read_lock()
-- --- * and rcu_read_unlock(), and may be nested. In addition, but only in
-- --- * v5.0 and later, regions of code across which interrupts, preemption,
-- --- * or softirqs have been disabled also serve as RCU read-side critical
-- --- * sections. This includes hardware interrupt handlers, softirq handlers,
-- --- * and NMI handlers.
-- --- *
-- --- * Note that all CPUs must agree that the grace period extended beyond
-- --- * all pre-existing RCU read-side critical section. On systems with more
-- --- * than one CPU, this means that when "func()" is invoked, each CPU is
-- --- * guaranteed to have executed a full memory barrier since the end of its
-- --- * last RCU read-side critical section whose beginning preceded the call
-- --- * to call_rcu(). It also means that each CPU executing an RCU read-side
-- --- * critical section that continues beyond the start of "func()" must have
-- --- * executed a memory barrier after the call_rcu() but before the beginning
-- --- * of that RCU read-side critical section. Note that these guarantees
-- --- * include CPUs that are offline, idle, or executing in user mode, as
-- --- * well as CPUs that are executing in the kernel.
-- --- *
-- --- * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
-- --- * resulting RCU callback function "func()", then both CPU A and CPU B are
-- --- * guaranteed to execute a full memory barrier during the time interval
-- --- * between the call to call_rcu() and the invocation of "func()" -- even
-- --- * if CPU A and CPU B are the same CPU (but again only if the system has
-- --- * more than one CPU).
-- --- *
-- --- * Implementation of these memory-ordering guarantees is described here:
-- --- * Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.
-- --- */
-- ---void call_rcu(struct rcu_head *head, rcu_callback_t func)
++ +++static void
++ +++__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy)
{
static atomic_t doublefrees;
unsigned long flags;
}
check_cb_ovld(rdp);
-- --- if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
++ +++ if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy))
return; // Enqueued onto ->nocb_bypass, so just leave.
// If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
rcu_segcblist_enqueue(&rdp->cblist, head);
local_irq_restore(flags);
}
}
-- ---EXPORT_SYMBOL_GPL(call_rcu);
++ +++#ifdef CONFIG_RCU_LAZY
++ +++/**
++ +++ * call_rcu_hurry() - Queue RCU callback for invocation after grace period, and
++ +++ * flush all lazy callbacks (including the new one) to the main ->cblist while
++ +++ * doing so.
++ +++ *
++ +++ * @head: structure to be used for queueing the RCU updates.
++ +++ * @func: actual callback function to be invoked after the grace period
++ +++ *
++ +++ * The callback function will be invoked some time after a full grace
++ +++ * period elapses, in other words after all pre-existing RCU read-side
++ +++ * critical sections have completed.
++ +++ *
++ +++ * Use this API instead of call_rcu() if you don't want the callback to be
++ +++ * invoked after very long periods of time, which can happen on systems without
++ +++ * memory pressure and on systems which are lightly loaded or mostly idle.
++ +++ * This function will cause callbacks to be invoked sooner than later at the
++ +++ * expense of extra power. Other than that, this function is identical to, and
++ +++ * reuses call_rcu()'s logic. Refer to call_rcu() for more details about memory
++ +++ * ordering and other functionality.
++ +++ */
++ +++void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func)
++ +++{
++ +++ return __call_rcu_common(head, func, false);
++ +++}
++ +++EXPORT_SYMBOL_GPL(call_rcu_hurry);
++ +++#endif
++ +++
++ +++/**
++ +++ * call_rcu() - Queue an RCU callback for invocation after a grace period.
++ +++ * By default the callbacks are 'lazy' and are kept hidden from the main
++ +++ * ->cblist to prevent starting of grace periods too soon.
++ +++ * If you desire grace periods to start very soon, use call_rcu_hurry().
++ +++ *
++ +++ * @head: structure to be used for queueing the RCU updates.
++ +++ * @func: actual callback function to be invoked after the grace period
++ +++ *
++ +++ * The callback function will be invoked some time after a full grace
++ +++ * period elapses, in other words after all pre-existing RCU read-side
++ +++ * critical sections have completed. However, the callback function
++ +++ * might well execute concurrently with RCU read-side critical sections
++ +++ * that started after call_rcu() was invoked.
++ +++ *
++ +++ * RCU read-side critical sections are delimited by rcu_read_lock()
++ +++ * and rcu_read_unlock(), and may be nested. In addition, but only in
++ +++ * v5.0 and later, regions of code across which interrupts, preemption,
++ +++ * or softirqs have been disabled also serve as RCU read-side critical
++ +++ * sections. This includes hardware interrupt handlers, softirq handlers,
++ +++ * and NMI handlers.
++ +++ *
++ +++ * Note that all CPUs must agree that the grace period extended beyond
++ +++ * all pre-existing RCU read-side critical section. On systems with more
++ +++ * than one CPU, this means that when "func()" is invoked, each CPU is
++ +++ * guaranteed to have executed a full memory barrier since the end of its
++ +++ * last RCU read-side critical section whose beginning preceded the call
++ +++ * to call_rcu(). It also means that each CPU executing an RCU read-side
++ +++ * critical section that continues beyond the start of "func()" must have
++ +++ * executed a memory barrier after the call_rcu() but before the beginning
++ +++ * of that RCU read-side critical section. Note that these guarantees
++ +++ * include CPUs that are offline, idle, or executing in user mode, as
++ +++ * well as CPUs that are executing in the kernel.
++ +++ *
++ +++ * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
++ +++ * resulting RCU callback function "func()", then both CPU A and CPU B are
++ +++ * guaranteed to execute a full memory barrier during the time interval
++ +++ * between the call to call_rcu() and the invocation of "func()" -- even
++ +++ * if CPU A and CPU B are the same CPU (but again only if the system has
++ +++ * more than one CPU).
++ +++ *
++ +++ * Implementation of these memory-ordering guarantees is described here:
++ +++ * Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.
++ +++ */
++ +++void call_rcu(struct rcu_head *head, rcu_callback_t func)
++ +++{
++ +++ return __call_rcu_common(head, func, IS_ENABLED(CONFIG_RCU_LAZY));
++ +++}
++ +++EXPORT_SYMBOL_GPL(call_rcu);
/* Maximum number of jiffies to wait before draining a batch. */
#define KFREE_DRAIN_JIFFIES (5 * HZ)
if (rcu_gp_is_expedited())
synchronize_rcu_expedited();
else
-- --- wait_rcu_gp(call_rcu);
++ +++ wait_rcu_gp(call_rcu_hurry);
return;
}
{
unsigned long gseq = READ_ONCE(rcu_state.barrier_sequence);
unsigned long lseq = READ_ONCE(rdp->barrier_seq_snap);
++ +++ bool wake_nocb = false;
++ +++ bool was_alldone = false;
lockdep_assert_held(&rcu_state.barrier_lock);
if (rcu_seq_state(lseq) || !rcu_seq_state(gseq) || rcu_seq_ctr(lseq) != rcu_seq_ctr(gseq))
rdp->barrier_head.func = rcu_barrier_callback;
debug_rcu_head_queue(&rdp->barrier_head);
rcu_nocb_lock(rdp);
-- --- WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
++ +++ /*
++ +++ * Flush bypass and wakeup rcuog if we add callbacks to an empty regular
++ +++ * queue. This way we don't wait for bypass timer that can reach seconds
++ +++ * if it's fully lazy.
++ +++ */
++ +++ was_alldone = rcu_rdp_is_offloaded(rdp) && !rcu_segcblist_pend_cbs(&rdp->cblist);
++ +++ WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, false));
++ +++ wake_nocb = was_alldone && rcu_segcblist_pend_cbs(&rdp->cblist);
if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
atomic_inc(&rcu_state.barrier_cpu_count);
} else {
rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence);
}
rcu_nocb_unlock(rdp);
++ +++ if (wake_nocb)
++ +++ wake_nocb_gp(rdp, false);
smp_store_release(&rdp->barrier_seq_snap, gseq);
}
// Do any dangling deferred wakeups.
do_nocb_deferred_wakeup(rdp);
- ---- /* QS for any half-done expedited grace period. */
- ---- rcu_report_exp_rdp(rdp);
rcu_preempt_deferred_qs(current);
/* Remove outgoing CPU from mask in the leaf rcu_node structure. */
my_rdp = this_cpu_ptr(&rcu_data);
my_rnp = my_rdp->mynode;
rcu_nocb_lock(my_rdp); /* irqs already disabled. */
-- --- WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
++ +++ WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies, false));
raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
/* Leverage recent GPs and set GP for new callbacks. */
needwake = rcu_advance_cbs(my_rnp, rdp) ||
projects / linux.git / commitdiff