Set maximum number of finished RCU callbacks to
process in one batch.
+++ ++++ rcutree.csd_lock_suppress_rcu_stall= [KNL]
+++ ++++ Do only a one-line RCU CPU stall warning when
+++ ++++ there is an ongoing too-long CSD-lock wait.
+++ ++++
rcutree.do_rcu_barrier= [KNL]
Request a call to rcu_barrier(). This is
throttled so that userspace tests can safely
Time to wait (s) after boot before inducing stall.
rcutorture.stall_cpu_irqsoff= [KNL]
-- ----- Disable interrupts while stalling if set.
++ +++++ Disable interrupts while stalling if set, but only
++ +++++ on the first stall in the set.
++ +++++
++ +++++ rcutorture.stall_cpu_repeat= [KNL]
++ +++++ Number of times to repeat the stall sequence,
++ +++++ so that rcutorture.stall_cpu_repeat=3 will result
++ +++++ in four stall sequences.
rcutorture.stall_gp_kthread= [KNL]
Duration (s) of forced sleep within RCU
of zero will disable batching. Batching is
always disabled for synchronize_rcu_tasks().
----- -- rcupdate.rcu_tasks_rude_lazy_ms= [KNL]
----- -- Set timeout in milliseconds RCU Tasks
----- -- Rude asynchronous callback batching for
----- -- call_rcu_tasks_rude(). A negative value
----- -- will take the default. A value of zero will
----- -- disable batching. Batching is always disabled
----- -- for synchronize_rcu_tasks_rude().
----- --
rcupdate.rcu_tasks_trace_lazy_ms= [KNL]
Set timeout in milliseconds RCU Tasks
Trace asynchronous callback batching for
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
++++ +++#define RCU_SEQ_CTR_SHIFT 2
++++ +++#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1)
++++ +++
/* Exported common interfaces */
void call_rcu(struct rcu_head *head, rcu_callback_t func);
void rcu_barrier_tasks(void);
----- --void rcu_barrier_tasks_rude(void);
void synchronize_rcu(void);
struct rcu_gp_oldstate;
int rcu_nocb_cpu_offload(int cpu);
int rcu_nocb_cpu_deoffload(int cpu);
void rcu_nocb_flush_deferred_wakeup(void);
+ ++++++
+ ++++++#define RCU_NOCB_LOCKDEP_WARN(c, s) RCU_LOCKDEP_WARN(c, s)
+ ++++++
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
+ ++++++
static inline void rcu_init_nohz(void) { }
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
+ ++++++
+ ++++++#define RCU_NOCB_LOCKDEP_WARN(c, s)
+ ++++++
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
/*
} while (0)
void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks(void);
+++++ ++void rcu_tasks_torture_stats_print(char *tt, char *tf);
# else
# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
# define call_rcu_tasks call_rcu
rcu_tasks_trace_qs_blkd(t); \
} \
} while (0)
+++++ ++void rcu_tasks_trace_torture_stats_print(char *tt, char *tf);
# else
# define rcu_tasks_trace_qs(t) do { } while (0)
# endif
} while (0)
# ifdef CONFIG_TASKS_RUDE_RCU
----- --void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks_rude(void);
+++++ ++void rcu_tasks_rude_torture_stats_print(char *tt, char *tf);
# endif
#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
* grace-period sequence number.
*/
---- ---#define RCU_SEQ_CTR_SHIFT 2
---- ---#define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1)
---- ---
/* Low-order bit definition for polled grace-period APIs. */
#define RCU_GET_STATE_COMPLETED 0x1
kmem_dump_obj(rhp);
}
+++++ ++static inline bool rcu_barrier_cb_is_done(struct rcu_head *rhp)
+++++ ++{
+++++ ++ return rhp->next == rhp;
+++++ ++}
+++++ ++
extern int rcu_cpu_stall_suppress_at_boot;
static inline bool rcu_stall_is_suppressed_at_boot(void)
#endif
#ifdef CONFIG_TINY_RCU
-------static inline bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) { return false; }
+++++++static inline bool rcu_watching_zero_in_eqs(int cpu, int *vp) { return false; }
static inline unsigned long rcu_get_gp_seq(void) { return 0; }
static inline unsigned long rcu_exp_batches_completed(void) { return 0; }
static inline unsigned long
static inline void rcu_gp_slow_register(atomic_t *rgssp) { }
static inline void rcu_gp_slow_unregister(atomic_t *rgssp) { }
#else /* #ifdef CONFIG_TINY_RCU */
-------bool rcu_dynticks_zero_in_eqs(int cpu, int *vp);
+++++++bool rcu_watching_zero_in_eqs(int cpu, int *vp);
unsigned long rcu_get_gp_seq(void);
unsigned long rcu_exp_batches_completed(void);
unsigned long srcu_batches_completed(struct srcu_struct *sp);
torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall.");
torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");
++ +++++torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one.");
torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s).");
torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
torture_param(int, stutter, 5, "Number of seconds to run/halt test");
bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2);
unsigned long (*get_gp_state)(void);
void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);
-- ----- unsigned long (*get_gp_completed)(void);
-- ----- void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);
unsigned long (*start_gp_poll)(void);
void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp);
bool (*poll_gp_state)(unsigned long oldstate);
bool (*poll_need_2gp)(bool poll, bool poll_full);
void (*cond_sync)(unsigned long oldstate);
void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp);
++ +++++ int poll_active;
++ +++++ int poll_active_full;
call_rcu_func_t call;
void (*cb_barrier)(void);
void (*fqs)(void);
.get_comp_state_full = get_completed_synchronize_rcu_full,
.get_gp_state = get_state_synchronize_rcu,
.get_gp_state_full = get_state_synchronize_rcu_full,
-- ----- .get_gp_completed = get_completed_synchronize_rcu,
-- ----- .get_gp_completed_full = get_completed_synchronize_rcu_full,
.start_gp_poll = start_poll_synchronize_rcu,
.start_gp_poll_full = start_poll_synchronize_rcu_full,
.poll_gp_state = poll_state_synchronize_rcu,
.poll_need_2gp = rcu_poll_need_2gp,
.cond_sync = cond_synchronize_rcu,
.cond_sync_full = cond_synchronize_rcu_full,
++ +++++ .poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE,
++ +++++ .poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE,
.get_gp_state_exp = get_state_synchronize_rcu,
.start_gp_poll_exp = start_poll_synchronize_rcu_expedited,
.start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full,
.deferred_free = srcu_torture_deferred_free,
.sync = srcu_torture_synchronize,
.exp_sync = srcu_torture_synchronize_expedited,
++ +++++ .same_gp_state = same_state_synchronize_srcu,
++ +++++ .get_comp_state = get_completed_synchronize_srcu,
.get_gp_state = srcu_torture_get_gp_state,
.start_gp_poll = srcu_torture_start_gp_poll,
.poll_gp_state = srcu_torture_poll_gp_state,
++ +++++ .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE,
.call = srcu_torture_call,
.cb_barrier = srcu_torture_barrier,
.stats = srcu_torture_stats,
.deferred_free = srcu_torture_deferred_free,
.sync = srcu_torture_synchronize,
.exp_sync = srcu_torture_synchronize_expedited,
++ +++++ .same_gp_state = same_state_synchronize_srcu,
++ +++++ .get_comp_state = get_completed_synchronize_srcu,
.get_gp_state = srcu_torture_get_gp_state,
.start_gp_poll = srcu_torture_start_gp_poll,
.poll_gp_state = srcu_torture_poll_gp_state,
++ +++++ .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE,
.call = srcu_torture_call,
.cb_barrier = srcu_torture_barrier,
.stats = srcu_torture_stats,
* Definitions for rude RCU-tasks torture testing.
*/
----- --static void rcu_tasks_rude_torture_deferred_free(struct rcu_torture *p)
----- --{
----- -- call_rcu_tasks_rude(&p->rtort_rcu, rcu_torture_cb);
----- --}
----- --
static struct rcu_torture_ops tasks_rude_ops = {
.ttype = RCU_TASKS_RUDE_FLAVOR,
.init = rcu_sync_torture_init,
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
.readunlock = rcu_torture_read_unlock_trivial,
.get_gp_seq = rcu_no_completed,
----- -- .deferred_free = rcu_tasks_rude_torture_deferred_free,
.sync = synchronize_rcu_tasks_rude,
.exp_sync = synchronize_rcu_tasks_rude,
----- -- .call = call_rcu_tasks_rude,
----- -- .cb_barrier = rcu_barrier_tasks_rude,
.gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread,
.get_gp_data = rcu_tasks_rude_get_gp_data,
.cbflood_max = 50000,
} else if (gp_sync && !cur_ops->sync) {
pr_alert("%s: gp_sync without primitives.\n", __func__);
}
++ +++++ pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes);
}
/*
int i;
int idx;
int oldnice = task_nice(current);
-- ----- struct rcu_gp_oldstate rgo[NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE];
++ +++++ struct rcu_gp_oldstate *rgo = NULL;
++ +++++ int rgo_size = 0;
struct rcu_torture *rp;
struct rcu_torture *old_rp;
static DEFINE_TORTURE_RANDOM(rand);
unsigned long stallsdone = jiffies;
bool stutter_waited;
-- ----- unsigned long ulo[NUM_ACTIVE_RCU_POLL_OLDSTATE];
++ +++++ unsigned long *ulo = NULL;
++ +++++ int ulo_size = 0;
// If a new stall test is added, this must be adjusted.
if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu)
-- ----- stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) * HZ;
++ +++++ stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) *
++ +++++ HZ * (stall_cpu_repeat + 1);
VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
if (!can_expedite)
pr_alert("%s" TORTURE_FLAG
torture_kthread_stopping("rcu_torture_writer");
return 0;
}
++ +++++ if (cur_ops->poll_active > 0) {
++ +++++ ulo = kzalloc(cur_ops->poll_active * sizeof(ulo[0]), GFP_KERNEL);
++ +++++ if (!WARN_ON(!ulo))
++ +++++ ulo_size = cur_ops->poll_active;
++ +++++ }
++ +++++ if (cur_ops->poll_active_full > 0) {
++ +++++ rgo = kzalloc(cur_ops->poll_active_full * sizeof(rgo[0]), GFP_KERNEL);
++ +++++ if (!WARN_ON(!rgo))
++ +++++ rgo_size = cur_ops->poll_active_full;
++ +++++ }
do {
rcu_torture_writer_state = RTWS_FIXED_DELAY;
rcu_torture_writer_state_getname(),
rcu_torture_writer_state,
cookie, cur_ops->get_gp_state());
-- ----- if (cur_ops->get_gp_completed) {
-- ----- cookie = cur_ops->get_gp_completed();
++ +++++ if (cur_ops->get_comp_state) {
++ +++++ cookie = cur_ops->get_comp_state();
WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
}
cur_ops->readunlock(idx);
rcu_torture_writer_state_getname(),
rcu_torture_writer_state,
cpumask_pr_args(cpu_online_mask));
-- ----- if (cur_ops->get_gp_completed_full) {
-- ----- cur_ops->get_gp_completed_full(&cookie_full);
++ +++++ if (cur_ops->get_comp_state_full) {
++ +++++ cur_ops->get_comp_state_full(&cookie_full);
WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));
}
cur_ops->readunlock(idx);
break;
case RTWS_POLL_GET:
rcu_torture_writer_state = RTWS_POLL_GET;
-- ----- for (i = 0; i < ARRAY_SIZE(ulo); i++)
++ +++++ for (i = 0; i < ulo_size; i++)
ulo[i] = cur_ops->get_comp_state();
gp_snap = cur_ops->start_gp_poll();
rcu_torture_writer_state = RTWS_POLL_WAIT;
while (!cur_ops->poll_gp_state(gp_snap)) {
gp_snap1 = cur_ops->get_gp_state();
-- ----- for (i = 0; i < ARRAY_SIZE(ulo); i++)
++ +++++ for (i = 0; i < ulo_size; i++)
if (cur_ops->poll_gp_state(ulo[i]) ||
cur_ops->same_gp_state(ulo[i], gp_snap1)) {
ulo[i] = gp_snap1;
break;
}
-- ----- WARN_ON_ONCE(i >= ARRAY_SIZE(ulo));
++ +++++ WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size);
torture_hrtimeout_jiffies(torture_random(&rand) % 16,
&rand);
}
break;
case RTWS_POLL_GET_FULL:
rcu_torture_writer_state = RTWS_POLL_GET_FULL;
-- ----- for (i = 0; i < ARRAY_SIZE(rgo); i++)
++ +++++ for (i = 0; i < rgo_size; i++)
cur_ops->get_comp_state_full(&rgo[i]);
cur_ops->start_gp_poll_full(&gp_snap_full);
rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
cur_ops->get_gp_state_full(&gp_snap1_full);
-- ----- for (i = 0; i < ARRAY_SIZE(rgo); i++)
++ +++++ for (i = 0; i < rgo_size; i++)
if (cur_ops->poll_gp_state_full(&rgo[i]) ||
cur_ops->same_gp_state_full(&rgo[i],
&gp_snap1_full)) {
rgo[i] = gp_snap1_full;
break;
}
-- ----- WARN_ON_ONCE(i >= ARRAY_SIZE(rgo));
++ +++++ WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size);
torture_hrtimeout_jiffies(torture_random(&rand) % 16,
&rand);
}
pr_alert("%s" TORTURE_FLAG
" Dynamic grace-period expediting was disabled.\n",
torture_type);
++ +++++ kfree(ulo);
++ +++++ kfree(rgo);
rcu_torture_writer_state = RTWS_STOPPING;
torture_kthread_stopping("rcu_torture_writer");
return 0;
"test_boost=%d/%d test_boost_interval=%d "
"test_boost_duration=%d shutdown_secs=%d "
"stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "
-- ----- "stall_cpu_block=%d "
++ +++++ "stall_cpu_block=%d stall_cpu_repeat=%d "
"n_barrier_cbs=%d "
"onoff_interval=%d onoff_holdoff=%d "
"read_exit_delay=%d read_exit_burst=%d "
test_boost, cur_ops->can_boost,
test_boost_interval, test_boost_duration, shutdown_secs,
stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,
-- ----- stall_cpu_block,
++ +++++ stall_cpu_block, stall_cpu_repeat,
n_barrier_cbs,
onoff_interval, onoff_holdoff,
read_exit_delay, read_exit_burst,
* induces a CPU stall for the time specified by stall_cpu. If a new
* stall test is added, stallsdone in rcu_torture_writer() must be adjusted.
*/
-- -----static int rcu_torture_stall(void *args)
++ +++++static void rcu_torture_stall_one(int rep, int irqsoff)
{
int idx;
-- ----- int ret;
unsigned long stop_at;
-- ----- VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
-- ----- if (rcu_cpu_stall_notifiers) {
-- ----- ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block);
-- ----- if (ret)
-- ----- pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n",
-- ----- __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : "");
-- ----- }
if (stall_cpu_holdoff > 0) {
VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
stop_at = ktime_get_seconds() + stall_cpu;
/* RCU CPU stall is expected behavior in following code. */
idx = cur_ops->readlock();
-- ----- if (stall_cpu_irqsoff)
++ +++++ if (irqsoff)
local_irq_disable();
else if (!stall_cpu_block)
preempt_disable();
-- ----- pr_alert("%s start on CPU %d.\n",
-- ----- __func__, raw_smp_processor_id());
++ +++++ pr_alert("%s start stall episode %d on CPU %d.\n",
++ +++++ __func__, rep + 1, raw_smp_processor_id());
while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) &&
!kthread_should_stop())
if (stall_cpu_block) {
} else if (stall_no_softlockup) {
touch_softlockup_watchdog();
}
-- ----- if (stall_cpu_irqsoff)
++ +++++ if (irqsoff)
local_irq_enable();
else if (!stall_cpu_block)
preempt_enable();
cur_ops->readunlock(idx);
}
++ +++++}
++ +++++
++ +++++/*
++ +++++ * CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many
++ +++++ * additional times as specified by the stall_cpu_repeat module parameter.
++ +++++ * Note that stall_cpu_irqsoff is ignored on the second and subsequent
++ +++++ * stall.
++ +++++ */
++ +++++static int rcu_torture_stall(void *args)
++ +++++{
++ +++++ int i;
++ +++++ int repeat = stall_cpu_repeat;
++ +++++ int ret;
++ +++++
++ +++++ VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
++ +++++ if (repeat < 0) {
++ +++++ repeat = 0;
++ +++++ WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST));
++ +++++ }
++ +++++ if (rcu_cpu_stall_notifiers) {
++ +++++ ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block);
++ +++++ if (ret)
++ +++++ pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n",
++ +++++ __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : "");
++ +++++ }
++ +++++ for (i = 0; i <= repeat; i++) {
++ +++++ if (kthread_should_stop())
++ +++++ break;
++ +++++ rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0);
++ +++++ }
pr_alert("%s end.\n", __func__);
if (rcu_cpu_stall_notifiers && !ret) {
ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block);
rcu_torture_fwd_prog_cond_resched(freed);
if (tick_nohz_full_enabled()) {
local_irq_save(flags);
------- rcu_momentary_dyntick_idle();
+++++++ rcu_momentary_eqs();
local_irq_restore(flags);
}
}
rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
if (tick_nohz_full_enabled()) {
local_irq_save(flags);
------- rcu_momentary_dyntick_idle();
+++++++ rcu_momentary_eqs();
local_irq_restore(flags);
}
}
* @rtp_blkd_tasks: List of tasks blocked as readers.
* @rtp_exit_list: List of tasks in the latter portion of do_exit().
* @cpu: CPU number corresponding to this entry.
+++++ ++ * @index: Index of this CPU in rtpcp_array of the rcu_tasks structure.
* @rtpp: Pointer to the rcu_tasks structure.
*/
struct rcu_tasks_percpu {
struct list_head rtp_blkd_tasks;
struct list_head rtp_exit_list;
int cpu;
+++++ ++ int index;
struct rcu_tasks *rtpp;
};
* @init_fract: Initial backoff sleep interval.
* @gp_jiffies: Time of last @gp_state transition.
* @gp_start: Most recent grace-period start in jiffies.
----- -- * @tasks_gp_seq: Number of grace periods completed since boot.
+++++ ++ * @tasks_gp_seq: Number of grace periods completed since boot in upper bits.
* @n_ipis: Number of IPIs sent to encourage grace periods to end.
* @n_ipis_fails: Number of IPI-send failures.
* @kthread_ptr: This flavor's grace-period/callback-invocation kthread.
* @call_func: This flavor's call_rcu()-equivalent function.
* @wait_state: Task state for synchronous grace-period waits (default TASK_UNINTERRUPTIBLE).
* @rtpcpu: This flavor's rcu_tasks_percpu structure.
+++++ ++ * @rtpcp_array: Array of pointers to rcu_tasks_percpu structure of CPUs in cpu_possible_mask.
* @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks.
* @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing.
* @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing.
* @barrier_q_count: Number of queues being waited on.
* @barrier_q_completion: Barrier wait/wakeup mechanism.
* @barrier_q_seq: Sequence number for barrier operations.
+++++ ++ * @barrier_q_start: Most recent barrier start in jiffies.
* @name: This flavor's textual name.
* @kname: This flavor's kthread name.
*/
call_rcu_func_t call_func;
unsigned int wait_state;
struct rcu_tasks_percpu __percpu *rtpcpu;
+++++ ++ struct rcu_tasks_percpu **rtpcp_array;
int percpu_enqueue_shift;
int percpu_enqueue_lim;
int percpu_dequeue_lim;
atomic_t barrier_q_count;
struct completion barrier_q_completion;
unsigned long barrier_q_seq;
+++++ ++ unsigned long barrier_q_start;
char *name;
char *kname;
};
static int rcu_task_lazy_lim __read_mostly = 32;
module_param(rcu_task_lazy_lim, int, 0444);
+++++ ++static int rcu_task_cpu_ids;
+++++ ++
/* RCU tasks grace-period state for debugging. */
#define RTGS_INIT 0
#define RTGS_WAIT_WAIT_CBS 1
int cpu;
int lim;
int shift;
+++++ ++ int maxcpu;
+++++ ++ int index = 0;
if (rcu_task_enqueue_lim < 0) {
rcu_task_enqueue_lim = 1;
}
lim = rcu_task_enqueue_lim;
----- -- if (lim > nr_cpu_ids)
----- -- lim = nr_cpu_ids;
----- -- shift = ilog2(nr_cpu_ids / lim);
----- -- if (((nr_cpu_ids - 1) >> shift) >= lim)
----- -- shift++;
----- -- WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
----- -- WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
----- -- smp_store_release(&rtp->percpu_enqueue_lim, lim);
+++++ ++ rtp->rtpcp_array = kcalloc(num_possible_cpus(), sizeof(struct rcu_tasks_percpu *), GFP_KERNEL);
+++++ ++ BUG_ON(!rtp->rtpcp_array);
+++++ ++
for_each_possible_cpu(cpu) {
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);
rtpcp->cpu = cpu;
rtpcp->rtpp = rtp;
+++++ ++ rtpcp->index = index;
+++++ ++ rtp->rtpcp_array[index] = rtpcp;
+++++ ++ index++;
if (!rtpcp->rtp_blkd_tasks.next)
INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);
if (!rtpcp->rtp_exit_list.next)
INIT_LIST_HEAD(&rtpcp->rtp_exit_list);
+++++ ++ rtpcp->barrier_q_head.next = &rtpcp->barrier_q_head;
+++++ ++ maxcpu = cpu;
}
----- -- pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d.\n", rtp->name,
----- -- data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim), rcu_task_cb_adjust);
+++++ ++ rcu_task_cpu_ids = maxcpu + 1;
+++++ ++ if (lim > rcu_task_cpu_ids)
+++++ ++ lim = rcu_task_cpu_ids;
+++++ ++ shift = ilog2(rcu_task_cpu_ids / lim);
+++++ ++ if (((rcu_task_cpu_ids - 1) >> shift) >= lim)
+++++ ++ shift++;
+++++ ++ WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
+++++ ++ WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
+++++ ++ smp_store_release(&rtp->percpu_enqueue_lim, lim);
+++++ ++
+++++ ++ pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d rcu_task_cpu_ids=%d.\n",
+++++ ++ rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim),
+++++ ++ rcu_task_cb_adjust, rcu_task_cpu_ids);
}
// Compute wakeup time for lazy callback timer.
rcu_read_lock();
ideal_cpu = smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift);
chosen_cpu = cpumask_next(ideal_cpu - 1, cpu_possible_mask);
+++++ ++ WARN_ON_ONCE(chosen_cpu >= rcu_task_cpu_ids);
rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu);
if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled.
raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
rtpcp->rtp_n_lock_retries = 0;
}
if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim &&
----- -- READ_ONCE(rtp->percpu_enqueue_lim) != nr_cpu_ids)
+++++ ++ READ_ONCE(rtp->percpu_enqueue_lim) != rcu_task_cpu_ids)
needadjust = true; // Defer adjustment to avoid deadlock.
}
// Queuing callbacks before initialization not yet supported.
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
if (unlikely(needadjust)) {
raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
----- -- if (rtp->percpu_enqueue_lim != nr_cpu_ids) {
+++++ ++ if (rtp->percpu_enqueue_lim != rcu_task_cpu_ids) {
WRITE_ONCE(rtp->percpu_enqueue_shift, 0);
----- -- WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);
----- -- smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);
+++++ ++ WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids);
+++++ ++ smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids);
pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
}
raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
struct rcu_tasks *rtp;
struct rcu_tasks_percpu *rtpcp;
+++++ ++ rhp->next = rhp; // Mark the callback as having been invoked.
rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head);
rtp = rtpcp->rtpp;
if (atomic_dec_and_test(&rtp->barrier_q_count))
// Wait for all in-flight callbacks for the specified RCU Tasks flavor.
// Operates in a manner similar to rcu_barrier().
----- --static void rcu_barrier_tasks_generic(struct rcu_tasks *rtp)
+++++ ++static void __maybe_unused rcu_barrier_tasks_generic(struct rcu_tasks *rtp)
{
int cpu;
unsigned long flags;
mutex_unlock(&rtp->barrier_q_mutex);
return;
}
+++++ ++ rtp->barrier_q_start = jiffies;
rcu_seq_start(&rtp->barrier_q_seq);
init_completion(&rtp->barrier_q_completion);
atomic_set(&rtp->barrier_q_count, 2);
dequeue_limit = smp_load_acquire(&rtp->percpu_dequeue_lim);
for (cpu = 0; cpu < dequeue_limit; cpu++) {
+++++ ++ if (!cpu_possible(cpu))
+++++ ++ continue;
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
/* Advance and accelerate any new callbacks. */
if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
if (rtp->percpu_enqueue_lim > 1) {
----- -- WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(nr_cpu_ids));
+++++ ++ WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(rcu_task_cpu_ids));
smp_store_release(&rtp->percpu_enqueue_lim, 1);
rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
gpdone = false;
pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name);
}
if (rtp->percpu_dequeue_lim == 1) {
----- -- for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) {
+++++ ++ for (cpu = rtp->percpu_dequeue_lim; cpu < rcu_task_cpu_ids; cpu++) {
+++++ ++ if (!cpu_possible(cpu))
+++++ ++ continue;
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist));
// Advance callbacks and invoke any that are ready.
static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp)
{
----- -- int cpu;
----- -- int cpunext;
int cpuwq;
unsigned long flags;
int len;
+++++ ++ int index;
struct rcu_head *rhp;
struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
struct rcu_tasks_percpu *rtpcp_next;
----- -- cpu = rtpcp->cpu;
----- -- cpunext = cpu * 2 + 1;
----- -- if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
----- -- rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext);
----- -- cpuwq = rcu_cpu_beenfullyonline(cpunext) ? cpunext : WORK_CPU_UNBOUND;
----- -- queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
----- -- cpunext++;
----- -- if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
----- -- rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext);
----- -- cpuwq = rcu_cpu_beenfullyonline(cpunext) ? cpunext : WORK_CPU_UNBOUND;
+++++ ++ index = rtpcp->index * 2 + 1;
+++++ ++ if (index < num_possible_cpus()) {
+++++ ++ rtpcp_next = rtp->rtpcp_array[index];
+++++ ++ if (rtpcp_next->cpu < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
+++++ ++ cpuwq = rcu_cpu_beenfullyonline(rtpcp_next->cpu) ? rtpcp_next->cpu : WORK_CPU_UNBOUND;
queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
+++++ ++ index++;
+++++ ++ if (index < num_possible_cpus()) {
+++++ ++ rtpcp_next = rtp->rtpcp_array[index];
+++++ ++ if (rtpcp_next->cpu < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
+++++ ++ cpuwq = rcu_cpu_beenfullyonline(rtpcp_next->cpu) ? rtpcp_next->cpu : WORK_CPU_UNBOUND;
+++++ ++ queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
+++++ ++ }
+++++ ++ }
}
}
----- -- if (rcu_segcblist_empty(&rtpcp->cblist) || !cpu_possible(cpu))
+++++ ++ if (rcu_segcblist_empty(&rtpcp->cblist))
return;
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
}
----- --#endif /* #ifndef CONFIG_TINY_RCU */
----- --#ifndef CONFIG_TINY_RCU
/* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */
static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s)
{
rtp->lazy_jiffies,
s);
}
+++++ ++
+++++ ++/* Dump out more rcutorture-relevant state common to all RCU-tasks flavors. */
+++++ ++static void rcu_tasks_torture_stats_print_generic(struct rcu_tasks *rtp, char *tt,
+++++ ++ char *tf, char *tst)
+++++ ++{
+++++ ++ cpumask_var_t cm;
+++++ ++ int cpu;
+++++ ++ bool gotcb = false;
+++++ ++ unsigned long j = jiffies;
+++++ ++
+++++ ++ pr_alert("%s%s Tasks%s RCU g%ld gp_start %lu gp_jiffies %lu gp_state %d (%s).\n",
+++++ ++ tt, tf, tst, data_race(rtp->tasks_gp_seq),
+++++ ++ j - data_race(rtp->gp_start), j - data_race(rtp->gp_jiffies),
+++++ ++ data_race(rtp->gp_state), tasks_gp_state_getname(rtp));
+++++ ++ pr_alert("\tEnqueue shift %d limit %d Dequeue limit %d gpseq %lu.\n",
+++++ ++ data_race(rtp->percpu_enqueue_shift),
+++++ ++ data_race(rtp->percpu_enqueue_lim),
+++++ ++ data_race(rtp->percpu_dequeue_lim),
+++++ ++ data_race(rtp->percpu_dequeue_gpseq));
+++++ ++ (void)zalloc_cpumask_var(&cm, GFP_KERNEL);
+++++ ++ pr_alert("\tCallback counts:");
+++++ ++ for_each_possible_cpu(cpu) {
+++++ ++ long n;
+++++ ++ struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
+++++ ++
+++++ ++ if (cpumask_available(cm) && !rcu_barrier_cb_is_done(&rtpcp->barrier_q_head))
+++++ ++ cpumask_set_cpu(cpu, cm);
+++++ ++ n = rcu_segcblist_n_cbs(&rtpcp->cblist);
+++++ ++ if (!n)
+++++ ++ continue;
+++++ ++ pr_cont(" %d:%ld", cpu, n);
+++++ ++ gotcb = true;
+++++ ++ }
+++++ ++ if (gotcb)
+++++ ++ pr_cont(".\n");
+++++ ++ else
+++++ ++ pr_cont(" (none).\n");
+++++ ++ pr_alert("\tBarrier seq %lu start %lu count %d holdout CPUs ",
+++++ ++ data_race(rtp->barrier_q_seq), j - data_race(rtp->barrier_q_start),
+++++ ++ atomic_read(&rtp->barrier_q_count));
+++++ ++ if (cpumask_available(cm) && !cpumask_empty(cm))
+++++ ++ pr_cont(" %*pbl.\n", cpumask_pr_args(cm));
+++++ ++ else
+++++ ++ pr_cont("(none).\n");
+++++ ++ free_cpumask_var(cm);
+++++ ++}
+++++ ++
#endif // #ifndef CONFIG_TINY_RCU
static void exit_tasks_rcu_finish_trace(struct task_struct *t);
show_rcu_tasks_generic_gp_kthread(&rcu_tasks, "");
}
EXPORT_SYMBOL_GPL(show_rcu_tasks_classic_gp_kthread);
+++++ ++
+++++ ++void rcu_tasks_torture_stats_print(char *tt, char *tf)
+++++ ++{
+++++ ++ rcu_tasks_torture_stats_print_generic(&rcu_tasks, tt, tf, "");
+++++ ++}
+++++ ++EXPORT_SYMBOL_GPL(rcu_tasks_torture_stats_print);
#endif // !defined(CONFIG_TINY_RCU)
struct task_struct *get_rcu_tasks_gp_kthread(void)
////////////////////////////////////////////////////////////////////////
//
----- --// "Rude" variant of Tasks RCU, inspired by Steve Rostedt's trick of
----- --// passing an empty function to schedule_on_each_cpu(). This approach
----- --// provides an asynchronous call_rcu_tasks_rude() API and batching of
----- --// concurrent calls to the synchronous synchronize_rcu_tasks_rude() API.
----- --// This invokes schedule_on_each_cpu() in order to send IPIs far and wide
----- --// and induces otherwise unnecessary context switches on all online CPUs,
----- --// whether idle or not.
+++++ ++// "Rude" variant of Tasks RCU, inspired by Steve Rostedt's
+++++ ++// trick of passing an empty function to schedule_on_each_cpu().
+++++ ++// This approach provides batching of concurrent calls to the synchronous
+++++ ++// synchronize_rcu_tasks_rude() API. This invokes schedule_on_each_cpu()
+++++ ++// in order to send IPIs far and wide and induces otherwise unnecessary
+++++ ++// context switches on all online CPUs, whether idle or not.
//
// Callback handling is provided by the rcu_tasks_kthread() function.
//
schedule_on_each_cpu(rcu_tasks_be_rude);
}
----- --void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func);
+++++ ++static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func);
DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude,
"RCU Tasks Rude");
----- --/**
+++++ ++/*
* call_rcu_tasks_rude() - Queue a callback rude task-based grace period
* @rhp: structure to be used for queueing the RCU updates.
* @func: actual callback function to be invoked after the grace period
*
* See the description of call_rcu() for more detailed information on
* memory ordering guarantees.
+++++ ++ *
+++++ ++ * This is no longer exported, and is instead reserved for use by
+++++ ++ * synchronize_rcu_tasks_rude().
*/
----- --void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func)
+++++ ++static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func)
{
call_rcu_tasks_generic(rhp, func, &rcu_tasks_rude);
}
----- --EXPORT_SYMBOL_GPL(call_rcu_tasks_rude);
/**
* synchronize_rcu_tasks_rude - wait for a rude rcu-tasks grace period
}
EXPORT_SYMBOL_GPL(synchronize_rcu_tasks_rude);
----- --/**
----- -- * rcu_barrier_tasks_rude - Wait for in-flight call_rcu_tasks_rude() callbacks.
----- -- *
----- -- * Although the current implementation is guaranteed to wait, it is not
----- -- * obligated to, for example, if there are no pending callbacks.
----- -- */
----- --void rcu_barrier_tasks_rude(void)
----- --{
----- -- rcu_barrier_tasks_generic(&rcu_tasks_rude);
----- --}
----- --EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude);
----- --
----- --int rcu_tasks_rude_lazy_ms = -1;
----- --module_param(rcu_tasks_rude_lazy_ms, int, 0444);
----- --
static int __init rcu_spawn_tasks_rude_kthread(void)
{
rcu_tasks_rude.gp_sleep = HZ / 10;
----- -- if (rcu_tasks_rude_lazy_ms >= 0)
----- -- rcu_tasks_rude.lazy_jiffies = msecs_to_jiffies(rcu_tasks_rude_lazy_ms);
rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude);
return 0;
}
show_rcu_tasks_generic_gp_kthread(&rcu_tasks_rude, "");
}
EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread);
+++++ ++
+++++ ++void rcu_tasks_rude_torture_stats_print(char *tt, char *tf)
+++++ ++{
+++++ ++ rcu_tasks_torture_stats_print_generic(&rcu_tasks_rude, tt, tf, "");
+++++ ++}
+++++ ++EXPORT_SYMBOL_GPL(rcu_tasks_rude_torture_stats_print);
#endif // !defined(CONFIG_TINY_RCU)
struct task_struct *get_rcu_tasks_rude_gp_kthread(void)
// However, we cannot safely change its state.
n_heavy_reader_attempts++;
// Check for "running" idle tasks on offline CPUs.
------- if (!rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting))
+++++++ if (!rcu_watching_zero_in_eqs(cpu, &t->trc_reader_nesting))
return -EINVAL; // No quiescent state, do it the hard way.
n_heavy_reader_updates++;
nesting = 0;
show_rcu_tasks_generic_gp_kthread(&rcu_tasks_trace, buf);
}
EXPORT_SYMBOL_GPL(show_rcu_tasks_trace_gp_kthread);
+++++ ++
+++++ ++void rcu_tasks_trace_torture_stats_print(char *tt, char *tf)
+++++ ++{
+++++ ++ rcu_tasks_torture_stats_print_generic(&rcu_tasks_trace, tt, tf, "");
+++++ ++}
+++++ ++EXPORT_SYMBOL_GPL(rcu_tasks_trace_torture_stats_print);
#endif // !defined(CONFIG_TINY_RCU)
struct task_struct *get_rcu_tasks_trace_gp_kthread(void)
/* If not defined, the test is skipped. */
.notrun = IS_ENABLED(CONFIG_TASKS_RCU),
},
----- -- {
----- -- .name = "call_rcu_tasks_rude()",
----- -- /* If not defined, the test is skipped. */
----- -- .notrun = IS_ENABLED(CONFIG_TASKS_RUDE_RCU),
----- -- },
{
.name = "call_rcu_tasks_trace()",
/* If not defined, the test is skipped. */
}
};
+++++ ++#if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU)
static void test_rcu_tasks_callback(struct rcu_head *rhp)
{
struct rcu_tasks_test_desc *rttd =
rttd->notrun = false;
}
+++++ ++#endif // #if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU)
static void rcu_tasks_initiate_self_tests(void)
{
#ifdef CONFIG_TASKS_RUDE_RCU
pr_info("Running RCU Tasks Rude wait API self tests\n");
----- -- tests[1].runstart = jiffies;
synchronize_rcu_tasks_rude();
----- -- call_rcu_tasks_rude(&tests[1].rh, test_rcu_tasks_callback);
#endif
#ifdef CONFIG_TASKS_TRACE_RCU
pr_info("Running RCU Tasks Trace wait API self tests\n");
----- -- tests[2].runstart = jiffies;
+++++ ++ tests[1].runstart = jiffies;
synchronize_rcu_tasks_trace();
----- -- call_rcu_tasks_trace(&tests[2].rh, test_rcu_tasks_callback);
+++++ ++ call_rcu_tasks_trace(&tests[1].rh, test_rcu_tasks_callback);
#endif
}
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
.gpwrap = true,
- ------#ifdef CONFIG_RCU_NOCB_CPU
- ------ .cblist.flags = SEGCBLIST_RCU_CORE,
- ------#endif
};
static struct rcu_state rcu_state = {
.level = { &rcu_state.node[0] },
.srs_cleanup_work = __WORK_INITIALIZER(rcu_state.srs_cleanup_work,
rcu_sr_normal_gp_cleanup_work),
.srs_cleanups_pending = ATOMIC_INIT(0),
+ ++++++#ifdef CONFIG_RCU_NOCB_CPU
+ ++++++ .nocb_mutex = __MUTEX_INITIALIZER(rcu_state.nocb_mutex),
+ ++++++#endif
};
/* Dump rcu_node combining tree at boot to verify correct setup. */
}
/*
------- * Reset the current CPU's ->dynticks counter to indicate that the
+++++++ * Reset the current CPU's RCU_WATCHING counter to indicate that the
* newly onlined CPU is no longer in an extended quiescent state.
* This will either leave the counter unchanged, or increment it
* to the next non-quiescent value.
*
* The non-atomic test/increment sequence works because the upper bits
------- * of the ->dynticks counter are manipulated only by the corresponding CPU,
+++++++ * of the ->state variable are manipulated only by the corresponding CPU,
* or when the corresponding CPU is offline.
*/
-------static void rcu_dynticks_eqs_online(void)
+++++++static void rcu_watching_online(void)
{
------- if (ct_dynticks() & RCU_DYNTICKS_IDX)
+++++++ if (ct_rcu_watching() & CT_RCU_WATCHING)
return;
------- ct_state_inc(RCU_DYNTICKS_IDX);
+++++++ ct_state_inc(CT_RCU_WATCHING);
}
/*
------- * Return true if the snapshot returned from rcu_dynticks_snap()
+++++++ * Return true if the snapshot returned from ct_rcu_watching()
* indicates that RCU is in an extended quiescent state.
*/
-------static bool rcu_dynticks_in_eqs(int snap)
+++++++static bool rcu_watching_snap_in_eqs(int snap)
{
------- return !(snap & RCU_DYNTICKS_IDX);
+++++++ return !(snap & CT_RCU_WATCHING);
}
-------/*
------- * Return true if the CPU corresponding to the specified rcu_data
------- * structure has spent some time in an extended quiescent state since
------- * rcu_dynticks_snap() returned the specified snapshot.
+++++++/**
+++++++ * rcu_watching_snap_stopped_since() - Has RCU stopped watching a given CPU
+++++++ * since the specified @snap?
+++++++ *
+++++++ * @rdp: The rcu_data corresponding to the CPU for which to check EQS.
+++++++ * @snap: rcu_watching snapshot taken when the CPU wasn't in an EQS.
+++++++ *
+++++++ * Returns true if the CPU corresponding to @rdp has spent some time in an
+++++++ * extended quiescent state since @snap. Note that this doesn't check if it
+++++++ * /still/ is in an EQS, just that it went through one since @snap.
+++++++ *
+++++++ * This is meant to be used in a loop waiting for a CPU to go through an EQS.
*/
-------static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
+++++++static bool rcu_watching_snap_stopped_since(struct rcu_data *rdp, int snap)
{
/*
* The first failing snapshot is already ordered against the accesses
* performed by the remote CPU prior to entering idle and therefore can
* rely solely on acquire semantics.
*/
------- return snap != ct_dynticks_cpu_acquire(rdp->cpu);
+++++++ if (WARN_ON_ONCE(rcu_watching_snap_in_eqs(snap)))
+++++++ return true;
+++++++
+++++++ return snap != ct_rcu_watching_cpu_acquire(rdp->cpu);
}
/*
* Return true if the referenced integer is zero while the specified
* CPU remains within a single extended quiescent state.
*/
-------bool rcu_dynticks_zero_in_eqs(int cpu, int *vp)
+++++++bool rcu_watching_zero_in_eqs(int cpu, int *vp)
{
int snap;
// If not quiescent, force back to earlier extended quiescent state.
------- snap = ct_dynticks_cpu(cpu) & ~RCU_DYNTICKS_IDX;
------- smp_rmb(); // Order ->dynticks and *vp reads.
+++++++ snap = ct_rcu_watching_cpu(cpu) & ~CT_RCU_WATCHING;
+++++++ smp_rmb(); // Order CT state and *vp reads.
if (READ_ONCE(*vp))
return false; // Non-zero, so report failure;
------- smp_rmb(); // Order *vp read and ->dynticks re-read.
+++++++ smp_rmb(); // Order *vp read and CT state re-read.
// If still in the same extended quiescent state, we are good!
------- return snap == ct_dynticks_cpu(cpu);
+++++++ return snap == ct_rcu_watching_cpu(cpu);
}
/*
*
* The caller must have disabled interrupts and must not be idle.
*/
-------notrace void rcu_momentary_dyntick_idle(void)
+++++++notrace void rcu_momentary_eqs(void)
{
int seq;
raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
------- seq = ct_state_inc(2 * RCU_DYNTICKS_IDX);
+++++++ seq = ct_state_inc(2 * CT_RCU_WATCHING);
/* It is illegal to call this from idle state. */
------- WARN_ON_ONCE(!(seq & RCU_DYNTICKS_IDX));
+++++++ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING));
rcu_preempt_deferred_qs(current);
}
-------EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
+++++++EXPORT_SYMBOL_GPL(rcu_momentary_eqs);
/**
* rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle
lockdep_assert_irqs_disabled();
/* Check for counter underflows */
------- RCU_LOCKDEP_WARN(ct_dynticks_nesting() < 0,
------- "RCU dynticks_nesting counter underflow!");
------- RCU_LOCKDEP_WARN(ct_dynticks_nmi_nesting() <= 0,
------- "RCU dynticks_nmi_nesting counter underflow/zero!");
+++++++ RCU_LOCKDEP_WARN(ct_nesting() < 0,
+++++++ "RCU nesting counter underflow!");
+++++++ RCU_LOCKDEP_WARN(ct_nmi_nesting() <= 0,
+++++++ "RCU nmi_nesting counter underflow/zero!");
/* Are we at first interrupt nesting level? */
------- nesting = ct_dynticks_nmi_nesting();
+++++++ nesting = ct_nmi_nesting();
if (nesting > 1)
return false;
WARN_ON_ONCE(!nesting && !is_idle_task(current));
/* Does CPU appear to be idle from an RCU standpoint? */
------- return ct_dynticks_nesting() == 0;
+++++++ return ct_nesting() == 0;
}
#define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
{
lockdep_assert_irqs_disabled();
------- RCU_LOCKDEP_WARN(ct_dynticks_nesting() <= 0,
------- "RCU dynticks_nesting counter underflow/zero!");
------- RCU_LOCKDEP_WARN(ct_dynticks_nmi_nesting() !=
------- DYNTICK_IRQ_NONIDLE,
------- "Bad RCU dynticks_nmi_nesting counter\n");
------- RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
+++++++ RCU_LOCKDEP_WARN(ct_nesting() <= 0,
+++++++ "RCU nesting counter underflow/zero!");
+++++++ RCU_LOCKDEP_WARN(ct_nmi_nesting() !=
+++++++ CT_NESTING_IRQ_NONIDLE,
+++++++ "Bad RCU nmi_nesting counter\n");
+++++++ RCU_LOCKDEP_WARN(!rcu_is_watching_curr_cpu(),
"RCU in extended quiescent state!");
}
#endif /* #ifdef CONFIG_PROVE_RCU */
if (in_nmi())
return;
------- RCU_LOCKDEP_WARN(rcu_dynticks_curr_cpu_in_eqs(),
+++++++ RCU_LOCKDEP_WARN(!rcu_is_watching_curr_cpu(),
"Illegal rcu_irq_enter_check_tick() from extended quiescent state");
if (!tick_nohz_full_cpu(rdp->cpu) ||
bool ret;
preempt_disable_notrace();
------- ret = !rcu_dynticks_curr_cpu_in_eqs();
+++++++ ret = rcu_is_watching_curr_cpu();
preempt_enable_notrace();
return ret;
}
}
/*
------- * Snapshot the specified CPU's dynticks counter so that we can later
+++++++ * Snapshot the specified CPU's RCU_WATCHING counter so that we can later
* credit them with an implicit quiescent state. Return 1 if this CPU
* is in dynticks idle mode, which is an extended quiescent state.
*/
-------static int dyntick_save_progress_counter(struct rcu_data *rdp)
+++++++static int rcu_watching_snap_save(struct rcu_data *rdp)
{
/*
* Full ordering between remote CPU's post idle accesses and updater's
* Ordering between remote CPU's pre idle accesses and post grace period
* updater's accesses is enforced by the below acquire semantic.
*/
------- rdp->dynticks_snap = ct_dynticks_cpu_acquire(rdp->cpu);
------- if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
+++++++ rdp->watching_snap = ct_rcu_watching_cpu_acquire(rdp->cpu);
+++++++ if (rcu_watching_snap_in_eqs(rdp->watching_snap)) {
trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
rcu_gpnum_ovf(rdp->mynode, rdp);
return 1;
/*
* Returns positive if the specified CPU has passed through a quiescent state
* by virtue of being in or having passed through an dynticks idle state since
------- * the last call to dyntick_save_progress_counter() for this same CPU, or by
+++++++ * the last call to rcu_watching_snap_save() for this same CPU, or by
* virtue of having been offline.
*
* Returns negative if the specified CPU needs a force resched.
*
* Returns zero otherwise.
*/
-------static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
+++++++static int rcu_watching_snap_recheck(struct rcu_data *rdp)
{
unsigned long jtsq;
int ret = 0;
* read-side critical section that started before the beginning
* of the current RCU grace period.
*/
------- if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
+++++++ if (rcu_watching_snap_stopped_since(rdp, rdp->watching_snap)) {
trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
rcu_gpnum_ovf(rnp, rdp);
return 1;
* the done tail list manipulations are protected here.
*/
done = smp_load_acquire(&rcu_state.srs_done_tail);
------- if (!done)
+++++++ if (WARN_ON_ONCE(!done))
return;
WARN_ON_ONCE(!rcu_sr_is_wait_head(done));
if (first_time) {
/* Collect dyntick-idle snapshots. */
------- force_qs_rnp(dyntick_save_progress_counter);
+++++++ force_qs_rnp(rcu_watching_snap_save);
} else {
/* Handle dyntick-idle and offline CPUs. */
------- force_qs_rnp(rcu_implicit_dynticks_qs);
+++++++ force_qs_rnp(rcu_watching_snap_recheck);
}
/* Clear flag to prevent immediate re-entry. */
if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
{
unsigned long flags;
unsigned long mask;
- ------ bool needacc = false;
struct rcu_node *rnp;
WARN_ON_ONCE(rdp->cpu != smp_processor_id());
* to return true. So complain, but don't awaken.
*/
WARN_ON_ONCE(rcu_accelerate_cbs(rnp, rdp));
- ------ } else if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {
- ------ /*
- ------ * ...but NOCB kthreads may miss or delay callbacks acceleration
- ------ * if in the middle of a (de-)offloading process.
- ------ */
- ------ needacc = true;
}
rcu_disable_urgency_upon_qs(rdp);
rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
/* ^^^ Released rnp->lock */
- ------
- ------ if (needacc) {
- ------ rcu_nocb_lock_irqsave(rdp, flags);
- ------ rcu_accelerate_cbs_unlocked(rnp, rdp);
- ------ rcu_nocb_unlock_irqrestore(rdp, flags);
- ------ }
}
}
unsigned long flags;
struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
struct rcu_node *rnp = rdp->mynode;
- ------ /*
- ------ * On RT rcu_core() can be preempted when IRQs aren't disabled.
- ------ * Therefore this function can race with concurrent NOCB (de-)offloading
- ------ * on this CPU and the below condition must be considered volatile.
- ------ * However if we race with:
- ------ *
- ------ * _ Offloading: In the worst case we accelerate or process callbacks
- ------ * concurrently with NOCB kthreads. We are guaranteed to
- ------ * call rcu_nocb_lock() if that happens.
- ------ *
- ------ * _ Deoffloading: In the worst case we miss callbacks acceleration or
- ------ * processing. This is fine because the early stage
- ------ * of deoffloading invokes rcu_core() after setting
- ------ * SEGCBLIST_RCU_CORE. So we guarantee that we'll process
- ------ * what could have been dismissed without the need to wait
- ------ * for the next rcu_pending() check in the next jiffy.
- ------ */
- ------ const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist);
if (cpu_is_offline(smp_processor_id()))
return;
/* No grace period and unregistered callbacks? */
if (!rcu_gp_in_progress() &&
- ------ rcu_segcblist_is_enabled(&rdp->cblist) && do_batch) {
- ------ rcu_nocb_lock_irqsave(rdp, flags);
+ ++++++ rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp)) {
+ ++++++ local_irq_save(flags);
if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
rcu_accelerate_cbs_unlocked(rnp, rdp);
- ------ rcu_nocb_unlock_irqrestore(rdp, flags);
+ ++++++ local_irq_restore(flags);
}
rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
/* If there are callbacks ready, invoke them. */
- ------ if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) &&
+ ++++++ if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) &&
likely(READ_ONCE(rcu_scheduler_fully_active))) {
rcu_do_batch(rdp);
/* Re-invoke RCU core processing if there are callbacks remaining. */
struct list_head list;
struct rcu_gp_oldstate gp_snap;
unsigned long nr_records;
------- void *records[];
+++++++ void *records[] __counted_by(nr_records);
};
/*
if (delayed_work_pending(&krcp->monitor_work)) {
delay_left = krcp->monitor_work.timer.expires - jiffies;
if (delay < delay_left)
------ - mod_delayed_work(system_wq, &krcp->monitor_work, delay);
++++++ + mod_delayed_work(system_unbound_wq, &krcp->monitor_work, delay);
return;
}
------ - queue_delayed_work(system_wq, &krcp->monitor_work, delay);
++++++ + queue_delayed_work(system_unbound_wq, &krcp->monitor_work, delay);
}
static void
// be that the work is in the pending state when
// channels have been detached following by each
// other.
------ - queue_rcu_work(system_wq, &krwp->rcu_work);
++++++ + queue_rcu_work(system_unbound_wq, &krwp->rcu_work);
}
}
if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
!atomic_xchg(&krcp->work_in_progress, 1)) {
if (atomic_read(&krcp->backoff_page_cache_fill)) {
------ - queue_delayed_work(system_wq,
++++++ + queue_delayed_work(system_unbound_wq,
&krcp->page_cache_work,
msecs_to_jiffies(rcu_delay_page_cache_fill_msec));
} else {
}
// Finally insert and update the GP for this page.
------- bnode->records[bnode->nr_records++] = ptr;
+++++++ bnode->nr_records++;
+++++++ bnode->records[bnode->nr_records - 1] = ptr;
get_state_synchronize_rcu_full(&bnode->gp_snap);
atomic_inc(&(*krcp)->bulk_count[idx]);
{
unsigned long __maybe_unused s = rcu_state.barrier_sequence;
+++++ ++ rhp->next = rhp; // Mark the callback as having been invoked.
if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
rcu_barrier_trace(TPS("LastCB"), -1, s);
complete(&rcu_state.barrier_completion);
/* Set up local state, ensuring consistent view of global state. */
rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
INIT_WORK(&rdp->strict_work, strict_work_handler);
------- WARN_ON_ONCE(ct->dynticks_nesting != 1);
------- WARN_ON_ONCE(rcu_dynticks_in_eqs(ct_dynticks_cpu(cpu)));
+++++++ WARN_ON_ONCE(ct->nesting != 1);
+++++++ WARN_ON_ONCE(rcu_watching_snap_in_eqs(ct_rcu_watching_cpu(cpu)));
rdp->barrier_seq_snap = rcu_state.barrier_sequence;
rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
rdp->rcu_ofl_gp_state = RCU_GP_CLEANED;
rdp->qlen_last_fqs_check = 0;
rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
rdp->blimit = blimit;
------- ct->dynticks_nesting = 1; /* CPU not up, no tearing. */
+++++++ ct->nesting = 1; /* CPU not up, no tearing. */
raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
/*
rnp = rdp->mynode;
mask = rdp->grpmask;
arch_spin_lock(&rcu_state.ofl_lock);
------- rcu_dynticks_eqs_online();
+++++++ rcu_watching_online();
raw_spin_lock(&rcu_state.barrier_lock);
raw_spin_lock_rcu_node(rnp);
WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask);
while (i > rnp->grphi)
rnp++;
per_cpu_ptr(&rcu_data, i)->mynode = rnp;
+++++ ++ per_cpu_ptr(&rcu_data, i)->barrier_head.next =
+++++ ++ &per_cpu_ptr(&rcu_data, i)->barrier_head;
rcu_boot_init_percpu_data(i);
}
}
long blimit; /* Upper limit on a processed batch */
/* 3) dynticks interface. */
------- int dynticks_snap; /* Per-GP tracking for dynticks. */
+++++++ int watching_snap; /* Per-GP tracking for dynticks. */
bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */
bool rcu_urgent_qs; /* GP old need light quiescent state. */
bool rcu_forced_tick; /* Forced tick to provide QS. */
/* 4) rcu_barrier(), OOM callbacks, and expediting. */
unsigned long barrier_seq_snap; /* Snap of rcu_state.barrier_sequence. */
struct rcu_head barrier_head;
------- int exp_dynticks_snap; /* Double-check need for IPI. */
+++++++ int exp_watching_snap; /* Double-check need for IPI. */
/* 5) Callback offloading. */
#ifdef CONFIG_RCU_NOCB_CPU
arch_spinlock_t ofl_lock ____cacheline_internodealigned_in_smp;
/* Synchronize offline with */
/* GP pre-initialization. */
- ------ int nocb_is_setup; /* nocb is setup from boot */
/* synchronize_rcu() part. */
struct llist_head srs_next; /* request a GP users. */
struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
struct work_struct srs_cleanup_work;
atomic_t srs_cleanups_pending; /* srs inflight worker cleanups. */
+ ++++++
+ ++++++#ifdef CONFIG_RCU_NOCB_CPU
+ ++++++ struct mutex nocb_mutex; /* Guards (de-)offloading */
+ ++++++ int nocb_is_setup; /* nocb is setup from boot */
+ ++++++#endif
};
/* Values for rcu_state structure's gp_flags field. */
* post grace period updater's accesses is enforced by the
* below acquire semantic.
*/
------- snap = ct_dynticks_cpu_acquire(cpu);
------- if (rcu_dynticks_in_eqs(snap))
+++++++ snap = ct_rcu_watching_cpu_acquire(cpu);
+++++++ if (rcu_watching_snap_in_eqs(snap))
mask_ofl_test |= mask;
else
------- rdp->exp_dynticks_snap = snap;
+++++++ rdp->exp_watching_snap = snap;
}
}
mask_ofl_ipi = rnp->expmask & ~mask_ofl_test;
unsigned long mask = rdp->grpmask;
retry_ipi:
------- if (rcu_dynticks_in_eqs_since(rdp, rdp->exp_dynticks_snap)) {
+++++++ if (rcu_watching_snap_stopped_since(rdp, rdp->exp_watching_snap)) {
mask_ofl_test |= mask;
continue;
}
return false;
}
+++ ++++/*
+++ ++++ * Print out an expedited RCU CPU stall warning message.
+++ ++++ */
+++ ++++static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigned long j)
+++ ++++{
+++ ++++ int cpu;
+++ ++++ unsigned long mask;
+++ ++++ int ndetected;
+++ ++++ struct rcu_node *rnp;
+++ ++++ struct rcu_node *rnp_root = rcu_get_root();
+++ ++++
+++ ++++ if (READ_ONCE(csd_lock_suppress_rcu_stall) && csd_lock_is_stuck()) {
+++ ++++ pr_err("INFO: %s detected expedited stalls, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name);
+++ ++++ return;
+++ ++++ }
+++ ++++ pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {", rcu_state.name);
+++ ++++ ndetected = 0;
+++ ++++ rcu_for_each_leaf_node(rnp) {
+++ ++++ ndetected += rcu_print_task_exp_stall(rnp);
+++ ++++ for_each_leaf_node_possible_cpu(rnp, cpu) {
+++ ++++ struct rcu_data *rdp;
+++ ++++
+++ ++++ mask = leaf_node_cpu_bit(rnp, cpu);
+++ ++++ if (!(READ_ONCE(rnp->expmask) & mask))
+++ ++++ continue;
+++ ++++ ndetected++;
+++ ++++ rdp = per_cpu_ptr(&rcu_data, cpu);
+++ ++++ pr_cont(" %d-%c%c%c%c", cpu,
+++ ++++ "O."[!!cpu_online(cpu)],
+++ ++++ "o."[!!(rdp->grpmask & rnp->expmaskinit)],
+++ ++++ "N."[!!(rdp->grpmask & rnp->expmaskinitnext)],
+++ ++++ "D."[!!data_race(rdp->cpu_no_qs.b.exp)]);
+++ ++++ }
+++ ++++ }
+++ ++++ pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
+++ ++++ j - jiffies_start, rcu_state.expedited_sequence, data_race(rnp_root->expmask),
+++ ++++ ".T"[!!data_race(rnp_root->exp_tasks)]);
+++ ++++ if (ndetected) {
+++ ++++ pr_err("blocking rcu_node structures (internal RCU debug):");
+++ ++++ rcu_for_each_node_breadth_first(rnp) {
+++ ++++ if (rnp == rnp_root)
+++ ++++ continue; /* printed unconditionally */
+++ ++++ if (sync_rcu_exp_done_unlocked(rnp))
+++ ++++ continue;
+++ ++++ pr_cont(" l=%u:%d-%d:%#lx/%c",
+++ ++++ rnp->level, rnp->grplo, rnp->grphi, data_race(rnp->expmask),
+++ ++++ ".T"[!!data_race(rnp->exp_tasks)]);
+++ ++++ }
+++ ++++ pr_cont("\n");
+++ ++++ }
+++ ++++ rcu_for_each_leaf_node(rnp) {
+++ ++++ for_each_leaf_node_possible_cpu(rnp, cpu) {
+++ ++++ mask = leaf_node_cpu_bit(rnp, cpu);
+++ ++++ if (!(READ_ONCE(rnp->expmask) & mask))
+++ ++++ continue;
+++ ++++ dump_cpu_task(cpu);
+++ ++++ }
+++ ++++ rcu_exp_print_detail_task_stall_rnp(rnp);
+++ ++++ }
+++ ++++}
+++ ++++
/*
* Wait for the expedited grace period to elapse, issuing any needed
* RCU CPU stall warnings along the way.
unsigned long jiffies_stall;
unsigned long jiffies_start;
unsigned long mask;
--- ---- int ndetected;
struct rcu_data *rdp;
struct rcu_node *rnp;
--- ---- struct rcu_node *rnp_root = rcu_get_root();
unsigned long flags;
trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));
j = jiffies;
rcu_stall_notifier_call_chain(RCU_STALL_NOTIFY_EXP, (void *)(j - jiffies_start));
trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall"));
--- ---- pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
--- ---- rcu_state.name);
--- ---- ndetected = 0;
--- ---- rcu_for_each_leaf_node(rnp) {
--- ---- ndetected += rcu_print_task_exp_stall(rnp);
--- ---- for_each_leaf_node_possible_cpu(rnp, cpu) {
--- ---- struct rcu_data *rdp;
--- ----
--- ---- mask = leaf_node_cpu_bit(rnp, cpu);
--- ---- if (!(READ_ONCE(rnp->expmask) & mask))
--- ---- continue;
--- ---- ndetected++;
--- ---- rdp = per_cpu_ptr(&rcu_data, cpu);
--- ---- pr_cont(" %d-%c%c%c%c", cpu,
--- ---- "O."[!!cpu_online(cpu)],
--- ---- "o."[!!(rdp->grpmask & rnp->expmaskinit)],
--- ---- "N."[!!(rdp->grpmask & rnp->expmaskinitnext)],
--- ---- "D."[!!data_race(rdp->cpu_no_qs.b.exp)]);
--- ---- }
--- ---- }
--- ---- pr_cont(" } %lu jiffies s: %lu root: %#lx/%c\n",
--- ---- j - jiffies_start, rcu_state.expedited_sequence,
--- ---- data_race(rnp_root->expmask),
--- ---- ".T"[!!data_race(rnp_root->exp_tasks)]);
--- ---- if (ndetected) {
--- ---- pr_err("blocking rcu_node structures (internal RCU debug):");
--- ---- rcu_for_each_node_breadth_first(rnp) {
--- ---- if (rnp == rnp_root)
--- ---- continue; /* printed unconditionally */
--- ---- if (sync_rcu_exp_done_unlocked(rnp))
--- ---- continue;
--- ---- pr_cont(" l=%u:%d-%d:%#lx/%c",
--- ---- rnp->level, rnp->grplo, rnp->grphi,
--- ---- data_race(rnp->expmask),
--- ---- ".T"[!!data_race(rnp->exp_tasks)]);
--- ---- }
--- ---- pr_cont("\n");
--- ---- }
--- ---- rcu_for_each_leaf_node(rnp) {
--- ---- for_each_leaf_node_possible_cpu(rnp, cpu) {
--- ---- mask = leaf_node_cpu_bit(rnp, cpu);
--- ---- if (!(READ_ONCE(rnp->expmask) & mask))
--- ---- continue;
--- ---- preempt_disable(); // For smp_processor_id() in dump_cpu_task().
--- ---- dump_cpu_task(cpu);
--- ---- preempt_enable();
--- ---- }
--- ---- rcu_exp_print_detail_task_stall_rnp(rnp);
--- ---- }
+++ ++++ synchronize_rcu_expedited_stall(jiffies_start, j);
jiffies_stall = 3 * rcu_exp_jiffies_till_stall_check() + 3;
panic_on_rcu_stall();
}
#ifdef CONFIG_RCU_NOCB_CPU
static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
- ------static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)
- ------{
- ------ return lockdep_is_held(&rdp->nocb_lock);
- ------}
static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
{
raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
if (needwake) {
trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));
- ------ wake_up_process(rdp_gp->nocb_gp_kthread);
+ ++++++ swake_up_one_online(&rdp_gp->nocb_gp_wq);
}
return needwake;
return false;
}
- ------ // In the process of (de-)offloading: no bypassing, but
- ------ // locking.
- ------ if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {
- ------ rcu_nocb_lock(rdp);
- ------ *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
- ------ return false; /* Not offloaded, no bypassing. */
- ------ }
- ------
// Don't use ->nocb_bypass during early boot.
if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
rcu_nocb_lock(rdp);
trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));
}
rcu_nocb_bypass_unlock(rdp);
- ------ smp_mb(); /* Order enqueue before wake. */
+ ++++++
// A wake up of the grace period kthread or timer adjustment
// needs to be done only if:
// 1. Bypass list was fully empty before (this is the first
}
}
- ------static int nocb_gp_toggle_rdp(struct rcu_data *rdp)
+ ++++++static void nocb_gp_toggle_rdp(struct rcu_data *rdp_gp, struct rcu_data *rdp)
{
struct rcu_segcblist *cblist = &rdp->cblist;
unsigned long flags;
- ------ int ret;
- ------ rcu_nocb_lock_irqsave(rdp, flags);
- ------ if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) &&
- ------ !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) {
+ ++++++ /*
+ ++++++ * Locking orders future de-offloaded callbacks enqueue against previous
+ ++++++ * handling of this rdp. Ie: Make sure rcuog is done with this rdp before
+ ++++++ * deoffloaded callbacks can be enqueued.
+ ++++++ */
+ ++++++ raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
+ ++++++ if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) {
/*
* Offloading. Set our flag and notify the offload worker.
* We will handle this rdp until it ever gets de-offloaded.
*/
- ------ rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP);
- ------ ret = 1;
- ------ } else if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED) &&
- ------ rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) {
+ ++++++ list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);
+ ++++++ rcu_segcblist_set_flags(cblist, SEGCBLIST_OFFLOADED);
+ ++++++ } else {
/*
* De-offloading. Clear our flag and notify the de-offload worker.
* We will ignore this rdp until it ever gets re-offloaded.
*/
- ------ rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP);
- ------ ret = 0;
- ------ } else {
- ------ WARN_ON_ONCE(1);
- ------ ret = -1;
+ ++++++ list_del(&rdp->nocb_entry_rdp);
+ ++++++ rcu_segcblist_clear_flags(cblist, SEGCBLIST_OFFLOADED);
}
- ------
- ------ rcu_nocb_unlock_irqrestore(rdp, flags);
- ------
- ------ return ret;
+ ++++++ raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
}
static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)
}
if (rdp_toggling) {
- ------ int ret;
- ------
- ------ ret = nocb_gp_toggle_rdp(rdp_toggling);
- ------ if (ret == 1)
- ------ list_add_tail(&rdp_toggling->nocb_entry_rdp, &my_rdp->nocb_head_rdp);
- ------ else if (ret == 0)
- ------ list_del(&rdp_toggling->nocb_entry_rdp);
- ------
+ ++++++ nocb_gp_toggle_rdp(my_rdp, rdp_toggling);
swake_up_one(&rdp_toggling->nocb_state_wq);
}
WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));
local_irq_save(flags);
------- rcu_momentary_dyntick_idle();
+++++++ rcu_momentary_eqs();
local_irq_restore(flags);
/*
* Disable BH to provide the expected environment. Also, when
}
EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup);
- ------static int rdp_offload_toggle(struct rcu_data *rdp,
- ------ bool offload, unsigned long flags)
- ------ __releases(rdp->nocb_lock)
+ ++++++static int rcu_nocb_queue_toggle_rdp(struct rcu_data *rdp)
{
- ------ struct rcu_segcblist *cblist = &rdp->cblist;
struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
bool wake_gp = false;
- ------
- ------ rcu_segcblist_offload(cblist, offload);
- ------ rcu_nocb_unlock_irqrestore(rdp, flags);
+ ++++++ unsigned long flags;
raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
// Queue this rdp for add/del to/from the list to iterate on rcuog
return wake_gp;
}
- ------static long rcu_nocb_rdp_deoffload(void *arg)
+ ++++++static bool rcu_nocb_rdp_deoffload_wait_cond(struct rcu_data *rdp)
{
- ------ struct rcu_data *rdp = arg;
- ------ struct rcu_segcblist *cblist = &rdp->cblist;
unsigned long flags;
- ------ int wake_gp;
- ------ struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
+ ++++++ bool ret;
/*
- ------ * rcu_nocb_rdp_deoffload() may be called directly if
- ------ * rcuog/o[p] spawn failed, because at this time the rdp->cpu
- ------ * is not online yet.
+ ++++++ * Locking makes sure rcuog is done handling this rdp before deoffloaded
+ ++++++ * enqueue can happen. Also it keeps the SEGCBLIST_OFFLOADED flag stable
+ ++++++ * while the ->nocb_lock is held.
*/
- ------ WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu));
+ ++++++ raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
+ ++++++ ret = !rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
+ ++++++ raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
+ ++++++
+ ++++++ return ret;
+ ++++++}
+ ++++++
+ ++++++static int rcu_nocb_rdp_deoffload(struct rcu_data *rdp)
+ ++++++{
+ ++++++ unsigned long flags;
+ ++++++ int wake_gp;
+ ++++++ struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
+ ++++++
+ ++++++ /* CPU must be offline, unless it's early boot */
+ ++++++ WARN_ON_ONCE(cpu_online(rdp->cpu) && rdp->cpu != raw_smp_processor_id());
pr_info("De-offloading %d\n", rdp->cpu);
- ------ rcu_nocb_lock_irqsave(rdp, flags);
- ------ /*
- ------ * Flush once and for all now. This suffices because we are
- ------ * running on the target CPU holding ->nocb_lock (thus having
- ------ * interrupts disabled), and because rdp_offload_toggle()
- ------ * invokes rcu_segcblist_offload(), which clears SEGCBLIST_OFFLOADED.
- ------ * Thus future calls to rcu_segcblist_completely_offloaded() will
- ------ * return false, which means that future calls to rcu_nocb_try_bypass()
- ------ * will refuse to put anything into the bypass.
- ------ */
- ------ WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, false));
+ ++++++ /* Flush all callbacks from segcblist and bypass */
+ ++++++ rcu_barrier();
+ ++++++
/*
- ------ * Start with invoking rcu_core() early. This way if the current thread
- ------ * happens to preempt an ongoing call to rcu_core() in the middle,
- ------ * leaving some work dismissed because rcu_core() still thinks the rdp is
- ------ * completely offloaded, we are guaranteed a nearby future instance of
- ------ * rcu_core() to catch up.
+ ++++++ * Make sure the rcuoc kthread isn't in the middle of a nocb locked
+ ++++++ * sequence while offloading is deactivated, along with nocb locking.
*/
- ------ rcu_segcblist_set_flags(cblist, SEGCBLIST_RCU_CORE);
- ------ invoke_rcu_core();
- ------ wake_gp = rdp_offload_toggle(rdp, false, flags);
+ ++++++ if (rdp->nocb_cb_kthread)
+ ++++++ kthread_park(rdp->nocb_cb_kthread);
+ ++++++
+ ++++++ rcu_nocb_lock_irqsave(rdp, flags);
+ ++++++ WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
+ ++++++ WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist));
+ ++++++ rcu_nocb_unlock_irqrestore(rdp, flags);
+ ++++++
+ ++++++ wake_gp = rcu_nocb_queue_toggle_rdp(rdp);
mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
+ ++++++
if (rdp_gp->nocb_gp_kthread) {
if (wake_gp)
wake_up_process(rdp_gp->nocb_gp_kthread);
swait_event_exclusive(rdp->nocb_state_wq,
- ------ !rcu_segcblist_test_flags(cblist,
- ------ SEGCBLIST_KTHREAD_GP));
- ------ if (rdp->nocb_cb_kthread)
- ------ kthread_park(rdp->nocb_cb_kthread);
+ ++++++ rcu_nocb_rdp_deoffload_wait_cond(rdp));
} else {
/*
* No kthread to clear the flags for us or remove the rdp from the nocb list
* to iterate. Do it here instead. Locking doesn't look stricly necessary
* but we stick to paranoia in this rare path.
*/
- ------ rcu_nocb_lock_irqsave(rdp, flags);
- ------ rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP);
- ------ rcu_nocb_unlock_irqrestore(rdp, flags);
+ ++++++ raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
+ ++++++ rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
+ ++++++ raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
list_del(&rdp->nocb_entry_rdp);
}
- ------ mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
- ------
- ------ /*
- ------ * Lock one last time to acquire latest callback updates from kthreads
- ------ * so we can later handle callbacks locally without locking.
- ------ */
- ------ rcu_nocb_lock_irqsave(rdp, flags);
- ------ /*
- ------ * Theoretically we could clear SEGCBLIST_LOCKING after the nocb
- ------ * lock is released but how about being paranoid for once?
- ------ */
- ------ rcu_segcblist_clear_flags(cblist, SEGCBLIST_LOCKING);
- ------ /*
- ------ * Without SEGCBLIST_LOCKING, we can't use
- ------ * rcu_nocb_unlock_irqrestore() anymore.
- ------ */
- ------ raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
- ------
- ------ /* Sanity check */
- ------ WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
+ ++++++ mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
return 0;
}
int ret = 0;
cpus_read_lock();
- ------ mutex_lock(&rcu_state.barrier_mutex);
+ ++++++ mutex_lock(&rcu_state.nocb_mutex);
if (rcu_rdp_is_offloaded(rdp)) {
- ------ if (cpu_online(cpu)) {
- ------ ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);
+ ++++++ if (!cpu_online(cpu)) {
+ ++++++ ret = rcu_nocb_rdp_deoffload(rdp);
if (!ret)
cpumask_clear_cpu(cpu, rcu_nocb_mask);
} else {
- ------ pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu);
+ ++++++ pr_info("NOCB: Cannot CB-deoffload online CPU %d\n", rdp->cpu);
ret = -EINVAL;
}
}
- ------ mutex_unlock(&rcu_state.barrier_mutex);
+ ++++++ mutex_unlock(&rcu_state.nocb_mutex);
cpus_read_unlock();
return ret;
}
EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload);
- ------static long rcu_nocb_rdp_offload(void *arg)
+ ++++++static bool rcu_nocb_rdp_offload_wait_cond(struct rcu_data *rdp)
{
- ------ struct rcu_data *rdp = arg;
- ------ struct rcu_segcblist *cblist = &rdp->cblist;
unsigned long flags;
+ ++++++ bool ret;
+ ++++++
+ ++++++ raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
+ ++++++ ret = rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
+ ++++++ raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
+ ++++++
+ ++++++ return ret;
+ ++++++}
+ ++++++
+ ++++++static int rcu_nocb_rdp_offload(struct rcu_data *rdp)
+ ++++++{
int wake_gp;
struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
- ------ WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());
+ ++++++ WARN_ON_ONCE(cpu_online(rdp->cpu));
/*
* For now we only support re-offload, ie: the rdp must have been
* offloaded on boot first.
pr_info("Offloading %d\n", rdp->cpu);
- ------ /*
- ------ * Can't use rcu_nocb_lock_irqsave() before SEGCBLIST_LOCKING
- ------ * is set.
- ------ */
- ------ raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
+ ++++++ WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
+ ++++++ WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist));
- ------ /*
- ------ * We didn't take the nocb lock while working on the
- ------ * rdp->cblist with SEGCBLIST_LOCKING cleared (pure softirq/rcuc mode).
- ------ * Every modifications that have been done previously on
- ------ * rdp->cblist must be visible remotely by the nocb kthreads
- ------ * upon wake up after reading the cblist flags.
- ------ *
- ------ * The layout against nocb_lock enforces that ordering:
- ------ *
- ------ * __rcu_nocb_rdp_offload() nocb_cb_wait()/nocb_gp_wait()
- ------ * ------------------------- ----------------------------
- ------ * WRITE callbacks rcu_nocb_lock()
- ------ * rcu_nocb_lock() READ flags
- ------ * WRITE flags READ callbacks
- ------ * rcu_nocb_unlock() rcu_nocb_unlock()
- ------ */
- ------ wake_gp = rdp_offload_toggle(rdp, true, flags);
+ ++++++ wake_gp = rcu_nocb_queue_toggle_rdp(rdp);
if (wake_gp)
wake_up_process(rdp_gp->nocb_gp_kthread);
- ------ kthread_unpark(rdp->nocb_cb_kthread);
- ------
swait_event_exclusive(rdp->nocb_state_wq,
- ------ rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP));
+ ++++++ rcu_nocb_rdp_offload_wait_cond(rdp));
- ------ /*
- ------ * All kthreads are ready to work, we can finally relieve rcu_core() and
- ------ * enable nocb bypass.
- ------ */
- ------ rcu_nocb_lock_irqsave(rdp, flags);
- ------ rcu_segcblist_clear_flags(cblist, SEGCBLIST_RCU_CORE);
- ------ rcu_nocb_unlock_irqrestore(rdp, flags);
+ ++++++ kthread_unpark(rdp->nocb_cb_kthread);
return 0;
}
int ret = 0;
cpus_read_lock();
- ------ mutex_lock(&rcu_state.barrier_mutex);
+ ++++++ mutex_lock(&rcu_state.nocb_mutex);
if (!rcu_rdp_is_offloaded(rdp)) {
- ------ if (cpu_online(cpu)) {
- ------ ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);
+ ++++++ if (!cpu_online(cpu)) {
+ ++++++ ret = rcu_nocb_rdp_offload(rdp);
if (!ret)
cpumask_set_cpu(cpu, rcu_nocb_mask);
} else {
- ------ pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu);
+ ++++++ pr_info("NOCB: Cannot CB-offload online CPU %d\n", rdp->cpu);
ret = -EINVAL;
}
}
- ------ mutex_unlock(&rcu_state.barrier_mutex);
+ ++++++ mutex_unlock(&rcu_state.nocb_mutex);
cpus_read_unlock();
return ret;
return 0;
/* Protect rcu_nocb_mask against concurrent (de-)offloading. */
- ------ if (!mutex_trylock(&rcu_state.barrier_mutex))
+ ++++++ if (!mutex_trylock(&rcu_state.nocb_mutex))
return 0;
/* Snapshot count of all CPUs */
count += READ_ONCE(rdp->lazy_len);
}
- ------ mutex_unlock(&rcu_state.barrier_mutex);
+ ++++++ mutex_unlock(&rcu_state.nocb_mutex);
return count ? count : SHRINK_EMPTY;
}
* Protect against concurrent (de-)offloading. Otherwise nocb locking
* may be ignored or imbalanced.
*/
- ------ if (!mutex_trylock(&rcu_state.barrier_mutex)) {
+ ++++++ if (!mutex_trylock(&rcu_state.nocb_mutex)) {
/*
- ------ * But really don't insist if barrier_mutex is contended since we
+ ++++++ * But really don't insist if nocb_mutex is contended since we
* can't guarantee that it will never engage in a dependency
* chain involving memory allocation. The lock is seldom contended
* anyway.
break;
}
- ------ mutex_unlock(&rcu_state.barrier_mutex);
+ ++++++ mutex_unlock(&rcu_state.nocb_mutex);
return count ? count : SHRINK_STOP;
}
rdp = per_cpu_ptr(&rcu_data, cpu);
if (rcu_segcblist_empty(&rdp->cblist))
rcu_segcblist_init(&rdp->cblist);
- ------ rcu_segcblist_offload(&rdp->cblist, true);
- ------ rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP);
- ------ rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE);
+ ++++++ rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
}
rcu_organize_nocb_kthreads();
}
"rcuog/%d", rdp_gp->cpu);
if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__)) {
mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
- ------ goto end;
+ ++++++ goto err;
}
WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
if (kthread_prio)
t = kthread_create(rcu_nocb_cb_kthread, rdp,
"rcuo%c/%d", rcu_state.abbr, cpu);
if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__))
- ------ goto end;
+ ++++++ goto err;
if (rcu_rdp_is_offloaded(rdp))
wake_up_process(t);
WRITE_ONCE(rdp->nocb_cb_kthread, t);
WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
return;
- ------end:
- ------ mutex_lock(&rcu_state.barrier_mutex);
+ ++++++
+ ++++++err:
+ ++++++ /*
+ ++++++ * No need to protect against concurrent rcu_barrier()
+ ++++++ * because the number of callbacks should be 0 for a non-boot CPU,
+ ++++++ * therefore rcu_barrier() shouldn't even try to grab the nocb_lock.
+ ++++++ * But hold nocb_mutex to avoid nocb_lock imbalance from shrinker.
+ ++++++ */
+ ++++++ WARN_ON_ONCE(system_state > SYSTEM_BOOTING && rcu_segcblist_n_cbs(&rdp->cblist));
+ ++++++ mutex_lock(&rcu_state.nocb_mutex);
if (rcu_rdp_is_offloaded(rdp)) {
rcu_nocb_rdp_deoffload(rdp);
cpumask_clear_cpu(cpu, rcu_nocb_mask);
}
- ------ mutex_unlock(&rcu_state.barrier_mutex);
+ ++++++ mutex_unlock(&rcu_state.nocb_mutex);
}
/* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
- ------static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)
- ------{
- ------ return 0;
- ------}
- ------
- ------static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
- ------{
- ------ return false;
- ------}
- ------
/* No ->nocb_lock to acquire. */
static void rcu_nocb_lock(struct rcu_data *rdp)
{
* timers have their own means of synchronization against the
* offloaded state updaters.
*/
- ------ RCU_LOCKDEP_WARN(
+ ++++++ RCU_NOCB_LOCKDEP_WARN(
!(lockdep_is_held(&rcu_state.barrier_mutex) ||
(IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_held()) ||
- ------ rcu_lockdep_is_held_nocb(rdp) ||
+ ++++++ lockdep_is_held(&rdp->nocb_lock) ||
+ ++++++ lockdep_is_held(&rcu_state.nocb_mutex) ||
(!(IS_ENABLED(CONFIG_PREEMPT_COUNT) && preemptible()) &&
rdp == this_cpu_ptr(&rcu_data)) ||
rcu_current_is_nocb_kthread(rdp)),
/*
* Register an urgently needed quiescent state. If there is an
------- * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
+++++++ * emergency, invoke rcu_momentary_eqs() to do a heavy-weight
* dyntick-idle quiescent state visible to other CPUs, which will in
* some cases serve for expedited as well as normal grace periods.
* Either way, register a lightweight quiescent state.
this_cpu_write(rcu_data.rcu_urgent_qs, false);
if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) {
local_irq_save(flags);
------- rcu_momentary_dyntick_idle();
+++++++ rcu_momentary_eqs();
local_irq_restore(flags);
}
rcu_qs();
goto out;
this_cpu_write(rcu_data.rcu_urgent_qs, false);
if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs)))
------- rcu_momentary_dyntick_idle();
+++++++ rcu_momentary_eqs();
out:
rcu_tasks_qs(current, preempt);
trace_rcu_utilization(TPS("End context switch"));
#include <linux/kvm_para.h>
#include <linux/rcu_notifier.h>
+++ ++++#include <linux/smp.h>
//////////////////////////////////////////////////////////////////////////////
//
struct rcu_node *rnp;
rcu_for_each_leaf_node(rnp) {
+++ ++++ printk_deferred_enter();
raw_spin_lock_irqsave_rcu_node(rnp, flags);
for_each_leaf_node_possible_cpu(rnp, cpu)
if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
dump_cpu_task(cpu);
}
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
+++ ++++ printk_deferred_exit();
}
}
}
delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq);
falsepositive = rcu_is_gp_kthread_starving(NULL) &&
------- rcu_dynticks_in_eqs(ct_dynticks_cpu(cpu));
+++++++ rcu_watching_snap_in_eqs(ct_rcu_watching_cpu(cpu));
rcuc_starved = rcu_is_rcuc_kthread_starving(rdp, &j);
if (rcuc_starved)
// Print signed value, as negative values indicate a probable bug.
rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' :
"!."[!delta],
ticks_value, ticks_title,
------- ct_dynticks_cpu(cpu) & 0xffff,
------- ct_dynticks_nesting_cpu(cpu), ct_dynticks_nmi_nesting_cpu(cpu),
+++++++ ct_rcu_watching_cpu(cpu) & 0xffff,
+++++++ ct_nesting_cpu(cpu), ct_nmi_nesting_cpu(cpu),
rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
rcuc_starved ? buf : "",
set_preempt_need_resched();
}
+++ ++++static bool csd_lock_suppress_rcu_stall;
+++ ++++module_param(csd_lock_suppress_rcu_stall, bool, 0644);
+++ ++++
static void check_cpu_stall(struct rcu_data *rdp)
{
bool self_detected;
return;
rcu_stall_notifier_call_chain(RCU_STALL_NOTIFY_NORM, (void *)j - gps);
--- ---- if (self_detected) {
+++ ++++ if (READ_ONCE(csd_lock_suppress_rcu_stall) && csd_lock_is_stuck()) {
+++ ++++ pr_err("INFO: %s detected stall, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name);
+++ ++++ } else if (self_detected) {
/* We haven't checked in, so go dump stack. */
print_cpu_stall(gps);
} else {
preempt_count_set(PREEMPT_DISABLED);
}
rcu_sleep_check();
------- SCHED_WARN_ON(ct_state() == CONTEXT_USER);
+++++++ SCHED_WARN_ON(ct_state() == CT_STATE_USER);
profile_hit(SCHED_PROFILING, __builtin_return_address(0));
* we find a better solution.
*
* NB: There are buggy callers of this function. Ideally we
------- * should warn if prev_state != CONTEXT_USER, but that will trigger
+++++++ * should warn if prev_state != CT_STATE_USER, but that will trigger
* too frequently to make sense yet.
*/
enum ctx_state prev_state = exception_enter();
void dump_cpu_task(int cpu)
{
--- ---- if (cpu == smp_processor_id() && in_hardirq()) {
+++ ++++ if (in_hardirq() && cpu == smp_processor_id()) {
struct pt_regs *regs;
regs = get_irq_regs();
projects / J-linux.git / commitdiff