/* First, put new protection in place to avoid critical-section gap. */
if (statesnew & RCUTORTURE_RDR_BH)
local_bh_disable();
+++ + if (statesnew & RCUTORTURE_RDR_RBH)
+++ + rcu_read_lock_bh();
if (statesnew & RCUTORTURE_RDR_IRQ)
local_irq_disable();
if (statesnew & RCUTORTURE_RDR_PREEMPT)
preempt_disable();
--- - if (statesnew & RCUTORTURE_RDR_RBH)
--- - rcu_read_lock_bh();
if (statesnew & RCUTORTURE_RDR_SCHED)
rcu_read_lock_sched();
if (statesnew & RCUTORTURE_RDR_RCU)
idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT;
--- - /* Next, remove old protection, irq first due to bh conflict. */
+++ + /*
+++ + * Next, remove old protection, in decreasing order of strength
+++ + * to avoid unlock paths that aren't safe in the stronger
+++ + * context. Namely: BH can not be enabled with disabled interrupts.
+++ + * Additionally PREEMPT_RT requires that BH is enabled in preemptible
+++ + * context.
+++ + */
if (statesold & RCUTORTURE_RDR_IRQ)
local_irq_enable();
--- - if (statesold & RCUTORTURE_RDR_BH)
--- - local_bh_enable();
if (statesold & RCUTORTURE_RDR_PREEMPT)
preempt_enable();
--- - if (statesold & RCUTORTURE_RDR_RBH)
--- - rcu_read_unlock_bh();
if (statesold & RCUTORTURE_RDR_SCHED)
rcu_read_unlock_sched();
+++ + if (statesold & RCUTORTURE_RDR_BH)
+++ + local_bh_enable();
+++ + if (statesold & RCUTORTURE_RDR_RBH)
+++ + rcu_read_unlock_bh();
if (statesold & RCUTORTURE_RDR_RCU) {
bool lockit = !statesnew && !(torture_random(trsp) & 0xffff);
int mask = rcutorture_extend_mask_max();
unsigned long randmask1 = torture_random(trsp) >> 8;
unsigned long randmask2 = randmask1 >> 3;
+++ + unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
+++ + unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ;
+++ + unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT);
/* Mostly only one bit (need preemption!), sometimes lots of bits. */
mask = mask & randmask2;
else
mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));
--- - /* Can't enable bh w/irq disabled. */
--- - if ((mask & RCUTORTURE_RDR_IRQ) &&
--- - ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) ||
--- - (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH))))
--- - mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
+++ +
+++ + /*
+++ + * Can't enable bh w/irq disabled.
+++ + */
+++ + if (mask & RCUTORTURE_RDR_IRQ)
+++ + mask |= oldmask & bhs;
+++ +
+++ + /*
+++ + * Ideally these sequences would be detected in debug builds
+++ + * (regardless of RT), but until then don't stop testing
+++ + * them on non-RT.
+++ + */
+++ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
+++ + /* Can't modify BH in atomic context */
+++ + if (oldmask & preempts_irq)
+++ + mask &= ~bhs;
+++ + if ((oldmask | mask) & preempts_irq)
+++ + mask |= oldmask & bhs;
+++ + }
+++ +
return mask ?: RCUTORTURE_RDR_RCU;
}
}
if (stall_cpu > 0) {
VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing");
---- if (IS_MODULE(CONFIG_RCU_TORTURE_TESTS))
++++ if (IS_MODULE(CONFIG_RCU_TORTURE_TEST))
return -EINVAL; /* In module, can fail back to user. */
WARN_ON(1); /* Make sure rcutorture notices conflict. */
return 0;
static int rcu_torture_read_exit_init(void)
{
if (read_exit_burst <= 0)
--- - return -EINVAL;
+++ + return 0;
init_waitqueue_head(&read_exit_wq);
read_exit_child_stop = false;
read_exit_child_stopped = false;
rcutorture_seq_diff(gp_seq, start_gp_seq));
torture_stop_kthread(rcu_torture_stats, stats_task);
torture_stop_kthread(rcu_torture_fqs, fqs_task);
--- - if (rcu_torture_can_boost())
+++ + if (rcu_torture_can_boost() && rcutor_hp >= 0)
cpuhp_remove_state(rcutor_hp);
/*
rcu_torture_write_types();
firsterr = torture_create_kthread(rcu_torture_writer, NULL,
writer_task);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
if (nfakewriters > 0) {
fakewriter_tasks = kcalloc(nfakewriters,
for (i = 0; i < nfakewriters; i++) {
firsterr = torture_create_kthread(rcu_torture_fakewriter,
NULL, fakewriter_tasks[i]);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
rcu_torture_reader_mbchk[i].rtc_chkrdr = -1;
firsterr = torture_create_kthread(rcu_torture_reader, (void *)i,
reader_tasks[i]);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
nrealnocbers = nocbs_nthreads;
}
for (i = 0; i < nrealnocbers; i++) {
firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
if (stat_interval > 0) {
firsterr = torture_create_kthread(rcu_torture_stats, NULL,
stats_task);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
if (test_no_idle_hz && shuffle_interval > 0) {
firsterr = torture_shuffle_init(shuffle_interval * HZ);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
if (stutter < 0)
t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;
firsterr = torture_stutter_init(stutter * HZ, t);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
if (fqs_duration < 0)
/* Create the fqs thread */
firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
fqs_task);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
}
if (test_boost_interval < 1)
firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE",
rcutorture_booster_init,
rcutorture_booster_cleanup);
--- - if (firsterr < 0)
--- - goto unwind;
rcutor_hp = firsterr;
+++ + if (torture_init_error(firsterr))
+++ + goto unwind;
// Testing RCU priority boosting requires rcutorture do
// some serious abuse. Counter this by running ksoftirqd
}
shutdown_jiffies = jiffies + shutdown_secs * HZ;
firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval,
rcutorture_sync);
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
firsterr = rcu_torture_stall_init();
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
firsterr = rcu_torture_fwd_prog_init();
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
firsterr = rcu_torture_barrier_init();
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
firsterr = rcu_torture_read_exit_init();
--- - if (firsterr)
+++ + if (torture_init_error(firsterr))
goto unwind;
if (object_debug)
rcu_test_debug_objects();