rcu: Eliminate lockless accesses to rcu_sync->gp_count

The rcu_sync structure's ->gp_count field is always accessed under the
protection of that same structure's ->rss_lock field, with the exception
of a pair of WARN_ON_ONCE() calls just prior to acquiring that lock in
functions rcu_sync_exit() and rcu_sync_dtor().  These lockless accesses
are unnecessary and impair KCSAN's ability to catch bugs that might be
inserted via other lockless accesses.

This commit therefore moves those WARN_ON_ONCE() calls under the lock.

Signed-off-by: Oleg Nesterov <[email protected]>
Signed-off-by: Paul E. McKenney <[email protected]>

diff --git a/kernel/rcu/sync.c b/kernel/rcu/sync.c
index 6c2bd9001adcdbede753450918e416b9bd4db5e1..da60a9947c005d7bcef584a788201d21d9be19c7 100644 (file)
--- a/kernel/rcu/sync.c
+++ b/kernel/rcu/sync.c
@@ -122,7 +122,7 @@ void rcu_sync_enter(struct rcu_sync *rsp)
                 * we are called at early boot time but this shouldn't happen.
                 */
        }
-       WRITE_ONCE(rsp->gp_count, rsp->gp_count + 1);
+       rsp->gp_count++;
        spin_unlock_irq(&rsp->rss_lock);
 
        if (gp_state == GP_IDLE) {
@@ -151,15 +151,11 @@ void rcu_sync_enter(struct rcu_sync *rsp)
  */
 void rcu_sync_exit(struct rcu_sync *rsp)
 {
-       int gpc;
-
        WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE);
-       WARN_ON_ONCE(READ_ONCE(rsp->gp_count) == 0);
 
        spin_lock_irq(&rsp->rss_lock);
-       gpc = rsp->gp_count - 1;
-       WRITE_ONCE(rsp->gp_count, gpc);
-       if (!gpc) {
+       WARN_ON_ONCE(rsp->gp_count == 0);
+       if (!--rsp->gp_count) {
                if (rsp->gp_state == GP_PASSED) {
                        WRITE_ONCE(rsp->gp_state, GP_EXIT);
                        rcu_sync_call(rsp);
@@ -178,10 +174,10 @@ void rcu_sync_dtor(struct rcu_sync *rsp)
 {
        int gp_state;
 
-       WARN_ON_ONCE(READ_ONCE(rsp->gp_count));
        WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED);
 
        spin_lock_irq(&rsp->rss_lock);
+       WARN_ON_ONCE(rsp->gp_count);
        if (rsp->gp_state == GP_REPLAY)
                WRITE_ONCE(rsp->gp_state, GP_EXIT);
        gp_state = rsp->gp_state;