rcu: Optimize and protect atomic_cmpxchg() loop

This commit reworks the atomic_cmpxchg() loop in rcu_eqs_special_set()
to do only the initial read from the current CPU's rcu_data structure's
->dynticks field explicitly.  On subsequent passes, this value is instead
retained from the failing atomic_cmpxchg() operation.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 4a4a975..6c62481 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -342,14 +342,17 @@ bool rcu_eqs_special_set(int cpu)
 {
        int old;
        int new;
+       int new_old;
        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
 
+       new_old = atomic_read(&rdp->dynticks);
        do {
-               old = atomic_read(&rdp->dynticks);
+               old = new_old;
                if (old & RCU_DYNTICK_CTRL_CTR)
                        return false;
                new = old | RCU_DYNTICK_CTRL_MASK;
-       } while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);
+               new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
+       } while (new_old != old);
        return true;
 }