rcu: Replace flush_signals() with WARN_ON(signal_pending())

[android-x86/kernel.git] / kernel / rcu / tree_plugin.h

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 00dc411..bbb0a0c 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -134,7 +134,7 @@ static void __init rcu_bootup_announce(void)
  * Return the number of RCU-preempt batches processed thus far
  * for debug and statistics.
  */
-long rcu_batches_completed_preempt(void)
+static long rcu_batches_completed_preempt(void)
 {
        return rcu_preempt_state.completed;
 }
@@ -897,7 +897,8 @@ void synchronize_rcu_expedited(void)
 
        /* Clean up and exit. */
        smp_mb(); /* ensure expedited GP seen before counter increment. */
-       ACCESS_ONCE(sync_rcu_preempt_exp_count)++;
+       ACCESS_ONCE(sync_rcu_preempt_exp_count) =
+                                       sync_rcu_preempt_exp_count + 1;
 unlock_mb_ret:
        mutex_unlock(&sync_rcu_preempt_exp_mutex);
 mb_ret:
@@ -1625,7 +1626,7 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
 
        /* Exit early if we advanced recently. */
        if (jiffies == rdtp->last_advance_all)
-               return 0;
+               return false;
        rdtp->last_advance_all = jiffies;
 
        for_each_rcu_flavor(rsp) {
@@ -2074,9 +2075,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
 
        if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
                return;
-       if (!ACCESS_ONCE(rdp_leader->nocb_leader_wake) || force) {
+       if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
                /* Prior xchg orders against prior callback enqueue. */
-               ACCESS_ONCE(rdp_leader->nocb_leader_wake) = true;
+               ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
                wake_up(&rdp_leader->nocb_wq);
        }
 }
@@ -2120,16 +2121,23 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            TPS("WakeEmpty"));
                } else {
-                       rdp->nocb_defer_wakeup = true;
+                       rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            TPS("WakeEmptyIsDeferred"));
                }
                rdp->qlen_last_fqs_check = 0;
        } else if (len > rdp->qlen_last_fqs_check + qhimark) {
                /* ... or if many callbacks queued. */
-               wake_nocb_leader(rdp, true);
+               if (!irqs_disabled_flags(flags)) {
+                       wake_nocb_leader(rdp, true);
+                       trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                           TPS("WakeOvf"));
+               } else {
+                       rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
+                       trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
+                                           TPS("WakeOvfIsDeferred"));
+               }
                rdp->qlen_last_fqs_check = LONG_MAX / 2;
-               trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeOvf"));
        } else {
                trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));
        }
@@ -2229,7 +2237,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp)
                        (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c)));
                if (likely(d))
                        break;
-               flush_signals(current);
+               WARN_ON(signal_pending(current));
                trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait"));
        }
        trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait"));
@@ -2253,7 +2261,7 @@ wait_again:
        if (!rcu_nocb_poll) {
                trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
                wait_event_interruptible(my_rdp->nocb_wq,
-                                        ACCESS_ONCE(my_rdp->nocb_leader_wake));
+                               !ACCESS_ONCE(my_rdp->nocb_leader_sleep));
                /* Memory barrier handled by smp_mb() calls below and repoll. */
        } else if (firsttime) {
                firsttime = false; /* Don't drown trace log with "Poll"! */
@@ -2288,16 +2296,16 @@ wait_again:
                if (!rcu_nocb_poll)
                        trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,
                                            "WokeEmpty");
-               flush_signals(current);
+               WARN_ON(signal_pending(current));
                schedule_timeout_interruptible(1);
 
                /* Rescan in case we were a victim of memory ordering. */
-               my_rdp->nocb_leader_wake = false;
-               smp_mb();  /* Ensure _wake false before scan. */
+               my_rdp->nocb_leader_sleep = true;
+               smp_mb();  /* Ensure _sleep true before scan. */
                for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
                        if (ACCESS_ONCE(rdp->nocb_head)) {
                                /* Found CB, so short-circuit next wait. */
-                               my_rdp->nocb_leader_wake = true;
+                               my_rdp->nocb_leader_sleep = false;
                                break;
                        }
                goto wait_again;
@@ -2307,17 +2315,17 @@ wait_again:
        rcu_nocb_wait_gp(my_rdp);
 
        /*
-        * We left ->nocb_leader_wake set to reduce cache thrashing.
-        * We clear it now, but recheck for new callbacks while
+        * We left ->nocb_leader_sleep unset to reduce cache thrashing.
+        * We set it now, but recheck for new callbacks while
         * traversing our follower list.
         */
-       my_rdp->nocb_leader_wake = false;
-       smp_mb(); /* Ensure _wake false before scan of ->nocb_head. */
+       my_rdp->nocb_leader_sleep = true;
+       smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
 
        /* Each pass through the following loop wakes a follower, if needed. */
        for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
                if (ACCESS_ONCE(rdp->nocb_head))
-                       my_rdp->nocb_leader_wake = true; /* No need to wait. */
+                       my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
                if (!rdp->nocb_gp_head)
                        continue; /* No CBs, so no need to wake follower. */
 
@@ -2367,7 +2375,7 @@ static void nocb_follower_wait(struct rcu_data *rdp)
                if (!rcu_nocb_poll)
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            "WokeEmpty");
-               flush_signals(current);
+               WARN_ON(signal_pending(current));
                schedule_timeout_interruptible(1);
        }
 }
@@ -2428,15 +2436,16 @@ static int rcu_nocb_kthread(void *arg)
                        list = next;
                }
                trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
-               ACCESS_ONCE(rdp->nocb_p_count) -= c;
-               ACCESS_ONCE(rdp->nocb_p_count_lazy) -= cl;
+               ACCESS_ONCE(rdp->nocb_p_count) = rdp->nocb_p_count - c;
+               ACCESS_ONCE(rdp->nocb_p_count_lazy) =
+                                               rdp->nocb_p_count_lazy - cl;
                rdp->n_nocbs_invoked += c;
        }
        return 0;
 }
 
 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
-static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
+static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
 {
        return ACCESS_ONCE(rdp->nocb_defer_wakeup);
 }
@@ -2444,11 +2453,14 @@ static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
 /* Do a deferred wakeup of rcu_nocb_kthread(). */
 static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
 {
+       int ndw;
+
        if (!rcu_nocb_need_deferred_wakeup(rdp))
                return;
-       ACCESS_ONCE(rdp->nocb_defer_wakeup) = false;
-       wake_nocb_leader(rdp, false);
-       trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWakeEmpty"));
+       ndw = ACCESS_ONCE(rdp->nocb_defer_wakeup);
+       ACCESS_ONCE(rdp->nocb_defer_wakeup) = RCU_NOGP_WAKE_NOT;
+       wake_nocb_leader(rdp, ndw == RCU_NOGP_WAKE_FORCE);
+       trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));
 }
 
 /* Initialize per-rcu_data variables for no-CBs CPUs. */
@@ -2555,7 +2567,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
 {
 }
 
-static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
+static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
 {
        return false;
 }