rcu: Convert rcu_grace_period tracepoint to gp_seq

This commit makes the rcu_grace_period tracepoint use gp_seq instead
of ->gpnum or ->completed.  It also introduces a "cpuofl-bgp" string to
less obscurely indicate when a CPU has gone offline while a grace period
is waiting on it.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 5936aac..cd229e8 100644 (file)
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -52,6 +52,7 @@ TRACE_EVENT(rcu_utilization,
  *     "cpuqs": CPU passes through a quiescent state.
  *     "cpuonl": CPU comes online.
  *     "cpuofl": CPU goes offline.
+ *     "cpuofl-bgp": CPU goes offline while blocking a grace period.
  *     "reqwait": GP kthread sleeps waiting for grace-period request.
  *     "reqwaitsig": GP kthread awakened by signal from reqwait state.
  *     "fqswait": GP kthread waiting until time to force quiescent states.
@@ -63,24 +64,24 @@ TRACE_EVENT(rcu_utilization,
  */
 TRACE_EVENT(rcu_grace_period,
 
-       TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent),
+       TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent),
 
-       TP_ARGS(rcuname, gpnum, gpevent),
+       TP_ARGS(rcuname, gp_seq, gpevent),
 
        TP_STRUCT__entry(
                __field(const char *, rcuname)
-               __field(unsigned long, gpnum)
+               __field(unsigned long, gp_seq)
                __field(const char *, gpevent)
        ),
 
        TP_fast_assign(
                __entry->rcuname = rcuname;
-               __entry->gpnum = gpnum;
+               __entry->gp_seq = gp_seq;
                __entry->gpevent = gpevent;
        ),
 
        TP_printk("%s %lu %s",
-                 __entry->rcuname, __entry->gpnum, __entry->gpevent)
+                 __entry->rcuname, __entry->gp_seq, __entry->gpevent)
 );
 
 /*
@@ -753,7 +754,7 @@ TRACE_EVENT(rcu_barrier,
 
 #else /* #ifdef CONFIG_RCU_TRACE */
 
-#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
+#define trace_rcu_grace_period(rcuname, gp_seq, gpevent) do { } while (0)
 #define trace_rcu_future_grace_period(rcuname, gpnum, completed, c, \
                                      level, grplo, grphi, event) \
                                      do { } while (0)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 4826598..7ce85ad 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -234,7 +234,7 @@ void rcu_sched_qs(void)
        if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
                return;
        trace_rcu_grace_period(TPS("rcu_sched"),
-                              __this_cpu_read(rcu_sched_data.gpnum),
+                              __this_cpu_read(rcu_sched_data.gp_seq),
                               TPS("cpuqs"));
        __this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
        if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
@@ -249,7 +249,7 @@ void rcu_bh_qs(void)
        RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
        if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
                trace_rcu_grace_period(TPS("rcu_bh"),
-                                      __this_cpu_read(rcu_bh_data.gpnum),
+                                      __this_cpu_read(rcu_bh_data.gp_seq),
                                       TPS("cpuqs"));
                __this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
        }
@@ -1615,7 +1615,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
                trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread"));
                goto unlock_out;
        }
-       trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), TPS("newreq"));
+       trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq"));
        ret = true;  /* Caller must wake GP kthread. */
 unlock_out:
        /* Push furthest requested GP to leaf node and rcu_data structure. */
@@ -1702,9 +1702,9 @@ static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp,
 
        /* Trace depending on how much we were able to accelerate. */
        if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
-               trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));
+               trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB"));
        else
-               trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));
+               trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB"));
        return ret;
 }
 
@@ -1774,7 +1774,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
                 * go looking for one.
                 */
                rdp->gpnum = rnp->gpnum;
-               trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
+               trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpustart"));
                need_gp = !!(rnp->qsmask & rdp->grpmask);
                rdp->cpu_no_qs.b.norm = need_gp;
                rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
@@ -1851,7 +1851,7 @@ static bool rcu_gp_init(struct rcu_state *rsp)
        rcu_seq_start(&rsp->gp_seq);
        if (WARN_ON_ONCE(((rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT) != rcu_seq_ctr(rnp->gp_seq))) /* Catch any ->completed/->gp_seq mismatches. */
                pr_info("%s ->completed: %#lx (%#lx) ->gp_seq %#lx (%#lx)\n", __func__, rnp->completed, (rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT, rnp->gp_seq, rcu_seq_ctr(rnp->gp_seq));
-       trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
+       trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start"));
        raw_spin_unlock_irq_rcu_node(rnp);
 
        /*
@@ -1928,7 +1928,7 @@ static bool rcu_gp_init(struct rcu_state *rsp)
                if (rnp == rdp->mynode)
                        (void)__note_gp_changes(rsp, rnp, rdp);
                rcu_preempt_boost_start_gp(rnp);
-               trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
+               trace_rcu_grace_period_init(rsp->name, rnp->gp_seq,
                                            rnp->level, rnp->grplo,
                                            rnp->grphi, rnp->qsmask);
                raw_spin_unlock_irq_rcu_node(rnp);
@@ -2048,7 +2048,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        /* Declare grace period done. */
        WRITE_ONCE(rsp->completed, rsp->gpnum);
        rcu_seq_end(&rsp->gp_seq);
-       trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
+       trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end"));
        rsp->gp_state = RCU_GP_IDLE;
        /* Check for GP requests since above loop. */
        rdp = this_cpu_ptr(rsp->rda);
@@ -2061,7 +2061,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
        if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
                WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
                rsp->gp_req_activity = jiffies;
-               trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),
+               trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq),
                                       TPS("newreq"));
        } else {
                WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
@@ -2087,7 +2087,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
                /* Handle grace-period start. */
                for (;;) {
                        trace_rcu_grace_period(rsp->name,
-                                              READ_ONCE(rsp->gpnum),
+                                              READ_ONCE(rsp->gp_seq),
                                               TPS("reqwait"));
                        rsp->gp_state = RCU_GP_WAIT_GPS;
                        swait_event_idle(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
@@ -2100,7 +2100,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
                        WRITE_ONCE(rsp->gp_activity, jiffies);
                        WARN_ON(signal_pending(current));
                        trace_rcu_grace_period(rsp->name,
-                                              READ_ONCE(rsp->gpnum),
+                                              READ_ONCE(rsp->gp_seq),
                                               TPS("reqwaitsig"));
                }
 
@@ -2119,7 +2119,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
                                           jiffies + 3 * j);
                        }
                        trace_rcu_grace_period(rsp->name,
-                                              READ_ONCE(rsp->gpnum),
+                                              READ_ONCE(rsp->gp_seq),
                                               TPS("fqswait"));
                        rsp->gp_state = RCU_GP_WAIT_FQS;
                        ret = swait_event_idle_timeout(rsp->gp_wq,
@@ -2134,12 +2134,12 @@ static int __noreturn rcu_gp_kthread(void *arg)
                        if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||
                            (gf & RCU_GP_FLAG_FQS)) {
                                trace_rcu_grace_period(rsp->name,
-                                                      READ_ONCE(rsp->gpnum),
+                                                      READ_ONCE(rsp->gp_seq),
                                                       TPS("fqsstart"));
                                rcu_gp_fqs(rsp, first_gp_fqs);
                                first_gp_fqs = false;
                                trace_rcu_grace_period(rsp->name,
-                                                      READ_ONCE(rsp->gpnum),
+                                                      READ_ONCE(rsp->gp_seq),
                                                       TPS("fqsend"));
                                cond_resched_tasks_rcu_qs();
                                WRITE_ONCE(rsp->gp_activity, jiffies);
@@ -2158,7 +2158,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
                                WRITE_ONCE(rsp->gp_activity, jiffies);
                                WARN_ON(signal_pending(current));
                                trace_rcu_grace_period(rsp->name,
-                                                      READ_ONCE(rsp->gpnum),
+                                                      READ_ONCE(rsp->gp_seq),
                                                       TPS("fqswaitsig"));
                                ret = 1; /* Keep old FQS timing. */
                                j = jiffies;
@@ -2388,17 +2388,16 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
  */
 static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
 {
-       RCU_TRACE(unsigned long mask;)
+       RCU_TRACE(bool blkd;)
        RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)
        RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)
 
        if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
                return;
 
-       RCU_TRACE(mask = rdp->grpmask;)
-       trace_rcu_grace_period(rsp->name,
-                              rnp->gpnum + 1 - !!(rnp->qsmask & mask),
-                              TPS("cpuofl"));
+       RCU_TRACE(blkd = !!(rnp->qsmask & rdp->grpmask);)
+       trace_rcu_grace_period(rsp->name, rnp->gp_seq,
+                              blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
 }
 
 /*
@@ -3538,7 +3537,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
        rdp->core_needs_qs = false;
        rdp->rcu_iw_pending = false;
        rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
-       trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
+       trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));
        raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
 }
 

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index ca73931..aca9d18 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -301,7 +301,7 @@ static void rcu_preempt_qs(void)
        RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_qs() invoked with preemption enabled!!!\n");
        if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
                trace_rcu_grace_period(TPS("rcu_preempt"),
-                                      __this_cpu_read(rcu_data_p->gpnum),
+                                      __this_cpu_read(rcu_data_p->gp_seq),
                                       TPS("cpuqs"));
                __this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
                barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */