rcu: Discard separate per-CPU callback counts

Back when there were multiple flavors of RCU, it was necessary to
separately count lazy and non-lazy callbacks for each CPU.  These counts
were used in CONFIG_RCU_FAST_NO_HZ kernels to determine how long a newly
idle CPU should be allowed to sleep before handling its RCU callbacks.
But now that there is only one flavor, the callback counts for a given
CPU's sole rcu_data structure are the counts for that CPU.

This commit therefore removes the rcu_data structure's ->nonlazy_posted
and ->nonlazy_posted_snap fields, the rcu_idle_count_callbacks_posted()
and rcu_cpu_has_callbacks() functions, repurposes the rcu_data structure's
->all_lazy field to record the laziness state at the beginning of the
latest idle sojourn, and modifies CONFIG_RCU_FAST_NO_HZ RCU CPU stall
warnings accordingly.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>

diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt
index 073dbc1..1ab70c3 100644 (file)
--- a/Documentation/RCU/stallwarn.txt
+++ b/Documentation/RCU/stallwarn.txt
@@ -219,17 +219,18 @@ an estimate of the total number of RCU callbacks queued across all CPUs
 In kernels with CONFIG_RCU_FAST_NO_HZ, more information is printed
 for each CPU:
 
-       0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 nonlazy_posted: 25 .D
+       0: (64628 ticks this GP) idle=dd5/3fffffffffffffff/0 softirq=82/543 last_accelerate: a345/d342 Nonlazy posted: ..D
 
 The "last_accelerate:" prints the low-order 16 bits (in hex) of the
 jiffies counter when this CPU last invoked rcu_try_advance_all_cbs()
 from rcu_needs_cpu() or last invoked rcu_accelerate_cbs() from
-rcu_prepare_for_idle().  The "nonlazy_posted:" prints the number
-of non-lazy callbacks posted since the last call to rcu_needs_cpu().
-Finally, an "L" indicates that there are currently no non-lazy callbacks
-("." is printed otherwise, as shown above) and "D" indicates that
-dyntick-idle processing is enabled ("." is printed otherwise, for example,
-if disabled via the "nohz=" kernel boot parameter).
+rcu_prepare_for_idle().  The "Nonlazy posted:" indicates lazy-callback
+status, so that an "l" indicates that all callbacks were lazy at the start
+of the last idle period and an "L" indicates that there are currently
+no non-lazy callbacks (in both cases, "." is printed otherwise, as
+shown above) and "D" indicates that dyntick-idle processing is enabled
+("." is printed otherwise, for example, if disabled via the "nohz="
+kernel boot parameter).
 
 If the grace period ends just as the stall warning starts printing,
 there will be a spurious stall-warning message, which will include

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e2bd42b..e53a586 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2878,9 +2878,6 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, int cpu, bool lazy)
                        rcu_segcblist_init(&rdp->cblist);
        }
        rcu_segcblist_enqueue(&rdp->cblist, head, lazy);
-       if (!lazy)
-               rcu_idle_count_callbacks_posted();
-
        if (__is_kfree_rcu_offset((unsigned long)func))
                trace_rcu_kfree_callback(rcu_state.name, head,
                                         (unsigned long)func,
@@ -3111,28 +3108,6 @@ static int rcu_pending(void)
 }
 
 /*
- * Return true if the specified CPU has any callback.  If all_lazy is
- * non-NULL, store an indication of whether all callbacks are lazy.
- * (If there are no callbacks, all of them are deemed to be lazy.)
- */
-static bool rcu_cpu_has_callbacks(bool *all_lazy)
-{
-       bool al = true;
-       bool hc = false;
-       struct rcu_data *rdp;
-
-       rdp = this_cpu_ptr(&rcu_data);
-       if (!rcu_segcblist_empty(&rdp->cblist)) {
-               hc = true;
-               if (rcu_segcblist_n_nonlazy_cbs(&rdp->cblist))
-                       al = false;
-       }
-       if (all_lazy)
-               *all_lazy = al;
-       return hc;
-}
-
-/*
  * Helper function for rcu_barrier() tracing.  If tracing is disabled,
  * the compiler is expected to optimize this away.
  */

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 50bb41c..20feecb 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -193,10 +193,7 @@ struct rcu_data {
        bool rcu_need_heavy_qs;         /* GP old, so heavy quiescent state! */
        bool rcu_urgent_qs;             /* GP old need light quiescent state. */
 #ifdef CONFIG_RCU_FAST_NO_HZ
-       bool all_lazy;                  /* Are all CPU's CBs lazy? */
-       unsigned long nonlazy_posted;   /* # times non-lazy CB posted to CPU. */
-       unsigned long nonlazy_posted_snap;
-                                       /* Nonlazy_posted snapshot. */
+       bool all_lazy;                  /* All CPU's CBs lazy at idle start? */
        unsigned long last_accelerate;  /* Last jiffy CBs were accelerated. */
        unsigned long last_advance_all; /* Last jiffy CBs were all advanced. */
        int tick_nohz_enabled_snap;     /* Previously seen value from sysfs. */
@@ -430,7 +427,6 @@ static void __init rcu_spawn_boost_kthreads(void);
 static void rcu_prepare_kthreads(int cpu);
 static void rcu_cleanup_after_idle(void);
 static void rcu_prepare_for_idle(void);
-static void rcu_idle_count_callbacks_posted(void);
 static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
 static bool rcu_preempt_need_deferred_qs(struct task_struct *t);
 static void rcu_preempt_deferred_qs(struct task_struct *t);

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 43f3f2e..abd238c 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1474,7 +1474,7 @@ static void rcu_prepare_kthreads(int cpu)
 int rcu_needs_cpu(u64 basemono, u64 *nextevt)
 {
        *nextevt = KTIME_MAX;
-       return rcu_cpu_has_callbacks(NULL);
+       return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist);
 }
 
 /*
@@ -1493,14 +1493,6 @@ static void rcu_prepare_for_idle(void)
 {
 }
 
-/*
- * Don't bother keeping a running count of the number of RCU callbacks
- * posted because CONFIG_RCU_FAST_NO_HZ=n.
- */
-static void rcu_idle_count_callbacks_posted(void)
-{
-}
-
 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
 
 /*
@@ -1583,11 +1575,8 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt)
 
        lockdep_assert_irqs_disabled();
 
-       /* Snapshot to detect later posting of non-lazy callback. */
-       rdp->nonlazy_posted_snap = rdp->nonlazy_posted;
-
        /* If no callbacks, RCU doesn't need the CPU. */
-       if (!rcu_cpu_has_callbacks(&rdp->all_lazy)) {
+       if (rcu_segcblist_empty(&rdp->cblist)) {
                *nextevt = KTIME_MAX;
                return 0;
        }
@@ -1601,11 +1590,12 @@ int rcu_needs_cpu(u64 basemono, u64 *nextevt)
        rdp->last_accelerate = jiffies;
 
        /* Request timer delay depending on laziness, and round. */
-       if (!rdp->all_lazy) {
+       rdp->all_lazy = !rcu_segcblist_n_nonlazy_cbs(&rdp->cblist);
+       if (rdp->all_lazy) {
+               dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
+       } else {
                dj = round_up(rcu_idle_gp_delay + jiffies,
                               rcu_idle_gp_delay) - jiffies;
-       } else {
-               dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
        }
        *nextevt = basemono + dj * TICK_NSEC;
        return 0;
@@ -1635,7 +1625,7 @@ static void rcu_prepare_for_idle(void)
        /* Handle nohz enablement switches conservatively. */
        tne = READ_ONCE(tick_nohz_active);
        if (tne != rdp->tick_nohz_enabled_snap) {
-               if (rcu_cpu_has_callbacks(NULL))
+               if (!rcu_segcblist_empty(&rdp->cblist))
                        invoke_rcu_core(); /* force nohz to see update. */
                rdp->tick_nohz_enabled_snap = tne;
                return;
@@ -1648,10 +1638,8 @@ static void rcu_prepare_for_idle(void)
         * callbacks, invoke RCU core for the side-effect of recalculating
         * idle duration on re-entry to idle.
         */
-       if (rdp->all_lazy &&
-           rdp->nonlazy_posted != rdp->nonlazy_posted_snap) {
+       if (rdp->all_lazy && rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)) {
                rdp->all_lazy = false;
-               rdp->nonlazy_posted_snap = rdp->nonlazy_posted;
                invoke_rcu_core();
                return;
        }
@@ -1687,19 +1675,6 @@ static void rcu_cleanup_after_idle(void)
                invoke_rcu_core();
 }
 
-/*
- * Keep a running count of the number of non-lazy callbacks posted
- * on this CPU.  This running counter (which is never decremented) allows
- * rcu_prepare_for_idle() to detect when something out of the idle loop
- * posts a callback, even if an equal number of callbacks are invoked.
- * Of course, callbacks should only be posted from within a trace event
- * designed to be called from idle or from within RCU_NONIDLE().
- */
-static void rcu_idle_count_callbacks_posted(void)
-{
-       __this_cpu_add(rcu_data.nonlazy_posted, 1);
-}
-
 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
 
 #ifdef CONFIG_RCU_FAST_NO_HZ
@@ -1707,13 +1682,12 @@ static void rcu_idle_count_callbacks_posted(void)
 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
 {
        struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
-       unsigned long nlpd = rdp->nonlazy_posted - rdp->nonlazy_posted_snap;
 
-       sprintf(cp, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
+       sprintf(cp, "last_accelerate: %04lx/%04lx, Nonlazy posted: %c%c%c",
                rdp->last_accelerate & 0xffff, jiffies & 0xffff,
-               ulong2long(nlpd),
-               rdp->all_lazy ? 'L' : '.',
-               rdp->tick_nohz_enabled_snap ? '.' : 'D');
+               ".l"[rdp->all_lazy],
+               ".L"[!rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)],
+               ".D"[!rdp->tick_nohz_enabled_snap]);
 }
 
 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */