sched/fair: Add NOHZ_STATS_KICK

Split the NOHZ idle balancer into doing two separate actions:

 - update blocked load statistic

 - actually load-balance

Since the latter requires the former, ensure this happens. For now
always tag both bits at the same time.

Prepares for a future where we can toggle only the STATS bit.

Suggested-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 96ad1c0..69c9a6b 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -583,7 +583,7 @@ static inline bool got_nohz_idle_kick(void)
 {
        int cpu = smp_processor_id();
 
-       if (!(atomic_read(nohz_flags(cpu)) & NOHZ_BALANCE_KICK))
+       if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
                return false;
 
        if (idle_cpu(cpu) && !need_resched())
@@ -593,7 +593,7 @@ static inline bool got_nohz_idle_kick(void)
         * We can't run Idle Load Balance on this CPU for this time so we
         * cancel it and clear NOHZ_BALANCE_KICK
         */
-       atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(cpu));
+       atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
        return false;
 }
 

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 5d15047..fc05896 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -9082,8 +9082,8 @@ static void nohz_balancer_kick(void)
        if (ilb_cpu >= nr_cpu_ids)
                return;
 
-       flags = atomic_fetch_or(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu));
-       if (flags & NOHZ_BALANCE_KICK)
+       flags = atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(ilb_cpu));
+       if (flags & NOHZ_KICK_MASK)
                return;
        /*
         * Use smp_send_reschedule() instead of resched_cpu().
@@ -9202,8 +9202,6 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
        int need_serialize, need_decay = 0;
        u64 max_cost = 0;
 
-       update_blocked_averages(cpu);
-
        rcu_read_lock();
        for_each_domain(cpu, sd) {
                /*
@@ -9298,20 +9296,27 @@ out:
  * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
  * rebalancing for all the CPUs for whom scheduler ticks are stopped.
  */
-static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
+static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
 {
-       int this_cpu = this_rq->cpu;
-       struct rq *rq;
-       int balance_cpu;
        /* Earliest time when we have to do rebalance again */
        unsigned long next_balance = jiffies + 60*HZ;
        int update_next_balance = 0;
+       int this_cpu = this_rq->cpu;
+       unsigned int flags;
+       int balance_cpu;
+       struct rq *rq;
 
-       if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_BALANCE_KICK))
-               return;
+       if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK))
+               return false;
 
-       if (idle != CPU_IDLE)
-               goto end;
+       if (idle != CPU_IDLE) {
+               atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
+               return false;
+       }
+
+       flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
+
+       SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
 
        for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
                if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
@@ -9339,7 +9344,9 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
                        cpu_load_update_idle(rq);
                        rq_unlock_irq(rq, &rf);
 
-                       rebalance_domains(rq, CPU_IDLE);
+                       update_blocked_averages(rq->cpu);
+                       if (flags & NOHZ_BALANCE_KICK)
+                               rebalance_domains(rq, CPU_IDLE);
                }
 
                if (time_after(next_balance, rq->next_balance)) {
@@ -9348,6 +9355,10 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
                }
        }
 
+       update_blocked_averages(this_cpu);
+       if (flags & NOHZ_BALANCE_KICK)
+               rebalance_domains(this_rq, CPU_IDLE);
+
        /*
         * next_balance will be updated only when there is a need.
         * When the CPU is attached to null domain for ex, it will not be
@@ -9355,8 +9366,8 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
         */
        if (likely(update_next_balance))
                nohz.next_balance = next_balance;
-end:
-       atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));
+
+       return true;
 }
 
 /*
@@ -9443,7 +9454,10 @@ unlock:
        return kick;
 }
 #else
-static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { }
+static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
+{
+       return false;
+}
 #endif
 
 /*
@@ -9464,7 +9478,11 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
         * load balance only within the local sched_domain hierarchy
         * and abort nohz_idle_balance altogether if we pull some load.
         */
-       nohz_idle_balance(this_rq, idle);
+       if (nohz_idle_balance(this_rq, idle))
+               return;
+
+       /* normal load balance */
+       update_blocked_averages(this_rq->cpu);
        rebalance_domains(this_rq, idle);
 }
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d98e761..5295f27 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2036,9 +2036,13 @@ extern void cfs_bandwidth_usage_dec(void);
 #ifdef CONFIG_NO_HZ_COMMON
 #define NOHZ_TICK_STOPPED_BIT  0
 #define NOHZ_BALANCE_KICK_BIT  1
+#define NOHZ_STATS_KICK_BIT    2
 
 #define NOHZ_TICK_STOPPED      BIT(NOHZ_TICK_STOPPED_BIT)
 #define NOHZ_BALANCE_KICK      BIT(NOHZ_BALANCE_KICK_BIT)
+#define NOHZ_STATS_KICK                BIT(NOHZ_STATS_KICK_BIT)
+
+#define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
 
 #define nohz_flags(cpu)        (&cpu_rq(cpu)->nohz_flags)