sched: Add static_key for asymmetric cpu capacity optimizations

The existing asymmetric cpu capacity code should cause minimal overhead
for others. Putting it behind a static_key, it has been done for SMT
optimizations, would make it easier to extend and improve without
causing harm to others moving forward.

cc: Ingo Molnar <mingo@redhat.com>
cc: Peter Zijlstra <peterz@infradead.org>

Signed-off-by: Morten Rasmussen <morten.rasmussen@arm.com>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 519dc4657f866247d4da15d9db8be70f4fa43a40..50241d6ce6897d96b814562d3962eb693c30a97e 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6213,6 +6213,9 @@ static int wake_cap(struct task_struct *p, int cpu, int prev_cpu)
 {
        long min_cap, max_cap;
 
+       if (!static_branch_unlikely(&sched_asym_cpucapacity))
+               return 0;
+
        min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu));
        max_cap = cpu_rq(cpu)->rd->max_cpu_capacity;
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d5a5e01faeb9550e175a641700a142f003c8d92c..8553177ef23103e84093cb54b6fe1d34babdc836 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1208,6 +1208,7 @@ DECLARE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
 DECLARE_PER_CPU(struct sched_domain *, sd_numa);
 DECLARE_PER_CPU(struct sched_domain *, sd_asym);
 DECLARE_PER_CPU(struct sched_domain *, sd_ea);
+extern struct static_key_false sched_asym_cpucapacity;
 
 struct sched_group_capacity {
        atomic_t                ref;

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 9a12693c7d009195bd6748a2dab8a0de2c49dffe..dcaaca2c0a885d6aff4c748b1f778c0d8e5c165a 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -585,6 +585,7 @@ DEFINE_PER_CPU(struct sched_domain_shared *, sd_llc_shared);
 DEFINE_PER_CPU(struct sched_domain *, sd_numa);
 DEFINE_PER_CPU(struct sched_domain *, sd_asym);
 DEFINE_PER_CPU(struct sched_domain *, sd_ea);
+DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
 
 static void update_top_cache_domain(int cpu)
 {
@@ -615,6 +616,21 @@ static void update_top_cache_domain(int cpu)
        rcu_assign_pointer(per_cpu(sd_ea, cpu), sd);
 }
 
+static void update_asym_cpucapacity(int cpu)
+{
+       int enable = false;
+
+       rcu_read_lock();
+       if (lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY))
+               enable = true;
+       rcu_read_unlock();
+
+       if (enable) {
+               /* This expects to be hotplug-safe */
+               static_branch_enable_cpuslocked(&sched_asym_cpucapacity);
+       }
+}
+
 /*
  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  * hold the hotplug lock.
@@ -1897,6 +1913,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
        }
        rcu_read_unlock();
 
+       if (!cpumask_empty(cpu_map))
+               update_asym_cpucapacity(cpumask_first(cpu_map));
+
        if (rq && sched_debug_enabled) {
                pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n",
                        cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);