sched/topology: Introduce sched_energy_present static key

In order to ensure a minimal performance impact on non-energy-aware
systems, introduce a static_key guarding the access to Energy-Aware
Scheduling (EAS) code.

The static key is set iff all the following conditions are met for at
least one root domain:
  1. all online CPUs of the root domain are covered by the Energy
     Model (EM);
  2. the complexity of the root domain's EM is low enough to keep
     scheduling overheads low;
  3. the root domain has an asymmetric CPU capacity topology (detected
     by looking for the SD_ASYM_CPUCAPACITY flag in the sched_domain
     hierarchy).

The static key is checked in the rd_freq_domain() function which returns
the frequency domains of a root domain when they are available. As EAS
cannot be enabled with CONFIG_ENERGY_MODEL=n, rd_freq_domain() is
stubbed to 'NULL' to let the compiler remove the unused EAS code by
constant propagation.

cc: Ingo Molnar <mingo@redhat.com>
cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Quentin Perret <quentin.perret@arm.com>

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 2484334f58db9b97fc95aeffa4761313f32ad230..797a6676c5671dedfb0f8dadf17b93174111ac56 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2239,8 +2239,25 @@ unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned
 
 #ifdef CONFIG_SMP
 #ifdef CONFIG_ENERGY_MODEL
+extern struct static_key_false sched_energy_present;
+/**
+ * rd_freq_domain - Get the frequency domains of a root domain.
+ *
+ * Must be called from a RCU read-side critical section.
+ */
+static inline struct freq_domain *rd_freq_domain(struct root_domain *rd)
+{
+       if (!static_branch_unlikely(&sched_energy_present))
+               return NULL;
+
+       return rcu_dereference(rd->fd);
+}
 #define freq_domain_span(fd) (to_cpumask(((fd)->obj->cpus)))
 #else
+static inline struct freq_domain *rd_freq_domain(struct root_domain *rd)
+{
+       return NULL;
+}
 #define freq_domain_span(fd) NULL
 #endif
 #endif

diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 787dcae48ce875f41a8370763c685e532176cfba..9a12693c7d009195bd6748a2dab8a0de2c49dffe 100644 (file)
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -275,12 +275,32 @@ static void destroy_freq_domain_rcu(struct rcu_head *rp)
        free_fd(fd);
 }
 
+/*
+ * The complexity of the Energy Model is defined as: nr_fd * (nr_cpus + nr_cs)
+ * with: 'nr_fd' the number of frequency domains; 'nr_cpus' the number of CPUs;
+ * and 'nr_cs' the sum of the capacity states numbers of all frequency domains.
+ *
+ * It is generally not a good idea to use such a model in the wake-up path on
+ * very complex platforms because of the associated scheduling overheads. The
+ * arbitrary constraint below prevents that. It makes EAS usable up to 16 CPUs
+ * with per-CPU DVFS and less than 8 capacity states each, for example.
+ */
+#define EM_MAX_COMPLEXITY 2048
+
+
 static void build_freq_domains(const struct cpumask *cpu_map)
 {
+       int i, nr_fd = 0, nr_cs = 0, nr_cpus = cpumask_weight(cpu_map);
        struct freq_domain *fd = NULL, *tmp;
        int cpu = cpumask_first(cpu_map);
        struct root_domain *rd = cpu_rq(cpu)->rd;
-       int i;
+
+       /* EAS is enabled for asymmetric CPU capacity topologies. */
+       if (!per_cpu(sd_ea, cpu)) {
+               if (sched_debug())
+                       pr_info("rd %*pbl: !sd_ea\n", cpumask_pr_args(cpu_map));
+               goto free;
+       }
 
        for_each_cpu(i, cpu_map) {
                /* Skip already covered CPUs. */
@@ -293,6 +313,18 @@ static void build_freq_domains(const struct cpumask *cpu_map)
                        goto free;
                tmp->next = fd;
                fd = tmp;
+
+               /* Count freq. doms and perf states for the complexity check. */
+               nr_fd++;
+               nr_cs += em_fd_nr_cap_states(fd->obj);
+       }
+
+       /* Bail out if the Energy Model complexity is too high. */
+       if (nr_fd * (nr_cs + nr_cpus) > EM_MAX_COMPLEXITY) {
+               if (sched_debug())
+                       pr_info("rd %*pbl: EM complexity is too high\n ",
+                                               cpumask_pr_args(cpu_map));
+               goto free;
        }
 
        freq_domain_debug(cpu_map, fd);
@@ -312,6 +344,44 @@ free:
        if (tmp)
                call_rcu(&tmp->rcu, destroy_freq_domain_rcu);
 }
+
+/*
+ * This static_key is set if at least one root domain meets all the following
+ * conditions:
+ *    1. all CPUs of the root domain are covered by the EM;
+ *    2. the EM complexity is low enough to keep scheduling overheads low;
+ *    3. the SD_ASYM_CPUCAPACITY flag is set in the sched_domain hierarchy.
+ */
+DEFINE_STATIC_KEY_FALSE(sched_energy_present);
+
+static void sched_energy_start(int ndoms_new, cpumask_var_t doms_new[])
+{
+       /*
+        * The conditions for EAS to start are checked during the creation of
+        * root domains. If one of them meets all conditions, it will have a
+        * non-null list of frequency domains.
+        */
+       while (ndoms_new) {
+               if (cpu_rq(cpumask_first(doms_new[ndoms_new - 1]))->rd->fd)
+                       goto enable;
+               ndoms_new--;
+       }
+
+       if (static_branch_unlikely(&sched_energy_present)) {
+               if (sched_debug())
+                       pr_info("%s: stopping EAS\n", __func__);
+               static_branch_disable_cpuslocked(&sched_energy_present);
+       }
+
+       return;
+
+enable:
+       if (!static_branch_unlikely(&sched_energy_present)) {
+               if (sched_debug())
+                       pr_info("%s: starting EAS\n", __func__);
+               static_branch_enable_cpuslocked(&sched_energy_present);
+       }
+}
 #else
 static void free_rd_fd(struct root_domain *rd) { }
 #endif
@@ -2046,6 +2116,7 @@ match2:
 match3:
                ;
        }
+       sched_energy_start(ndoms_new, doms_new);
 #endif
 
        /* Remember the new sched domains: */