cpuidle: Export the next timer expiration for CPUs

To be able to predict the sleep duration for a CPU entering idle, it
is essential to know the expiration time of the next timer.  Both the
teo and the menu cpuidle governors already use this information for
CPU idle state selection.

Moving forward, a similar prediction needs to be made for a group of
idle CPUs rather than for a single one and the following changes
implement a new genpd governor for that purpose.

In order to support that feature, add a new function called
tick_nohz_get_next_hrtimer() that will return the next hrtimer
expiration time of a given CPU to be invoked after deciding
whether or not to stop the scheduler tick on that CPU.

Make the cpuidle core call tick_nohz_get_next_hrtimer() right
before invoking the ->enter() callback provided by the cpuidle
driver for the given state and store its return value in the
per-CPU struct cpuidle_device, so as to make it available to code
outside of cpuidle.

Note that at the point when cpuidle calls tick_nohz_get_next_hrtimer(),
the governor's ->select() callback has already returned and indicated
whether or not the tick should be stopped, so in fact the value
returned by tick_nohz_get_next_hrtimer() always is the next hrtimer
expiration time for the given CPU, possibly including the tick (if
it hasn't been stopped).

Co-developed-by: Lina Iyer <lina.iyer@linaro.org>
Co-developed-by: Daniel Lezcano <daniel.lezcano@linaro.org>
Acked-by: Daniel Lezcano <daniel.lezcano@linaro.org>
Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
[ rjw: Subject & changelog ]
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>

diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 7f108309e871ea63fbc16b1b896e3ef4cd443216..0f4b7c45df3e103cba0a5a617b782cee522a44be 100644 (file)
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -328,9 +328,23 @@ int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
                  int index)
 {
+       int ret = 0;
+
+       /*
+        * Store the next hrtimer, which becomes either next tick or the next
+        * timer event, whatever expires first. Additionally, to make this data
+        * useful for consumers outside cpuidle, we rely on that the governor's
+        * ->select() callback have decided, whether to stop the tick or not.
+        */
+       WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());
+
        if (cpuidle_state_is_coupled(drv, index))
-               return cpuidle_enter_state_coupled(dev, drv, index);
-       return cpuidle_enter_state(dev, drv, index);
+               ret = cpuidle_enter_state_coupled(dev, drv, index);
+       else
+               ret = cpuidle_enter_state(dev, drv, index);
+
+       WRITE_ONCE(dev->next_hrtimer, 0);
+       return ret;
 }
 
 /**
@@ -511,6 +525,7 @@ static void __cpuidle_device_init(struct cpuidle_device *dev)
 {
        memset(dev->states_usage, 0, sizeof(dev->states_usage));
        dev->last_residency = 0;
+       dev->next_hrtimer = 0;
 }
 
 /**

diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
index 3b39472324a31c4580746300f348fa69ed8efdc3..bb9a0db89f1ab8e8d5387c93e4a98e81cba5da60 100644 (file)
--- a/include/linux/cpuidle.h
+++ b/include/linux/cpuidle.h
@@ -83,6 +83,7 @@ struct cpuidle_device {
        unsigned int            use_deepest_state:1;
        unsigned int            poll_time_limit:1;
        unsigned int            cpu;
+       ktime_t                 next_hrtimer;
 
        int                     last_residency;
        struct cpuidle_state_usage      states_usage[CPUIDLE_STATE_MAX];

diff --git a/include/linux/tick.h b/include/linux/tick.h
index 55388ab45fd4d474984beb5e4049471f8d29f646..8891b5ac3e4031887e4e98d74831bb0e2d71914d 100644 (file)
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -122,6 +122,7 @@ extern void tick_nohz_idle_enter(void);
 extern void tick_nohz_idle_exit(void);
 extern void tick_nohz_irq_exit(void);
 extern bool tick_nohz_idle_got_tick(void);
+extern ktime_t tick_nohz_get_next_hrtimer(void);
 extern ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next);
 extern unsigned long tick_nohz_get_idle_calls(void);
 extern unsigned long tick_nohz_get_idle_calls_cpu(int cpu);
@@ -145,7 +146,11 @@ static inline void tick_nohz_idle_restart_tick(void) { }
 static inline void tick_nohz_idle_enter(void) { }
 static inline void tick_nohz_idle_exit(void) { }
 static inline bool tick_nohz_idle_got_tick(void) { return false; }
-
+static inline ktime_t tick_nohz_get_next_hrtimer(void)
+{
+       /* Next wake up is the tick period, assume it starts now */
+       return ktime_add(ktime_get(), TICK_NSEC);
+}
 static inline ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
 {
        *delta_next = TICK_NSEC;

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 6fa52cd6df0be2f53c8e79bf0e8cf16bcf0310f0..8d18e03124ff4ce2d629b6f81951626ae1e76d86 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -1022,6 +1022,18 @@ bool tick_nohz_idle_got_tick(void)
        return false;
 }
 
+/**
+ * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
+ * or the tick, whatever that expires first. Note that, if the tick has been
+ * stopped, it returns the next hrtimer.
+ *
+ * Called from power state control code with interrupts disabled
+ */
+ktime_t tick_nohz_get_next_hrtimer(void)
+{
+       return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
+}
+
 /**
  * tick_nohz_get_sleep_length - return the expected length of the current sleep
  * @delta_next: duration until the next event if the tick cannot be stopped