static inline void idle_loop_prolog(unsigned long *in_purr, ktime_t *kt_before)
{
- *kt_before = ktime_get_real();
+ *kt_before = ktime_get();
*in_purr = mfspr(SPRN_PURR);
/*
* Indicate to the HV that we are idle. Now would be
get_lppaca()->wait_state_cycles += mfspr(SPRN_PURR) - in_purr;
get_lppaca()->idle = 0;
- return ktime_to_us(ktime_sub(ktime_get_real(), kt_before));
+ return ktime_to_us(ktime_sub(ktime_get(), kt_before));
}
static int snooze_loop(struct cpuidle_device *dev,
static int acpi_idle_enter_c1(struct cpuidle_device *dev,
struct cpuidle_driver *drv, int index)
{
- ktime_t kt1, kt2;
- s64 idle_time;
struct acpi_processor *pr;
struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
struct acpi_processor_cx *cx = cpuidle_get_statedata(state_usage);
pr = __this_cpu_read(processors);
- dev->last_residency = 0;
if (unlikely(!pr))
return -EINVAL;
- local_irq_disable();
-
-
lapic_timer_state_broadcast(pr, cx, 1);
- kt1 = ktime_get_real();
acpi_idle_do_entry(cx);
- kt2 = ktime_get_real();
- idle_time = ktime_to_us(ktime_sub(kt2, kt1));
-
- /* Update device last_residency*/
- dev->last_residency = (int)idle_time;
- local_irq_enable();
lapic_timer_state_broadcast(pr, cx, 0);
return index;
struct acpi_processor *pr;
struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
struct acpi_processor_cx *cx = cpuidle_get_statedata(state_usage);
- ktime_t kt1, kt2;
- s64 idle_time_ns;
- s64 idle_time;
pr = __this_cpu_read(processors);
- dev->last_residency = 0;
if (unlikely(!pr))
return -EINVAL;
- local_irq_disable();
-
-
if (cx->entry_method != ACPI_CSTATE_FFH) {
current_thread_info()->status &= ~TS_POLLING;
/*
if (unlikely(need_resched())) {
current_thread_info()->status |= TS_POLLING;
- local_irq_enable();
return -EINVAL;
}
}
if (cx->type == ACPI_STATE_C3)
ACPI_FLUSH_CPU_CACHE();
- kt1 = ktime_get_real();
/* Tell the scheduler that we are going deep-idle: */
sched_clock_idle_sleep_event();
acpi_idle_do_entry(cx);
- kt2 = ktime_get_real();
- idle_time_ns = ktime_to_ns(ktime_sub(kt2, kt1));
- idle_time = idle_time_ns;
- do_div(idle_time, NSEC_PER_USEC);
- /* Update device last_residency*/
- dev->last_residency = (int)idle_time;
+ sched_clock_idle_wakeup_event(0);
- /* Tell the scheduler how much we idled: */
- sched_clock_idle_wakeup_event(idle_time_ns);
-
- local_irq_enable();
if (cx->entry_method != ACPI_CSTATE_FFH)
current_thread_info()->status |= TS_POLLING;
struct acpi_processor *pr;
struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
struct acpi_processor_cx *cx = cpuidle_get_statedata(state_usage);
- ktime_t kt1, kt2;
- s64 idle_time_ns;
- s64 idle_time;
-
pr = __this_cpu_read(processors);
- dev->last_residency = 0;
if (unlikely(!pr))
return -EINVAL;
return drv->states[drv->safe_state_index].enter(dev,
drv, drv->safe_state_index);
} else {
- local_irq_disable();
acpi_safe_halt();
- local_irq_enable();
return -EBUSY;
}
}
- local_irq_disable();
-
-
if (cx->entry_method != ACPI_CSTATE_FFH) {
current_thread_info()->status &= ~TS_POLLING;
/*
if (unlikely(need_resched())) {
current_thread_info()->status |= TS_POLLING;
- local_irq_enable();
return -EINVAL;
}
}
*/
lapic_timer_state_broadcast(pr, cx, 1);
- kt1 = ktime_get_real();
/*
* disable bus master
* bm_check implies we need ARB_DIS
c3_cpu_count--;
raw_spin_unlock(&c3_lock);
}
- kt2 = ktime_get_real();
- idle_time_ns = ktime_to_ns(ktime_sub(kt2, kt1));
- idle_time = idle_time_ns;
- do_div(idle_time, NSEC_PER_USEC);
-
- /* Update device last_residency*/
- dev->last_residency = (int)idle_time;
- /* Tell the scheduler how much we idled: */
- sched_clock_idle_wakeup_event(idle_time_ns);
+ sched_clock_idle_wakeup_event(0);
- local_irq_enable();
if (cx->entry_method != ACPI_CSTATE_FFH)
current_thread_info()->status |= TS_POLLING;
struct cpuidle_driver acpi_idle_driver = {
.name = "acpi_idle",
.owner = THIS_MODULE,
+ .en_core_tk_irqen = 1,
};
/**
If you're using an ACPI-enabled platform, you should say Y here.
+config CPU_IDLE_MULTIPLE_DRIVERS
+ bool "Support multiple cpuidle drivers"
+ depends on CPU_IDLE
+ default n
+ help
+ Allows the cpuidle framework to use different drivers for each CPU.
+ This is useful if you have a system with different CPU latencies and
+ states. If unsure say N.
+
config CPU_IDLE_GOV_LADDER
bool
depends on CPU_IDLE
int cpuidle_play_dead(void)
{
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
- struct cpuidle_driver *drv = cpuidle_get_driver();
+ struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
int i, dead_state = -1;
int power_usage = -1;
/* This can be moved to within driver enter routine
* but that results in multiple copies of same code.
*/
- dev->states_usage[entered_state].time +=
- (unsigned long long)dev->last_residency;
+ dev->states_usage[entered_state].time += dev->last_residency;
dev->states_usage[entered_state].usage++;
} else {
dev->last_residency = 0;
int cpuidle_idle_call(void)
{
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
- struct cpuidle_driver *drv = cpuidle_get_driver();
+ struct cpuidle_driver *drv;
int next_state, entered_state;
if (off)
if (!dev || !dev->enabled)
return -EBUSY;
+ drv = cpuidle_get_cpu_driver(dev);
+
/* ask the governor for the next state */
next_state = cpuidle_curr_governor->select(drv, dev);
if (need_resched()) {
+ dev->last_residency = 0;
+ /* give the governor an opportunity to reflect on the outcome */
+ if (cpuidle_curr_governor->reflect)
+ cpuidle_curr_governor->reflect(dev, next_state);
local_irq_enable();
return 0;
}
int cpuidle_enable_device(struct cpuidle_device *dev)
{
int ret, i;
- struct cpuidle_driver *drv = cpuidle_get_driver();
+ struct cpuidle_driver *drv;
if (!dev)
return -EINVAL;
if (dev->enabled)
return 0;
+
+ drv = cpuidle_get_cpu_driver(dev);
+
if (!drv || !cpuidle_curr_governor)
return -EIO;
+
if (!dev->state_count)
dev->state_count = drv->state_count;
poll_idle_init(drv);
- if ((ret = cpuidle_add_state_sysfs(dev)))
+ ret = cpuidle_add_device_sysfs(dev);
+ if (ret)
return ret;
if (cpuidle_curr_governor->enable &&
return 0;
fail_sysfs:
- cpuidle_remove_state_sysfs(dev);
+ cpuidle_remove_device_sysfs(dev);
return ret;
}
*/
void cpuidle_disable_device(struct cpuidle_device *dev)
{
+ struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
+
if (!dev || !dev->enabled)
return;
- if (!cpuidle_get_driver() || !cpuidle_curr_governor)
+
+ if (!drv || !cpuidle_curr_governor)
return;
dev->enabled = 0;
if (cpuidle_curr_governor->disable)
- cpuidle_curr_governor->disable(cpuidle_get_driver(), dev);
+ cpuidle_curr_governor->disable(drv, dev);
- cpuidle_remove_state_sysfs(dev);
+ cpuidle_remove_device_sysfs(dev);
enabled_devices--;
}
static int __cpuidle_register_device(struct cpuidle_device *dev)
{
int ret;
- struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
- struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
+ struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
- if (!try_module_get(cpuidle_driver->owner))
+ if (!try_module_get(drv->owner))
return -EINVAL;
- init_completion(&dev->kobj_unregister);
-
per_cpu(cpuidle_devices, dev->cpu) = dev;
list_add(&dev->device_list, &cpuidle_detected_devices);
- ret = cpuidle_add_sysfs(cpu_dev);
+ ret = cpuidle_add_sysfs(dev);
if (ret)
goto err_sysfs;
return 0;
err_coupled:
- cpuidle_remove_sysfs(cpu_dev);
- wait_for_completion(&dev->kobj_unregister);
+ cpuidle_remove_sysfs(dev);
err_sysfs:
list_del(&dev->device_list);
per_cpu(cpuidle_devices, dev->cpu) = NULL;
- module_put(cpuidle_driver->owner);
+ module_put(drv->owner);
return ret;
}
*/
void cpuidle_unregister_device(struct cpuidle_device *dev)
{
- struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
- struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
+ struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
if (dev->registered == 0)
return;
cpuidle_disable_device(dev);
- cpuidle_remove_sysfs(cpu_dev);
+ cpuidle_remove_sysfs(dev);
list_del(&dev->device_list);
- wait_for_completion(&dev->kobj_unregister);
per_cpu(cpuidle_devices, dev->cpu) = NULL;
cpuidle_coupled_unregister_device(dev);
cpuidle_resume_and_unlock();
- module_put(cpuidle_driver->owner);
+ module_put(drv->owner);
}
EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
#ifndef __DRIVER_CPUIDLE_H
#define __DRIVER_CPUIDLE_H
-#include <linux/device.h>
-
/* For internal use only */
extern struct cpuidle_governor *cpuidle_curr_governor;
extern struct list_head cpuidle_governors;
extern int cpuidle_switch_governor(struct cpuidle_governor *gov);
/* sysfs */
+
+struct device;
+
extern int cpuidle_add_interface(struct device *dev);
extern void cpuidle_remove_interface(struct device *dev);
-extern int cpuidle_add_state_sysfs(struct cpuidle_device *device);
-extern void cpuidle_remove_state_sysfs(struct cpuidle_device *device);
-extern int cpuidle_add_sysfs(struct device *dev);
-extern void cpuidle_remove_sysfs(struct device *dev);
+extern int cpuidle_add_device_sysfs(struct cpuidle_device *device);
+extern void cpuidle_remove_device_sysfs(struct cpuidle_device *device);
+extern int cpuidle_add_sysfs(struct cpuidle_device *dev);
+extern void cpuidle_remove_sysfs(struct cpuidle_device *dev);
#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
bool cpuidle_state_is_coupled(struct cpuidle_device *dev,
#include "cpuidle.h"
-static struct cpuidle_driver *cpuidle_curr_driver;
DEFINE_SPINLOCK(cpuidle_driver_lock);
-int cpuidle_driver_refcount;
+
+static void __cpuidle_set_cpu_driver(struct cpuidle_driver *drv, int cpu);
+static struct cpuidle_driver * __cpuidle_get_cpu_driver(int cpu);
static void set_power_states(struct cpuidle_driver *drv)
{
drv->states[i].power_usage = -1 - i;
}
-/**
- * cpuidle_register_driver - registers a driver
- * @drv: the driver
- */
-int cpuidle_register_driver(struct cpuidle_driver *drv)
+static void __cpuidle_driver_init(struct cpuidle_driver *drv)
+{
+ drv->refcnt = 0;
+
+ if (!drv->power_specified)
+ set_power_states(drv);
+}
+
+static int __cpuidle_register_driver(struct cpuidle_driver *drv, int cpu)
{
if (!drv || !drv->state_count)
return -EINVAL;
if (cpuidle_disabled())
return -ENODEV;
- spin_lock(&cpuidle_driver_lock);
- if (cpuidle_curr_driver) {
- spin_unlock(&cpuidle_driver_lock);
+ if (__cpuidle_get_cpu_driver(cpu))
return -EBUSY;
+
+ __cpuidle_driver_init(drv);
+
+ __cpuidle_set_cpu_driver(drv, cpu);
+
+ return 0;
+}
+
+static void __cpuidle_unregister_driver(struct cpuidle_driver *drv, int cpu)
+{
+ if (drv != __cpuidle_get_cpu_driver(cpu))
+ return;
+
+ if (!WARN_ON(drv->refcnt > 0))
+ __cpuidle_set_cpu_driver(NULL, cpu);
+}
+
+#ifdef CONFIG_CPU_IDLE_MULTIPLE_DRIVERS
+
+static DEFINE_PER_CPU(struct cpuidle_driver *, cpuidle_drivers);
+
+static void __cpuidle_set_cpu_driver(struct cpuidle_driver *drv, int cpu)
+{
+ per_cpu(cpuidle_drivers, cpu) = drv;
+}
+
+static struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu)
+{
+ return per_cpu(cpuidle_drivers, cpu);
+}
+
+static void __cpuidle_unregister_all_cpu_driver(struct cpuidle_driver *drv)
+{
+ int cpu;
+ for_each_present_cpu(cpu)
+ __cpuidle_unregister_driver(drv, cpu);
+}
+
+static int __cpuidle_register_all_cpu_driver(struct cpuidle_driver *drv)
+{
+ int ret = 0;
+ int i, cpu;
+
+ for_each_present_cpu(cpu) {
+ ret = __cpuidle_register_driver(drv, cpu);
+ if (ret)
+ break;
}
- if (!drv->power_specified)
- set_power_states(drv);
+ if (ret)
+ for_each_present_cpu(i) {
+ if (i == cpu)
+ break;
+ __cpuidle_unregister_driver(drv, i);
+ }
- cpuidle_curr_driver = drv;
+ return ret;
+}
+
+int cpuidle_register_cpu_driver(struct cpuidle_driver *drv, int cpu)
+{
+ int ret;
+
+ spin_lock(&cpuidle_driver_lock);
+ ret = __cpuidle_register_driver(drv, cpu);
spin_unlock(&cpuidle_driver_lock);
- return 0;
+ return ret;
+}
+
+void cpuidle_unregister_cpu_driver(struct cpuidle_driver *drv, int cpu)
+{
+ spin_lock(&cpuidle_driver_lock);
+ __cpuidle_unregister_driver(drv, cpu);
+ spin_unlock(&cpuidle_driver_lock);
+}
+
+/**
+ * cpuidle_register_driver - registers a driver
+ * @drv: the driver
+ */
+int cpuidle_register_driver(struct cpuidle_driver *drv)
+{
+ int ret;
+
+ spin_lock(&cpuidle_driver_lock);
+ ret = __cpuidle_register_all_cpu_driver(drv);
+ spin_unlock(&cpuidle_driver_lock);
+
+ return ret;
}
EXPORT_SYMBOL_GPL(cpuidle_register_driver);
/**
- * cpuidle_get_driver - return the current driver
+ * cpuidle_unregister_driver - unregisters a driver
+ * @drv: the driver
*/
-struct cpuidle_driver *cpuidle_get_driver(void)
+void cpuidle_unregister_driver(struct cpuidle_driver *drv)
+{
+ spin_lock(&cpuidle_driver_lock);
+ __cpuidle_unregister_all_cpu_driver(drv);
+ spin_unlock(&cpuidle_driver_lock);
+}
+EXPORT_SYMBOL_GPL(cpuidle_unregister_driver);
+
+#else
+
+static struct cpuidle_driver *cpuidle_curr_driver;
+
+static inline void __cpuidle_set_cpu_driver(struct cpuidle_driver *drv, int cpu)
+{
+ cpuidle_curr_driver = drv;
+}
+
+static inline struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu)
{
return cpuidle_curr_driver;
}
-EXPORT_SYMBOL_GPL(cpuidle_get_driver);
+
+/**
+ * cpuidle_register_driver - registers a driver
+ * @drv: the driver
+ */
+int cpuidle_register_driver(struct cpuidle_driver *drv)
+{
+ int ret, cpu;
+
+ cpu = get_cpu();
+ spin_lock(&cpuidle_driver_lock);
+ ret = __cpuidle_register_driver(drv, cpu);
+ spin_unlock(&cpuidle_driver_lock);
+ put_cpu();
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(cpuidle_register_driver);
/**
* cpuidle_unregister_driver - unregisters a driver
*/
void cpuidle_unregister_driver(struct cpuidle_driver *drv)
{
- if (drv != cpuidle_curr_driver) {
- WARN(1, "invalid cpuidle_unregister_driver(%s)\n",
- drv->name);
- return;
- }
+ int cpu;
+ cpu = get_cpu();
spin_lock(&cpuidle_driver_lock);
+ __cpuidle_unregister_driver(drv, cpu);
+ spin_unlock(&cpuidle_driver_lock);
+ put_cpu();
+}
+EXPORT_SYMBOL_GPL(cpuidle_unregister_driver);
+#endif
+
+/**
+ * cpuidle_get_driver - return the current driver
+ */
+struct cpuidle_driver *cpuidle_get_driver(void)
+{
+ struct cpuidle_driver *drv;
+ int cpu;
- if (!WARN_ON(cpuidle_driver_refcount > 0))
- cpuidle_curr_driver = NULL;
+ cpu = get_cpu();
+ drv = __cpuidle_get_cpu_driver(cpu);
+ put_cpu();
+ return drv;
+}
+EXPORT_SYMBOL_GPL(cpuidle_get_driver);
+
+/**
+ * cpuidle_get_cpu_driver - return the driver tied with a cpu
+ */
+struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev)
+{
+ struct cpuidle_driver *drv;
+
+ if (!dev)
+ return NULL;
+
+ spin_lock(&cpuidle_driver_lock);
+ drv = __cpuidle_get_cpu_driver(dev->cpu);
spin_unlock(&cpuidle_driver_lock);
+
+ return drv;
}
-EXPORT_SYMBOL_GPL(cpuidle_unregister_driver);
+EXPORT_SYMBOL_GPL(cpuidle_get_cpu_driver);
struct cpuidle_driver *cpuidle_driver_ref(void)
{
spin_lock(&cpuidle_driver_lock);
- drv = cpuidle_curr_driver;
- cpuidle_driver_refcount++;
+ drv = cpuidle_get_driver();
+ drv->refcnt++;
spin_unlock(&cpuidle_driver_lock);
return drv;
void cpuidle_driver_unref(void)
{
+ struct cpuidle_driver *drv = cpuidle_get_driver();
+
spin_lock(&cpuidle_driver_lock);
- if (!WARN_ON(cpuidle_driver_refcount <= 0))
- cpuidle_driver_refcount--;
+ if (drv && !WARN_ON(drv->refcnt <= 0))
+ drv->refcnt--;
spin_unlock(&cpuidle_driver_lock);
}
#define MAX_INTERESTING 50000
#define STDDEV_THRESH 400
+/* 60 * 60 > STDDEV_THRESH * INTERVALS = 400 * 8 */
+#define MAX_DEVIATION 60
+
+static DEFINE_PER_CPU(struct hrtimer, menu_hrtimer);
+static DEFINE_PER_CPU(int, hrtimer_status);
+/* menu hrtimer mode */
+enum {MENU_HRTIMER_STOP, MENU_HRTIMER_REPEAT, MENU_HRTIMER_GENERAL};
/*
* Concepts and ideas behind the menu governor
*
*/
+/*
+ * The C-state residency is so long that is is worthwhile to exit
+ * from the shallow C-state and re-enter into a deeper C-state.
+ */
+static unsigned int perfect_cstate_ms __read_mostly = 30;
+module_param(perfect_cstate_ms, uint, 0000);
+
struct menu_device {
int last_state_idx;
int needs_update;
return div_u64(dividend + (divisor / 2), divisor);
}
+/* Cancel the hrtimer if it is not triggered yet */
+void menu_hrtimer_cancel(void)
+{
+ int cpu = smp_processor_id();
+ struct hrtimer *hrtmr = &per_cpu(menu_hrtimer, cpu);
+
+ /* The timer is still not time out*/
+ if (per_cpu(hrtimer_status, cpu)) {
+ hrtimer_cancel(hrtmr);
+ per_cpu(hrtimer_status, cpu) = MENU_HRTIMER_STOP;
+ }
+}
+EXPORT_SYMBOL_GPL(menu_hrtimer_cancel);
+
+/* Call back for hrtimer is triggered */
+static enum hrtimer_restart menu_hrtimer_notify(struct hrtimer *hrtimer)
+{
+ int cpu = smp_processor_id();
+ struct menu_device *data = &per_cpu(menu_devices, cpu);
+
+ /* In general case, the expected residency is much larger than
+ * deepest C-state target residency, but prediction logic still
+ * predicts a small predicted residency, so the prediction
+ * history is totally broken if the timer is triggered.
+ * So reset the correction factor.
+ */
+ if (per_cpu(hrtimer_status, cpu) == MENU_HRTIMER_GENERAL)
+ data->correction_factor[data->bucket] = RESOLUTION * DECAY;
+
+ per_cpu(hrtimer_status, cpu) = MENU_HRTIMER_STOP;
+
+ return HRTIMER_NORESTART;
+}
+
/*
* Try detecting repeating patterns by keeping track of the last 8
* intervals, and checking if the standard deviation of that set
* of points is below a threshold. If it is... then use the
* average of these 8 points as the estimated value.
*/
-static void detect_repeating_patterns(struct menu_device *data)
+static u32 get_typical_interval(struct menu_device *data)
{
- int i;
- uint64_t avg = 0;
- uint64_t stddev = 0; /* contains the square of the std deviation */
-
- /* first calculate average and standard deviation of the past */
- for (i = 0; i < INTERVALS; i++)
- avg += data->intervals[i];
- avg = avg / INTERVALS;
+ int i = 0, divisor = 0;
+ uint64_t max = 0, avg = 0, stddev = 0;
+ int64_t thresh = LLONG_MAX; /* Discard outliers above this value. */
+ unsigned int ret = 0;
- /* if the avg is beyond the known next tick, it's worthless */
- if (avg > data->expected_us)
- return;
+again:
- for (i = 0; i < INTERVALS; i++)
- stddev += (data->intervals[i] - avg) *
- (data->intervals[i] - avg);
-
- stddev = stddev / INTERVALS;
+ /* first calculate average and standard deviation of the past */
+ max = avg = divisor = stddev = 0;
+ for (i = 0; i < INTERVALS; i++) {
+ int64_t value = data->intervals[i];
+ if (value <= thresh) {
+ avg += value;
+ divisor++;
+ if (value > max)
+ max = value;
+ }
+ }
+ do_div(avg, divisor);
+ for (i = 0; i < INTERVALS; i++) {
+ int64_t value = data->intervals[i];
+ if (value <= thresh) {
+ int64_t diff = value - avg;
+ stddev += diff * diff;
+ }
+ }
+ do_div(stddev, divisor);
+ stddev = int_sqrt(stddev);
/*
- * now.. if stddev is small.. then assume we have a
- * repeating pattern and predict we keep doing this.
+ * If we have outliers to the upside in our distribution, discard
+ * those by setting the threshold to exclude these outliers, then
+ * calculate the average and standard deviation again. Once we get
+ * down to the bottom 3/4 of our samples, stop excluding samples.
+ *
+ * This can deal with workloads that have long pauses interspersed
+ * with sporadic activity with a bunch of short pauses.
+ *
+ * The typical interval is obtained when standard deviation is small
+ * or standard deviation is small compared to the average interval.
*/
-
- if (avg && stddev < STDDEV_THRESH)
+ if (((avg > stddev * 6) && (divisor * 4 >= INTERVALS * 3))
+ || stddev <= 20) {
data->predicted_us = avg;
+ ret = 1;
+ return ret;
+
+ } else if ((divisor * 4) > INTERVALS * 3) {
+ /* Exclude the max interval */
+ thresh = max - 1;
+ goto again;
+ }
+
+ return ret;
}
/**
int i;
int multiplier;
struct timespec t;
+ int repeat = 0, low_predicted = 0;
+ int cpu = smp_processor_id();
+ struct hrtimer *hrtmr = &per_cpu(menu_hrtimer, cpu);
if (data->needs_update) {
menu_update(drv, dev);
data->predicted_us = div_round64(data->expected_us * data->correction_factor[data->bucket],
RESOLUTION * DECAY);
- detect_repeating_patterns(data);
+ repeat = get_typical_interval(data);
/*
* We want to default to C1 (hlt), not to busy polling
if (s->disabled || su->disable)
continue;
- if (s->target_residency > data->predicted_us)
+ if (s->target_residency > data->predicted_us) {
+ low_predicted = 1;
continue;
+ }
if (s->exit_latency > latency_req)
continue;
if (s->exit_latency * multiplier > data->predicted_us)
}
}
+ /* not deepest C-state chosen for low predicted residency */
+ if (low_predicted) {
+ unsigned int timer_us = 0;
+ unsigned int perfect_us = 0;
+
+ /*
+ * Set a timer to detect whether this sleep is much
+ * longer than repeat mode predicted. If the timer
+ * triggers, the code will evaluate whether to put
+ * the CPU into a deeper C-state.
+ * The timer is cancelled on CPU wakeup.
+ */
+ timer_us = 2 * (data->predicted_us + MAX_DEVIATION);
+
+ perfect_us = perfect_cstate_ms * 1000;
+
+ if (repeat && (4 * timer_us < data->expected_us)) {
+ RCU_NONIDLE(hrtimer_start(hrtmr,
+ ns_to_ktime(1000 * timer_us),
+ HRTIMER_MODE_REL_PINNED));
+ /* In repeat case, menu hrtimer is started */
+ per_cpu(hrtimer_status, cpu) = MENU_HRTIMER_REPEAT;
+ } else if (perfect_us < data->expected_us) {
+ /*
+ * The next timer is long. This could be because
+ * we did not make a useful prediction.
+ * In that case, it makes sense to re-enter
+ * into a deeper C-state after some time.
+ */
+ RCU_NONIDLE(hrtimer_start(hrtmr,
+ ns_to_ktime(1000 * timer_us),
+ HRTIMER_MODE_REL_PINNED));
+ /* In general case, menu hrtimer is started */
+ per_cpu(hrtimer_status, cpu) = MENU_HRTIMER_GENERAL;
+ }
+
+ }
+
return data->last_state_idx;
}
struct cpuidle_device *dev)
{
struct menu_device *data = &per_cpu(menu_devices, dev->cpu);
+ struct hrtimer *t = &per_cpu(menu_hrtimer, dev->cpu);
+ hrtimer_init(t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ t->function = menu_hrtimer_notify;
memset(data, 0, sizeof(struct menu_device));
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/capability.h>
+#include <linux/device.h>
#include "cpuidle.h"
NULL
};
+struct cpuidle_state_kobj {
+ struct cpuidle_state *state;
+ struct cpuidle_state_usage *state_usage;
+ struct completion kobj_unregister;
+ struct kobject kobj;
+};
+
#define kobj_to_state_obj(k) container_of(k, struct cpuidle_state_kobj, kobj)
#define kobj_to_state(k) (kobj_to_state_obj(k)->state)
#define kobj_to_state_usage(k) (kobj_to_state_obj(k)->state_usage)
}
/**
- * cpuidle_add_driver_sysfs - adds driver-specific sysfs attributes
+ * cpuidle_add_state_sysfs - adds cpuidle states sysfs attributes
* @device: the target device
*/
-int cpuidle_add_state_sysfs(struct cpuidle_device *device)
+static int cpuidle_add_state_sysfs(struct cpuidle_device *device)
{
int i, ret = -ENOMEM;
struct cpuidle_state_kobj *kobj;
- struct cpuidle_driver *drv = cpuidle_get_driver();
+ struct cpuidle_driver *drv = cpuidle_get_cpu_driver(device);
/* state statistics */
- for (i = 0; i < device->state_count; i++) {
+ for (i = 0; i < drv->state_count; i++) {
kobj = kzalloc(sizeof(struct cpuidle_state_kobj), GFP_KERNEL);
if (!kobj)
goto error_state;
kobj->state_usage = &device->states_usage[i];
init_completion(&kobj->kobj_unregister);
- ret = kobject_init_and_add(&kobj->kobj, &ktype_state_cpuidle, &device->kobj,
- "state%d", i);
+ ret = kobject_init_and_add(&kobj->kobj, &ktype_state_cpuidle,
+ &device->kobj, "state%d", i);
if (ret) {
kfree(kobj);
goto error_state;
}
/**
- * cpuidle_remove_driver_sysfs - removes driver-specific sysfs attributes
+ * cpuidle_remove_driver_sysfs - removes the cpuidle states sysfs attributes
* @device: the target device
*/
-void cpuidle_remove_state_sysfs(struct cpuidle_device *device)
+static void cpuidle_remove_state_sysfs(struct cpuidle_device *device)
{
int i;
cpuidle_free_state_kobj(device, i);
}
+#ifdef CONFIG_CPU_IDLE_MULTIPLE_DRIVERS
+#define kobj_to_driver_kobj(k) container_of(k, struct cpuidle_driver_kobj, kobj)
+#define attr_to_driver_attr(a) container_of(a, struct cpuidle_driver_attr, attr)
+
+#define define_one_driver_ro(_name, show) \
+ static struct cpuidle_driver_attr attr_driver_##_name = \
+ __ATTR(_name, 0644, show, NULL)
+
+struct cpuidle_driver_kobj {
+ struct cpuidle_driver *drv;
+ struct completion kobj_unregister;
+ struct kobject kobj;
+};
+
+struct cpuidle_driver_attr {
+ struct attribute attr;
+ ssize_t (*show)(struct cpuidle_driver *, char *);
+ ssize_t (*store)(struct cpuidle_driver *, const char *, size_t);
+};
+
+static ssize_t show_driver_name(struct cpuidle_driver *drv, char *buf)
+{
+ ssize_t ret;
+
+ spin_lock(&cpuidle_driver_lock);
+ ret = sprintf(buf, "%s\n", drv ? drv->name : "none");
+ spin_unlock(&cpuidle_driver_lock);
+
+ return ret;
+}
+
+static void cpuidle_driver_sysfs_release(struct kobject *kobj)
+{
+ struct cpuidle_driver_kobj *driver_kobj = kobj_to_driver_kobj(kobj);
+ complete(&driver_kobj->kobj_unregister);
+}
+
+static ssize_t cpuidle_driver_show(struct kobject *kobj, struct attribute * attr,
+ char * buf)
+{
+ int ret = -EIO;
+ struct cpuidle_driver_kobj *driver_kobj = kobj_to_driver_kobj(kobj);
+ struct cpuidle_driver_attr *dattr = attr_to_driver_attr(attr);
+
+ if (dattr->show)
+ ret = dattr->show(driver_kobj->drv, buf);
+
+ return ret;
+}
+
+static ssize_t cpuidle_driver_store(struct kobject *kobj, struct attribute *attr,
+ const char *buf, size_t size)
+{
+ int ret = -EIO;
+ struct cpuidle_driver_kobj *driver_kobj = kobj_to_driver_kobj(kobj);
+ struct cpuidle_driver_attr *dattr = attr_to_driver_attr(attr);
+
+ if (dattr->store)
+ ret = dattr->store(driver_kobj->drv, buf, size);
+
+ return ret;
+}
+
+define_one_driver_ro(name, show_driver_name);
+
+static const struct sysfs_ops cpuidle_driver_sysfs_ops = {
+ .show = cpuidle_driver_show,
+ .store = cpuidle_driver_store,
+};
+
+static struct attribute *cpuidle_driver_default_attrs[] = {
+ &attr_driver_name.attr,
+ NULL
+};
+
+static struct kobj_type ktype_driver_cpuidle = {
+ .sysfs_ops = &cpuidle_driver_sysfs_ops,
+ .default_attrs = cpuidle_driver_default_attrs,
+ .release = cpuidle_driver_sysfs_release,
+};
+
+/**
+ * cpuidle_add_driver_sysfs - adds the driver name sysfs attribute
+ * @device: the target device
+ */
+static int cpuidle_add_driver_sysfs(struct cpuidle_device *dev)
+{
+ struct cpuidle_driver_kobj *kdrv;
+ struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
+ int ret;
+
+ kdrv = kzalloc(sizeof(*kdrv), GFP_KERNEL);
+ if (!kdrv)
+ return -ENOMEM;
+
+ kdrv->drv = drv;
+ init_completion(&kdrv->kobj_unregister);
+
+ ret = kobject_init_and_add(&kdrv->kobj, &ktype_driver_cpuidle,
+ &dev->kobj, "driver");
+ if (ret) {
+ kfree(kdrv);
+ return ret;
+ }
+
+ kobject_uevent(&kdrv->kobj, KOBJ_ADD);
+ dev->kobj_driver = kdrv;
+
+ return ret;
+}
+
+/**
+ * cpuidle_remove_driver_sysfs - removes the driver name sysfs attribute
+ * @device: the target device
+ */
+static void cpuidle_remove_driver_sysfs(struct cpuidle_device *dev)
+{
+ struct cpuidle_driver_kobj *kdrv = dev->kobj_driver;
+ kobject_put(&kdrv->kobj);
+ wait_for_completion(&kdrv->kobj_unregister);
+ kfree(kdrv);
+}
+#else
+static inline int cpuidle_add_driver_sysfs(struct cpuidle_device *dev)
+{
+ return 0;
+}
+
+static inline void cpuidle_remove_driver_sysfs(struct cpuidle_device *dev)
+{
+ ;
+}
+#endif
+
+/**
+ * cpuidle_add_device_sysfs - adds device specific sysfs attributes
+ * @device: the target device
+ */
+int cpuidle_add_device_sysfs(struct cpuidle_device *device)
+{
+ int ret;
+
+ ret = cpuidle_add_state_sysfs(device);
+ if (ret)
+ return ret;
+
+ ret = cpuidle_add_driver_sysfs(device);
+ if (ret)
+ cpuidle_remove_state_sysfs(device);
+ return ret;
+}
+
+/**
+ * cpuidle_remove_device_sysfs : removes device specific sysfs attributes
+ * @device : the target device
+ */
+void cpuidle_remove_device_sysfs(struct cpuidle_device *device)
+{
+ cpuidle_remove_driver_sysfs(device);
+ cpuidle_remove_state_sysfs(device);
+}
+
/**
* cpuidle_add_sysfs - creates a sysfs instance for the target device
* @dev: the target device
*/
-int cpuidle_add_sysfs(struct device *cpu_dev)
+int cpuidle_add_sysfs(struct cpuidle_device *dev)
{
- int cpu = cpu_dev->id;
- struct cpuidle_device *dev;
+ struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
int error;
- dev = per_cpu(cpuidle_devices, cpu);
+ init_completion(&dev->kobj_unregister);
+
error = kobject_init_and_add(&dev->kobj, &ktype_cpuidle, &cpu_dev->kobj,
"cpuidle");
if (!error)
* cpuidle_remove_sysfs - deletes a sysfs instance on the target device
* @dev: the target device
*/
-void cpuidle_remove_sysfs(struct device *cpu_dev)
+void cpuidle_remove_sysfs(struct cpuidle_device *dev)
{
- int cpu = cpu_dev->id;
- struct cpuidle_device *dev;
-
- dev = per_cpu(cpuidle_devices, cpu);
kobject_put(&dev->kobj);
+ wait_for_completion(&dev->kobj_unregister);
}
#include <linux/kernel.h>
#include <linux/cpuidle.h>
#include <linux/clockchips.h>
-#include <linux/hrtimer.h> /* ktime_get_real() */
#include <trace/events/power.h>
#include <linux/sched.h>
#include <linux/notifier.h>
static struct cpuidle_driver intel_idle_driver = {
.name = "intel_idle",
.owner = THIS_MODULE,
+ .en_core_tk_irqen = 1,
};
/* intel_idle.max_cstate=0 disables driver */
static int max_cstate = MWAIT_MAX_NUM_CSTATES - 1;
struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
unsigned long eax = (unsigned long)cpuidle_get_statedata(state_usage);
unsigned int cstate;
- ktime_t kt_before, kt_after;
- s64 usec_delta;
int cpu = smp_processor_id();
cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
if (!(lapic_timer_reliable_states & (1 << (cstate))))
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
- kt_before = ktime_get_real();
-
stop_critical_timings();
if (!need_resched()) {
start_critical_timings();
- kt_after = ktime_get_real();
- usec_delta = ktime_to_us(ktime_sub(kt_after, kt_before));
-
- local_irq_enable();
-
if (!(lapic_timer_reliable_states & (1 << (cstate))))
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
- /* Update cpuidle counters */
- dev->last_residency = (int)usec_delta;
-
return index;
}
st_usage->driver_data = data;
}
-struct cpuidle_state_kobj {
- struct cpuidle_state *state;
- struct cpuidle_state_usage *state_usage;
- struct completion kobj_unregister;
- struct kobject kobj;
-};
-
struct cpuidle_device {
unsigned int registered:1;
unsigned int enabled:1;
int state_count;
struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX];
struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX];
-
+ struct cpuidle_driver_kobj *kobj_driver;
struct list_head device_list;
struct kobject kobj;
struct completion kobj_unregister;
struct cpuidle_driver {
const char *name;
struct module *owner;
+ int refcnt;
unsigned int power_specified:1;
/* set to 1 to use the core cpuidle time keeping (for all states). */
struct cpuidle_driver *drv, int index));
extern int cpuidle_play_dead(void);
+extern struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev);
+extern int cpuidle_register_cpu_driver(struct cpuidle_driver *drv, int cpu);
+extern void cpuidle_unregister_cpu_driver(struct cpuidle_driver *drv, int cpu);
+
#else
static inline void disable_cpuidle(void) { }
static inline int cpuidle_idle_call(void) { return -ENODEV; }
struct cpuidle_driver *drv, int index))
{ return -ENODEV; }
static inline int cpuidle_play_dead(void) {return -ENODEV; }
-
#endif
#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; }
# endif /* !NO_HZ */
+# ifdef CONFIG_CPU_IDLE_GOV_MENU
+extern void menu_hrtimer_cancel(void);
+# else
+static inline void menu_hrtimer_cancel(void) {}
+# endif /* CONFIG_CPU_IDLE_GOV_MENU */
+
#endif
if (!ts->inidle)
return;
+ /* Cancel the timer because CPU already waken up from the C-states*/
+ menu_hrtimer_cancel();
__tick_nohz_idle_enter(ts);
}
ts->inidle = 0;
+ /* Cancel the timer because CPU already waken up from the C-states*/
+ menu_hrtimer_cancel();
if (ts->idle_active || ts->tick_stopped)
now = ktime_get();
projects / platform / adaptation / renesas_rcar / renesas_kernel.git / commitdiff