2 * drivers/cpufreq/cpufreq_ondemand.c
4 * Copyright (C) 2001 Russell King
5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
6 * Jun Nakajima <jun.nakajima@intel.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15 #include <linux/cpufreq.h>
16 #include <linux/init.h>
17 #include <linux/kernel.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/kobject.h>
20 #include <linux/module.h>
21 #include <linux/mutex.h>
22 #include <linux/percpu-defs.h>
23 #include <linux/slab.h>
24 #include <linux/sysfs.h>
25 #include <linux/tick.h>
26 #include <linux/types.h>
28 #include "cpufreq_governor.h"
30 /* On-demand governor macros */
31 #define DEF_FREQUENCY_DOWN_DIFFERENTIAL (10)
32 #define DEF_FREQUENCY_UP_THRESHOLD (80)
33 #define DEF_SAMPLING_DOWN_FACTOR (1)
34 #define MAX_SAMPLING_DOWN_FACTOR (100000)
35 #define MICRO_FREQUENCY_DOWN_DIFFERENTIAL (3)
36 #define MICRO_FREQUENCY_UP_THRESHOLD (95)
37 #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)
38 #define MIN_FREQUENCY_UP_THRESHOLD (11)
39 #define MAX_FREQUENCY_UP_THRESHOLD (100)
41 static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);
43 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
44 static struct cpufreq_governor cpufreq_gov_ondemand;
47 static void ondemand_powersave_bias_init_cpu(int cpu)
49 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
51 dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
52 dbs_info->freq_lo = 0;
56 * Not all CPUs want IO time to be accounted as busy; this depends on how
57 * efficient idling at a higher frequency/voltage is.
58 * Pavel Machek says this is not so for various generations of AMD and old
60 * Mike Chan (android.com) claims this is also not true for ARM.
61 * Because of this, whitelist specific known (series) of CPUs by default, and
62 * leave all others up to the user.
64 static int should_io_be_busy(void)
66 #if defined(CONFIG_X86)
68 * For Intel, Core 2 (model 15) and later have an efficient idle.
70 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
71 boot_cpu_data.x86 == 6 &&
72 boot_cpu_data.x86_model >= 15)
79 * Find right freq to be set now with powersave_bias on.
80 * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
81 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
83 static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
84 unsigned int freq_next, unsigned int relation)
86 unsigned int freq_req, freq_reduc, freq_avg;
87 unsigned int freq_hi, freq_lo;
88 unsigned int index = 0;
89 unsigned int jiffies_total, jiffies_hi, jiffies_lo;
90 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
92 struct dbs_data *dbs_data = policy->governor_data;
93 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
95 if (!dbs_info->freq_table) {
96 dbs_info->freq_lo = 0;
97 dbs_info->freq_lo_jiffies = 0;
101 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
103 freq_req = dbs_info->freq_table[index].frequency;
104 freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
105 freq_avg = freq_req - freq_reduc;
107 /* Find freq bounds for freq_avg in freq_table */
109 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
110 CPUFREQ_RELATION_H, &index);
111 freq_lo = dbs_info->freq_table[index].frequency;
113 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
114 CPUFREQ_RELATION_L, &index);
115 freq_hi = dbs_info->freq_table[index].frequency;
117 /* Find out how long we have to be in hi and lo freqs */
118 if (freq_hi == freq_lo) {
119 dbs_info->freq_lo = 0;
120 dbs_info->freq_lo_jiffies = 0;
123 jiffies_total = usecs_to_jiffies(od_tuners->sampling_rate);
124 jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
125 jiffies_hi += ((freq_hi - freq_lo) / 2);
126 jiffies_hi /= (freq_hi - freq_lo);
127 jiffies_lo = jiffies_total - jiffies_hi;
128 dbs_info->freq_lo = freq_lo;
129 dbs_info->freq_lo_jiffies = jiffies_lo;
130 dbs_info->freq_hi_jiffies = jiffies_hi;
134 static void ondemand_powersave_bias_init(void)
137 for_each_online_cpu(i) {
138 ondemand_powersave_bias_init_cpu(i);
142 static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq)
144 struct dbs_data *dbs_data = p->governor_data;
145 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
147 if (od_tuners->powersave_bias)
148 freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H);
149 else if (p->cur == p->max)
152 __cpufreq_driver_target(p, freq, od_tuners->powersave_bias ?
153 CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
157 * Every sampling_rate, we check, if current idle time is less than 20%
158 * (default), then we try to increase frequency. Every sampling_rate, we look
159 * for the lowest frequency which can sustain the load while keeping idle time
160 * over 30%. If such a frequency exist, we try to decrease to this frequency.
162 * Any frequency increase takes it to the maximum frequency. Frequency reduction
163 * happens at minimum steps of 5% (default) of current frequency
165 static void od_check_cpu(int cpu, unsigned int load_freq)
167 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
168 struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy;
169 struct dbs_data *dbs_data = policy->governor_data;
170 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
172 dbs_info->freq_lo = 0;
174 /* Check for frequency increase */
175 if (load_freq > od_tuners->up_threshold * policy->cur) {
176 /* If switching to max speed, apply sampling_down_factor */
177 if (policy->cur < policy->max)
178 dbs_info->rate_mult =
179 od_tuners->sampling_down_factor;
180 dbs_freq_increase(policy, policy->max);
184 /* Check for frequency decrease */
185 /* if we cannot reduce the frequency anymore, break out early */
186 if (policy->cur == policy->min)
190 * The optimal frequency is the frequency that is the lowest that can
191 * support the current CPU usage without triggering the up policy. To be
192 * safe, we focus 10 points under the threshold.
194 if (load_freq < od_tuners->adj_up_threshold
196 unsigned int freq_next;
197 freq_next = load_freq / od_tuners->adj_up_threshold;
199 /* No longer fully busy, reset rate_mult */
200 dbs_info->rate_mult = 1;
202 if (freq_next < policy->min)
203 freq_next = policy->min;
205 if (!od_tuners->powersave_bias) {
206 __cpufreq_driver_target(policy, freq_next,
209 int freq = powersave_bias_target(policy, freq_next,
211 __cpufreq_driver_target(policy, freq,
217 static void od_dbs_timer(struct work_struct *work)
219 struct delayed_work *dw = to_delayed_work(work);
220 struct od_cpu_dbs_info_s *dbs_info =
221 container_of(work, struct od_cpu_dbs_info_s, cdbs.work.work);
222 unsigned int cpu = dbs_info->cdbs.cur_policy->cpu;
223 struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info,
225 struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data;
226 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
227 int delay = 0, sample_type = core_dbs_info->sample_type;
229 mutex_lock(&core_dbs_info->cdbs.timer_mutex);
230 if (!need_load_eval(&core_dbs_info->cdbs, od_tuners->sampling_rate))
233 /* Common NORMAL_SAMPLE setup */
234 core_dbs_info->sample_type = OD_NORMAL_SAMPLE;
235 if (sample_type == OD_SUB_SAMPLE) {
236 delay = core_dbs_info->freq_lo_jiffies;
237 __cpufreq_driver_target(core_dbs_info->cdbs.cur_policy,
238 core_dbs_info->freq_lo, CPUFREQ_RELATION_H);
240 dbs_check_cpu(dbs_data, cpu);
241 if (core_dbs_info->freq_lo) {
242 /* Setup timer for SUB_SAMPLE */
243 core_dbs_info->sample_type = OD_SUB_SAMPLE;
244 delay = core_dbs_info->freq_hi_jiffies;
250 delay = delay_for_sampling_rate(od_tuners->sampling_rate
251 * core_dbs_info->rate_mult);
253 schedule_delayed_work_on(smp_processor_id(), dw, delay);
254 mutex_unlock(&core_dbs_info->cdbs.timer_mutex);
257 /************************** sysfs interface ************************/
258 static struct common_dbs_data od_dbs_cdata;
261 * update_sampling_rate - update sampling rate effective immediately if needed.
262 * @new_rate: new sampling rate
264 * If new rate is smaller than the old, simply updating
265 * dbs_tuners_int.sampling_rate might not be appropriate. For example, if the
266 * original sampling_rate was 1 second and the requested new sampling rate is 10
267 * ms because the user needs immediate reaction from ondemand governor, but not
268 * sure if higher frequency will be required or not, then, the governor may
269 * change the sampling rate too late; up to 1 second later. Thus, if we are
270 * reducing the sampling rate, we need to make the new value effective
273 static void update_sampling_rate(struct dbs_data *dbs_data,
274 unsigned int new_rate)
276 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
279 od_tuners->sampling_rate = new_rate = max(new_rate,
280 dbs_data->min_sampling_rate);
282 for_each_online_cpu(cpu) {
283 struct cpufreq_policy *policy;
284 struct od_cpu_dbs_info_s *dbs_info;
285 unsigned long next_sampling, appointed_at;
287 policy = cpufreq_cpu_get(cpu);
290 if (policy->governor != &cpufreq_gov_ondemand) {
291 cpufreq_cpu_put(policy);
294 dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
295 cpufreq_cpu_put(policy);
297 mutex_lock(&dbs_info->cdbs.timer_mutex);
299 if (!delayed_work_pending(&dbs_info->cdbs.work)) {
300 mutex_unlock(&dbs_info->cdbs.timer_mutex);
304 next_sampling = jiffies + usecs_to_jiffies(new_rate);
305 appointed_at = dbs_info->cdbs.work.timer.expires;
307 if (time_before(next_sampling, appointed_at)) {
309 mutex_unlock(&dbs_info->cdbs.timer_mutex);
310 cancel_delayed_work_sync(&dbs_info->cdbs.work);
311 mutex_lock(&dbs_info->cdbs.timer_mutex);
313 schedule_delayed_work_on(cpu, &dbs_info->cdbs.work,
314 usecs_to_jiffies(new_rate));
317 mutex_unlock(&dbs_info->cdbs.timer_mutex);
321 static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf,
326 ret = sscanf(buf, "%u", &input);
330 update_sampling_rate(dbs_data, input);
334 static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf,
337 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
341 ret = sscanf(buf, "%u", &input);
344 od_tuners->io_is_busy = !!input;
348 static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf,
351 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
354 ret = sscanf(buf, "%u", &input);
356 if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
357 input < MIN_FREQUENCY_UP_THRESHOLD) {
360 /* Calculate the new adj_up_threshold */
361 od_tuners->adj_up_threshold += input;
362 od_tuners->adj_up_threshold -= od_tuners->up_threshold;
364 od_tuners->up_threshold = input;
368 static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data,
369 const char *buf, size_t count)
371 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
372 unsigned int input, j;
374 ret = sscanf(buf, "%u", &input);
376 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
378 od_tuners->sampling_down_factor = input;
380 /* Reset down sampling multiplier in case it was active */
381 for_each_online_cpu(j) {
382 struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
384 dbs_info->rate_mult = 1;
389 static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf,
392 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
398 ret = sscanf(buf, "%u", &input);
405 if (input == od_tuners->ignore_nice) { /* nothing to do */
408 od_tuners->ignore_nice = input;
410 /* we need to re-evaluate prev_cpu_idle */
411 for_each_online_cpu(j) {
412 struct od_cpu_dbs_info_s *dbs_info;
413 dbs_info = &per_cpu(od_cpu_dbs_info, j);
414 dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
415 &dbs_info->cdbs.prev_cpu_wall);
416 if (od_tuners->ignore_nice)
417 dbs_info->cdbs.prev_cpu_nice =
418 kcpustat_cpu(j).cpustat[CPUTIME_NICE];
424 static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf,
427 struct od_dbs_tuners *od_tuners = dbs_data->tuners;
430 ret = sscanf(buf, "%u", &input);
438 od_tuners->powersave_bias = input;
439 ondemand_powersave_bias_init();
443 show_store_one(od, sampling_rate);
444 show_store_one(od, io_is_busy);
445 show_store_one(od, up_threshold);
446 show_store_one(od, sampling_down_factor);
447 show_store_one(od, ignore_nice);
448 show_store_one(od, powersave_bias);
449 declare_show_sampling_rate_min(od);
451 gov_sys_pol_attr_rw(sampling_rate);
452 gov_sys_pol_attr_rw(io_is_busy);
453 gov_sys_pol_attr_rw(up_threshold);
454 gov_sys_pol_attr_rw(sampling_down_factor);
455 gov_sys_pol_attr_rw(ignore_nice);
456 gov_sys_pol_attr_rw(powersave_bias);
457 gov_sys_pol_attr_ro(sampling_rate_min);
459 static struct attribute *dbs_attributes_gov_sys[] = {
460 &sampling_rate_min_gov_sys.attr,
461 &sampling_rate_gov_sys.attr,
462 &up_threshold_gov_sys.attr,
463 &sampling_down_factor_gov_sys.attr,
464 &ignore_nice_gov_sys.attr,
465 &powersave_bias_gov_sys.attr,
466 &io_is_busy_gov_sys.attr,
470 static struct attribute_group od_attr_group_gov_sys = {
471 .attrs = dbs_attributes_gov_sys,
475 static struct attribute *dbs_attributes_gov_pol[] = {
476 &sampling_rate_min_gov_pol.attr,
477 &sampling_rate_gov_pol.attr,
478 &up_threshold_gov_pol.attr,
479 &sampling_down_factor_gov_pol.attr,
480 &ignore_nice_gov_pol.attr,
481 &powersave_bias_gov_pol.attr,
482 &io_is_busy_gov_pol.attr,
486 static struct attribute_group od_attr_group_gov_pol = {
487 .attrs = dbs_attributes_gov_pol,
491 /************************** sysfs end ************************/
493 static int od_init(struct dbs_data *dbs_data)
495 struct od_dbs_tuners *tuners;
499 tuners = kzalloc(sizeof(struct od_dbs_tuners), GFP_KERNEL);
501 pr_err("%s: kzalloc failed\n", __func__);
506 idle_time = get_cpu_idle_time_us(cpu, NULL);
508 if (idle_time != -1ULL) {
509 /* Idle micro accounting is supported. Use finer thresholds */
510 tuners->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
511 tuners->adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD -
512 MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
514 * In nohz/micro accounting case we set the minimum frequency
515 * not depending on HZ, but fixed (very low). The deferred
516 * timer might skip some samples if idle/sleeping as needed.
518 dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
520 tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
521 tuners->adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD -
522 DEF_FREQUENCY_DOWN_DIFFERENTIAL;
524 /* For correct statistics, we need 10 ticks for each measure */
525 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
526 jiffies_to_usecs(10);
529 tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
530 tuners->ignore_nice = 0;
531 tuners->powersave_bias = 0;
532 tuners->io_is_busy = should_io_be_busy();
534 dbs_data->tuners = tuners;
535 pr_info("%s: tuners %p\n", __func__, tuners);
536 mutex_init(&dbs_data->mutex);
540 static void od_exit(struct dbs_data *dbs_data)
542 kfree(dbs_data->tuners);
545 define_get_cpu_dbs_routines(od_cpu_dbs_info);
547 static struct od_ops od_ops = {
548 .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu,
549 .powersave_bias_target = powersave_bias_target,
550 .freq_increase = dbs_freq_increase,
553 static struct common_dbs_data od_dbs_cdata = {
554 .governor = GOV_ONDEMAND,
555 .attr_group_gov_sys = &od_attr_group_gov_sys,
556 .attr_group_gov_pol = &od_attr_group_gov_pol,
557 .get_cpu_cdbs = get_cpu_cdbs,
558 .get_cpu_dbs_info_s = get_cpu_dbs_info_s,
559 .gov_dbs_timer = od_dbs_timer,
560 .gov_check_cpu = od_check_cpu,
566 static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy,
569 return cpufreq_governor_dbs(policy, &od_dbs_cdata, event);
572 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
575 struct cpufreq_governor cpufreq_gov_ondemand = {
577 .governor = od_cpufreq_governor_dbs,
578 .max_transition_latency = TRANSITION_LATENCY_LIMIT,
579 .owner = THIS_MODULE,
582 static int __init cpufreq_gov_dbs_init(void)
584 return cpufreq_register_governor(&cpufreq_gov_ondemand);
587 static void __exit cpufreq_gov_dbs_exit(void)
589 cpufreq_unregister_governor(&cpufreq_gov_ondemand);
592 MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
593 MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
594 MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
595 "Low Latency Frequency Transition capable processors");
596 MODULE_LICENSE("GPL");
598 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
599 fs_initcall(cpufreq_gov_dbs_init);
601 module_init(cpufreq_gov_dbs_init);
603 module_exit(cpufreq_gov_dbs_exit);