2 * acpi-cpufreq.c - ACPI Processor P-States Driver ($Revision: 1.4 $)
4 * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
5 * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
6 * Copyright (C) 2002 - 2004 Dominik Brodowski <linux@brodo.de>
7 * Copyright (C) 2006 Denis Sadykov <denis.m.sadykov@intel.com>
9 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or (at
14 * your option) any later version.
16 * This program is distributed in the hope that it will be useful, but
17 * WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
21 * You should have received a copy of the GNU General Public License along
22 * with this program; if not, write to the Free Software Foundation, Inc.,
23 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
25 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/init.h>
31 #include <linux/smp.h>
32 #include <linux/sched.h>
33 #include <linux/cpufreq.h>
34 #include <linux/compiler.h>
35 #include <linux/dmi.h>
36 #include <trace/power.h>
38 #include <linux/acpi.h>
39 #include <acpi/processor.h>
43 #include <asm/processor.h>
44 #include <asm/cpufeature.h>
45 #include <asm/delay.h>
46 #include <asm/uaccess.h>
48 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "acpi-cpufreq", msg)
50 MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski");
51 MODULE_DESCRIPTION("ACPI Processor P-States Driver");
52 MODULE_LICENSE("GPL");
55 UNDEFINED_CAPABLE = 0,
56 SYSTEM_INTEL_MSR_CAPABLE,
60 #define INTEL_MSR_RANGE (0xffff)
61 #define CPUID_6_ECX_APERFMPERF_CAPABILITY (0x1)
63 struct acpi_cpufreq_data {
64 struct acpi_processor_performance *acpi_data;
65 struct cpufreq_frequency_table *freq_table;
66 unsigned int max_freq;
68 unsigned int cpu_feature;
71 static DEFINE_PER_CPU(struct acpi_cpufreq_data *, drv_data);
73 /* acpi_perf_data is a pointer to percpu data. */
74 static struct acpi_processor_performance *acpi_perf_data;
76 static struct cpufreq_driver acpi_cpufreq_driver;
78 static unsigned int acpi_pstate_strict;
80 static int check_est_cpu(unsigned int cpuid)
82 struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
84 if (cpu->x86_vendor != X86_VENDOR_INTEL ||
85 !cpu_has(cpu, X86_FEATURE_EST))
91 static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data)
93 struct acpi_processor_performance *perf;
96 perf = data->acpi_data;
98 for (i=0; i<perf->state_count; i++) {
99 if (value == perf->states[i].status)
100 return data->freq_table[i].frequency;
105 static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data)
108 struct acpi_processor_performance *perf;
110 msr &= INTEL_MSR_RANGE;
111 perf = data->acpi_data;
113 for (i=0; data->freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
114 if (msr == perf->states[data->freq_table[i].index].status)
115 return data->freq_table[i].frequency;
117 return data->freq_table[0].frequency;
120 static unsigned extract_freq(u32 val, struct acpi_cpufreq_data *data)
122 switch (data->cpu_feature) {
123 case SYSTEM_INTEL_MSR_CAPABLE:
124 return extract_msr(val, data);
125 case SYSTEM_IO_CAPABLE:
126 return extract_io(val, data);
148 const struct cpumask *mask;
153 static long do_drv_read(void *_cmd)
155 struct drv_cmd *cmd = _cmd;
159 case SYSTEM_INTEL_MSR_CAPABLE:
160 rdmsr(cmd->addr.msr.reg, cmd->val, h);
162 case SYSTEM_IO_CAPABLE:
163 acpi_os_read_port((acpi_io_address)cmd->addr.io.port,
165 (u32)cmd->addr.io.bit_width);
173 static long do_drv_write(void *_cmd)
175 struct drv_cmd *cmd = _cmd;
179 case SYSTEM_INTEL_MSR_CAPABLE:
180 rdmsr(cmd->addr.msr.reg, lo, hi);
181 lo = (lo & ~INTEL_MSR_RANGE) | (cmd->val & INTEL_MSR_RANGE);
182 wrmsr(cmd->addr.msr.reg, lo, hi);
184 case SYSTEM_IO_CAPABLE:
185 acpi_os_write_port((acpi_io_address)cmd->addr.io.port,
187 (u32)cmd->addr.io.bit_width);
195 static void drv_read(struct drv_cmd *cmd)
199 work_on_cpu(cpumask_any(cmd->mask), do_drv_read, cmd);
202 static void drv_write(struct drv_cmd *cmd)
206 for_each_cpu(i, cmd->mask) {
207 work_on_cpu(i, do_drv_write, cmd);
211 static u32 get_cur_val(const struct cpumask *mask)
213 struct acpi_processor_performance *perf;
216 if (unlikely(cpumask_empty(mask)))
219 switch (per_cpu(drv_data, cpumask_first(mask))->cpu_feature) {
220 case SYSTEM_INTEL_MSR_CAPABLE:
221 cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
222 cmd.addr.msr.reg = MSR_IA32_PERF_STATUS;
224 case SYSTEM_IO_CAPABLE:
225 cmd.type = SYSTEM_IO_CAPABLE;
226 perf = per_cpu(drv_data, cpumask_first(mask))->acpi_data;
227 cmd.addr.io.port = perf->control_register.address;
228 cmd.addr.io.bit_width = perf->control_register.bit_width;
237 dprintk("get_cur_val = %u\n", cmd.val);
249 } aperf_cur, mperf_cur;
253 static long read_measured_perf_ctrs(void *_cur)
255 struct perf_cur *cur = _cur;
257 rdmsr(MSR_IA32_APERF, cur->aperf_cur.split.lo, cur->aperf_cur.split.hi);
258 rdmsr(MSR_IA32_MPERF, cur->mperf_cur.split.lo, cur->mperf_cur.split.hi);
260 wrmsr(MSR_IA32_APERF, 0, 0);
261 wrmsr(MSR_IA32_MPERF, 0, 0);
267 * Return the measured active (C0) frequency on this CPU since last call
270 * Return: Average CPU frequency in terms of max frequency (zero on error)
272 * We use IA32_MPERF and IA32_APERF MSRs to get the measured performance
273 * over a period of time, while CPU is in C0 state.
274 * IA32_MPERF counts at the rate of max advertised frequency
275 * IA32_APERF counts at the rate of actual CPU frequency
276 * Only IA32_APERF/IA32_MPERF ratio is architecturally defined and
277 * no meaning should be associated with absolute values of these MSRs.
279 static unsigned int get_measured_perf(struct cpufreq_policy *policy,
283 unsigned int perf_percent;
286 if (!work_on_cpu(cpu, read_measured_perf_ctrs, &cur))
291 * We dont want to do 64 bit divide with 32 bit kernel
292 * Get an approximate value. Return failure in case we cannot get
293 * an approximate value.
295 if (unlikely(cur.aperf_cur.split.hi || cur.mperf_cur.split.hi)) {
299 h = max_t(u32, cur.aperf_cur.split.hi, cur.mperf_cur.split.hi);
300 shift_count = fls(h);
302 cur.aperf_cur.whole >>= shift_count;
303 cur.mperf_cur.whole >>= shift_count;
306 if (((unsigned long)(-1) / 100) < cur.aperf_cur.split.lo) {
308 cur.aperf_cur.split.lo >>= shift_count;
309 cur.mperf_cur.split.lo >>= shift_count;
312 if (cur.aperf_cur.split.lo && cur.mperf_cur.split.lo)
313 perf_percent = (cur.aperf_cur.split.lo * 100) /
314 cur.mperf_cur.split.lo;
319 if (unlikely(((unsigned long)(-1) / 100) < cur.aperf_cur.whole)) {
321 cur.aperf_cur.whole >>= shift_count;
322 cur.mperf_cur.whole >>= shift_count;
325 if (cur.aperf_cur.whole && cur.mperf_cur.whole)
326 perf_percent = (cur.aperf_cur.whole * 100) /
333 retval = per_cpu(drv_data, policy->cpu)->max_freq * perf_percent / 100;
338 static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
340 struct acpi_cpufreq_data *data = per_cpu(drv_data, cpu);
342 unsigned int cached_freq;
344 dprintk("get_cur_freq_on_cpu (%d)\n", cpu);
346 if (unlikely(data == NULL ||
347 data->acpi_data == NULL || data->freq_table == NULL)) {
351 cached_freq = data->freq_table[data->acpi_data->state].frequency;
352 freq = extract_freq(get_cur_val(cpumask_of(cpu)), data);
353 if (freq != cached_freq) {
355 * The dreaded BIOS frequency change behind our back.
356 * Force set the frequency on next target call.
361 dprintk("cur freq = %u\n", freq);
366 static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq,
367 struct acpi_cpufreq_data *data)
369 unsigned int cur_freq;
372 for (i=0; i<100; i++) {
373 cur_freq = extract_freq(get_cur_val(mask), data);
374 if (cur_freq == freq)
381 static int acpi_cpufreq_target(struct cpufreq_policy *policy,
382 unsigned int target_freq, unsigned int relation)
384 struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu);
385 struct acpi_processor_performance *perf;
386 struct cpufreq_freqs freqs;
388 unsigned int next_state = 0; /* Index into freq_table */
389 unsigned int next_perf_state = 0; /* Index into perf table */
392 struct power_trace it;
394 dprintk("acpi_cpufreq_target %d (%d)\n", target_freq, policy->cpu);
396 if (unlikely(data == NULL ||
397 data->acpi_data == NULL || data->freq_table == NULL)) {
401 perf = data->acpi_data;
402 result = cpufreq_frequency_table_target(policy,
405 relation, &next_state);
406 if (unlikely(result)) {
411 next_perf_state = data->freq_table[next_state].index;
412 if (perf->state == next_perf_state) {
413 if (unlikely(data->resume)) {
414 dprintk("Called after resume, resetting to P%d\n",
418 dprintk("Already at target state (P%d)\n",
424 trace_power_mark(&it, POWER_PSTATE, next_perf_state);
426 switch (data->cpu_feature) {
427 case SYSTEM_INTEL_MSR_CAPABLE:
428 cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
429 cmd.addr.msr.reg = MSR_IA32_PERF_CTL;
430 cmd.val = (u32) perf->states[next_perf_state].control;
432 case SYSTEM_IO_CAPABLE:
433 cmd.type = SYSTEM_IO_CAPABLE;
434 cmd.addr.io.port = perf->control_register.address;
435 cmd.addr.io.bit_width = perf->control_register.bit_width;
436 cmd.val = (u32) perf->states[next_perf_state].control;
443 /* cpufreq holds the hotplug lock, so we are safe from here on */
444 if (policy->shared_type != CPUFREQ_SHARED_TYPE_ANY)
445 cmd.mask = policy->cpus;
447 cmd.mask = cpumask_of(policy->cpu);
449 freqs.old = perf->states[perf->state].core_frequency * 1000;
450 freqs.new = data->freq_table[next_state].frequency;
451 for_each_cpu(i, cmd.mask) {
453 cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
458 if (acpi_pstate_strict) {
459 if (!check_freqs(cmd.mask, freqs.new, data)) {
460 dprintk("acpi_cpufreq_target failed (%d)\n",
467 for_each_cpu(i, cmd.mask) {
469 cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
471 perf->state = next_perf_state;
477 static int acpi_cpufreq_verify(struct cpufreq_policy *policy)
479 struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu);
481 dprintk("acpi_cpufreq_verify\n");
483 return cpufreq_frequency_table_verify(policy, data->freq_table);
487 acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
489 struct acpi_processor_performance *perf = data->acpi_data;
492 /* search the closest match to cpu_khz */
495 unsigned long freqn = perf->states[0].core_frequency * 1000;
497 for (i=0; i<(perf->state_count-1); i++) {
499 freqn = perf->states[i+1].core_frequency * 1000;
500 if ((2 * cpu_khz) > (freqn + freq)) {
505 perf->state = perf->state_count-1;
508 /* assume CPU is at P0... */
510 return perf->states[0].core_frequency * 1000;
514 static void free_acpi_perf_data(void)
518 /* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */
519 for_each_possible_cpu(i)
520 free_cpumask_var(per_cpu_ptr(acpi_perf_data, i)
522 free_percpu(acpi_perf_data);
526 * acpi_cpufreq_early_init - initialize ACPI P-States library
528 * Initialize the ACPI P-States library (drivers/acpi/processor_perflib.c)
529 * in order to determine correct frequency and voltage pairings. We can
530 * do _PDC and _PSD and find out the processor dependency for the
531 * actual init that will happen later...
533 static int __init acpi_cpufreq_early_init(void)
536 dprintk("acpi_cpufreq_early_init\n");
538 acpi_perf_data = alloc_percpu(struct acpi_processor_performance);
539 if (!acpi_perf_data) {
540 dprintk("Memory allocation error for acpi_perf_data.\n");
543 for_each_possible_cpu(i) {
544 if (!alloc_cpumask_var_node(
545 &per_cpu_ptr(acpi_perf_data, i)->shared_cpu_map,
546 GFP_KERNEL, cpu_to_node(i))) {
548 /* Freeing a NULL pointer is OK: alloc_percpu zeroes. */
549 free_acpi_perf_data();
554 /* Do initialization in ACPI core */
555 acpi_processor_preregister_performance(acpi_perf_data);
561 * Some BIOSes do SW_ANY coordination internally, either set it up in hw
562 * or do it in BIOS firmware and won't inform about it to OS. If not
563 * detected, this has a side effect of making CPU run at a different speed
564 * than OS intended it to run at. Detect it and handle it cleanly.
566 static int bios_with_sw_any_bug;
568 static int sw_any_bug_found(const struct dmi_system_id *d)
570 bios_with_sw_any_bug = 1;
574 static const struct dmi_system_id sw_any_bug_dmi_table[] = {
576 .callback = sw_any_bug_found,
577 .ident = "Supermicro Server X6DLP",
579 DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"),
580 DMI_MATCH(DMI_BIOS_VERSION, "080010"),
581 DMI_MATCH(DMI_PRODUCT_NAME, "X6DLP"),
588 static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
591 unsigned int valid_states = 0;
592 unsigned int cpu = policy->cpu;
593 struct acpi_cpufreq_data *data;
594 unsigned int result = 0;
595 struct cpuinfo_x86 *c = &cpu_data(policy->cpu);
596 struct acpi_processor_performance *perf;
598 dprintk("acpi_cpufreq_cpu_init\n");
600 data = kzalloc(sizeof(struct acpi_cpufreq_data), GFP_KERNEL);
604 data->acpi_data = percpu_ptr(acpi_perf_data, cpu);
605 per_cpu(drv_data, cpu) = data;
607 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC))
608 acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
610 result = acpi_processor_register_performance(data->acpi_data, cpu);
614 perf = data->acpi_data;
615 policy->shared_type = perf->shared_type;
618 * Will let policy->cpus know about dependency only when software
619 * coordination is required.
621 if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
622 policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
623 cpumask_copy(policy->cpus, perf->shared_cpu_map);
625 cpumask_copy(policy->related_cpus, perf->shared_cpu_map);
628 dmi_check_system(sw_any_bug_dmi_table);
629 if (bios_with_sw_any_bug && cpumask_weight(policy->cpus) == 1) {
630 policy->shared_type = CPUFREQ_SHARED_TYPE_ALL;
631 cpumask_copy(policy->cpus, cpu_core_mask(cpu));
635 /* capability check */
636 if (perf->state_count <= 1) {
637 dprintk("No P-States\n");
642 if (perf->control_register.space_id != perf->status_register.space_id) {
647 switch (perf->control_register.space_id) {
648 case ACPI_ADR_SPACE_SYSTEM_IO:
649 dprintk("SYSTEM IO addr space\n");
650 data->cpu_feature = SYSTEM_IO_CAPABLE;
652 case ACPI_ADR_SPACE_FIXED_HARDWARE:
653 dprintk("HARDWARE addr space\n");
654 if (!check_est_cpu(cpu)) {
658 data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE;
661 dprintk("Unknown addr space %d\n",
662 (u32) (perf->control_register.space_id));
667 data->freq_table = kmalloc(sizeof(struct cpufreq_frequency_table) *
668 (perf->state_count+1), GFP_KERNEL);
669 if (!data->freq_table) {
674 /* detect transition latency */
675 policy->cpuinfo.transition_latency = 0;
676 for (i=0; i<perf->state_count; i++) {
677 if ((perf->states[i].transition_latency * 1000) >
678 policy->cpuinfo.transition_latency)
679 policy->cpuinfo.transition_latency =
680 perf->states[i].transition_latency * 1000;
683 data->max_freq = perf->states[0].core_frequency * 1000;
685 for (i=0; i<perf->state_count; i++) {
686 if (i>0 && perf->states[i].core_frequency >=
687 data->freq_table[valid_states-1].frequency / 1000)
690 data->freq_table[valid_states].index = i;
691 data->freq_table[valid_states].frequency =
692 perf->states[i].core_frequency * 1000;
695 data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END;
698 result = cpufreq_frequency_table_cpuinfo(policy, data->freq_table);
702 switch (perf->control_register.space_id) {
703 case ACPI_ADR_SPACE_SYSTEM_IO:
704 /* Current speed is unknown and not detectable by IO port */
705 policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu);
707 case ACPI_ADR_SPACE_FIXED_HARDWARE:
708 acpi_cpufreq_driver.get = get_cur_freq_on_cpu;
709 policy->cur = get_cur_freq_on_cpu(cpu);
715 /* notify BIOS that we exist */
716 acpi_processor_notify_smm(THIS_MODULE);
718 /* Check for APERF/MPERF support in hardware */
719 if (c->x86_vendor == X86_VENDOR_INTEL && c->cpuid_level >= 6) {
722 if (ecx & CPUID_6_ECX_APERFMPERF_CAPABILITY)
723 acpi_cpufreq_driver.getavg = get_measured_perf;
726 dprintk("CPU%u - ACPI performance management activated.\n", cpu);
727 for (i = 0; i < perf->state_count; i++)
728 dprintk(" %cP%d: %d MHz, %d mW, %d uS\n",
729 (i == perf->state ? '*' : ' '), i,
730 (u32) perf->states[i].core_frequency,
731 (u32) perf->states[i].power,
732 (u32) perf->states[i].transition_latency);
734 cpufreq_frequency_table_get_attr(data->freq_table, policy->cpu);
737 * the first call to ->target() should result in us actually
738 * writing something to the appropriate registers.
745 kfree(data->freq_table);
747 acpi_processor_unregister_performance(perf, cpu);
750 per_cpu(drv_data, cpu) = NULL;
755 static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy)
757 struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu);
759 dprintk("acpi_cpufreq_cpu_exit\n");
762 cpufreq_frequency_table_put_attr(policy->cpu);
763 per_cpu(drv_data, policy->cpu) = NULL;
764 acpi_processor_unregister_performance(data->acpi_data,
772 static int acpi_cpufreq_resume(struct cpufreq_policy *policy)
774 struct acpi_cpufreq_data *data = per_cpu(drv_data, policy->cpu);
776 dprintk("acpi_cpufreq_resume\n");
783 static struct freq_attr *acpi_cpufreq_attr[] = {
784 &cpufreq_freq_attr_scaling_available_freqs,
788 static struct cpufreq_driver acpi_cpufreq_driver = {
789 .verify = acpi_cpufreq_verify,
790 .target = acpi_cpufreq_target,
791 .init = acpi_cpufreq_cpu_init,
792 .exit = acpi_cpufreq_cpu_exit,
793 .resume = acpi_cpufreq_resume,
794 .name = "acpi-cpufreq",
795 .owner = THIS_MODULE,
796 .attr = acpi_cpufreq_attr,
799 static int __init acpi_cpufreq_init(void)
806 dprintk("acpi_cpufreq_init\n");
808 ret = acpi_cpufreq_early_init();
812 ret = cpufreq_register_driver(&acpi_cpufreq_driver);
814 free_acpi_perf_data();
819 static void __exit acpi_cpufreq_exit(void)
821 dprintk("acpi_cpufreq_exit\n");
823 cpufreq_unregister_driver(&acpi_cpufreq_driver);
825 free_percpu(acpi_perf_data);
828 module_param(acpi_pstate_strict, uint, 0644);
829 MODULE_PARM_DESC(acpi_pstate_strict,
830 "value 0 or non-zero. non-zero -> strict ACPI checks are "
831 "performed during frequency changes.");
833 late_initcall(acpi_cpufreq_init);
834 module_exit(acpi_cpufreq_exit);
836 MODULE_ALIAS("acpi");