drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 static inline bool has_target(void)
  51 {
  52         return cpufreq_driver->target_index || cpufreq_driver->target;
  53 }
  54
  55 /*
  56  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  57  * sections
  58  */
  59 static DECLARE_RWSEM(cpufreq_rwsem);
  60
  61 /* internal prototypes */
  62 static int __cpufreq_governor(struct cpufreq_policy *policy,
  63                 unsigned int event);
  64 static unsigned int __cpufreq_get(unsigned int cpu);
  65 static void handle_update(struct work_struct *work);
  66
  67 /**
  68  * Two notifier lists: the "policy" list is involved in the
  69  * validation process for a new CPU frequency policy; the
  70  * "transition" list for kernel code that needs to handle
  71  * changes to devices when the CPU clock speed changes.
  72  * The mutex locks both lists.
  73  */
  74 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  75 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  76
  77 static bool init_cpufreq_transition_notifier_list_called;
  78 static int __init init_cpufreq_transition_notifier_list(void)
  79 {
  80         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  81         init_cpufreq_transition_notifier_list_called = true;
  82         return 0;
  83 }
  84 pure_initcall(init_cpufreq_transition_notifier_list);
  85
  86 static int off __read_mostly;
  87 static int cpufreq_disabled(void)
  88 {
  89         return off;
  90 }
  91 void disable_cpufreq(void)
  92 {
  93         off = 1;
  94 }
  95 static LIST_HEAD(cpufreq_governor_list);
  96 static DEFINE_MUTEX(cpufreq_governor_mutex);
  97
  98 bool have_governor_per_policy(void)
  99 {
 100         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 101 }
 102 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 103
 104 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 105 {
 106         if (have_governor_per_policy())
 107                 return &policy->kobj;
 108         else
 109                 return cpufreq_global_kobject;
 110 }
 111 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 112
 113 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 114 {
 115         u64 idle_time;
 116         u64 cur_wall_time;
 117         u64 busy_time;
 118
 119         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 120
 121         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 122         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 127
 128         idle_time = cur_wall_time - busy_time;
 129         if (wall)
 130                 *wall = cputime_to_usecs(cur_wall_time);
 131
 132         return cputime_to_usecs(idle_time);
 133 }
 134
 135 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 136 {
 137         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 138
 139         if (idle_time == -1ULL)
 140                 return get_cpu_idle_time_jiffy(cpu, wall);
 141         else if (!io_busy)
 142                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 143
 144         return idle_time;
 145 }
 146 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 147
 148 /*
 149  * This is a generic cpufreq init() routine which can be used by cpufreq
 150  * drivers of SMP systems. It will do following:
 151  * - validate & show freq table passed
 152  * - set policies transition latency
 153  * - policy->cpus with all possible CPUs
 154  */
 155 int cpufreq_generic_init(struct cpufreq_policy *policy,
 156                 struct cpufreq_frequency_table *table,
 157                 unsigned int transition_latency)
 158 {
 159         int ret;
 160
 161         ret = cpufreq_table_validate_and_show(policy, table);
 162         if (ret) {
 163                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 164                 return ret;
 165         }
 166
 167         policy->cpuinfo.transition_latency = transition_latency;
 168
 169         /*
 170          * The driver only supports the SMP configuartion where all processors
 171          * share the clock and voltage and clock.
 172          */
 173         cpumask_setall(policy->cpus);
 174
 175         return 0;
 176 }
 177 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 178
 179 unsigned int cpufreq_generic_get(unsigned int cpu)
 180 {
 181         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 182
 183         if (!policy || IS_ERR(policy->clk)) {
 184                 pr_err("%s: No %s associated to cpu: %d\n", __func__,
 185                                 policy ? "clk" : "policy", cpu);
 186                 return 0;
 187         }
 188
 189         return clk_get_rate(policy->clk) / 1000;
 190 }
 191 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 192
 193 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 194 {
 195         struct cpufreq_policy *policy = NULL;
 196         unsigned long flags;
 197
 198         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 199                 return NULL;
 200
 201         if (!down_read_trylock(&cpufreq_rwsem))
 202                 return NULL;
 203
 204         /* get the cpufreq driver */
 205         read_lock_irqsave(&cpufreq_driver_lock, flags);
 206
 207         if (cpufreq_driver) {
 208                 /* get the CPU */
 209                 policy = per_cpu(cpufreq_cpu_data, cpu);
 210                 if (policy)
 211                         kobject_get(&policy->kobj);
 212         }
 213
 214         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 215
 216         if (!policy)
 217                 up_read(&cpufreq_rwsem);
 218
 219         return policy;
 220 }
 221 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 222
 223 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 224 {
 225         if (cpufreq_disabled())
 226                 return;
 227
 228         kobject_put(&policy->kobj);
 229         up_read(&cpufreq_rwsem);
 230 }
 231 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 232
 233 /*********************************************************************
 234  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 235  *********************************************************************/
 236
 237 /**
 238  * adjust_jiffies - adjust the system "loops_per_jiffy"
 239  *
 240  * This function alters the system "loops_per_jiffy" for the clock
 241  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 242  * systems as each CPU might be scaled differently. So, use the arch
 243  * per-CPU loops_per_jiffy value wherever possible.
 244  */
 245 #ifndef CONFIG_SMP
 246 static unsigned long l_p_j_ref;
 247 static unsigned int l_p_j_ref_freq;
 248
 249 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 250 {
 251         if (ci->flags & CPUFREQ_CONST_LOOPS)
 252                 return;
 253
 254         if (!l_p_j_ref_freq) {
 255                 l_p_j_ref = loops_per_jiffy;
 256                 l_p_j_ref_freq = ci->old;
 257                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 258                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 259         }
 260         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 261             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 262                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 263                                                                 ci->new);
 264                 pr_debug("scaling loops_per_jiffy to %lu "
 265                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 266         }
 267 }
 268 #else
 269 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 270 {
 271         return;
 272 }
 273 #endif
 274
 275 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 276                 struct cpufreq_freqs *freqs, unsigned int state)
 277 {
 278         BUG_ON(irqs_disabled());
 279
 280         if (cpufreq_disabled())
 281                 return;
 282
 283         freqs->flags = cpufreq_driver->flags;
 284         pr_debug("notification %u of frequency transition to %u kHz\n",
 285                 state, freqs->new);
 286
 287         switch (state) {
 288
 289         case CPUFREQ_PRECHANGE:
 290                 /* detect if the driver reported a value as "old frequency"
 291                  * which is not equal to what the cpufreq core thinks is
 292                  * "old frequency".
 293                  */
 294                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 295                         if ((policy) && (policy->cpu == freqs->cpu) &&
 296                             (policy->cur) && (policy->cur != freqs->old)) {
 297                                 pr_debug("Warning: CPU frequency is"
 298                                         " %u, cpufreq assumed %u kHz.\n",
 299                                         freqs->old, policy->cur);
 300                                 freqs->old = policy->cur;
 301                         }
 302                 }
 303                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 304                                 CPUFREQ_PRECHANGE, freqs);
 305                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 306                 break;
 307
 308         case CPUFREQ_POSTCHANGE:
 309                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 310                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 311                         (unsigned long)freqs->cpu);
 312                 trace_cpu_frequency(freqs->new, freqs->cpu);
 313                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 314                                 CPUFREQ_POSTCHANGE, freqs);
 315                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 316                         policy->cur = freqs->new;
 317                 break;
 318         }
 319 }
 320
 321 /**
 322  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 323  * on frequency transition.
 324  *
 325  * This function calls the transition notifiers and the "adjust_jiffies"
 326  * function. It is called twice on all CPU frequency changes that have
 327  * external effects.
 328  */
 329 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 330                 struct cpufreq_freqs *freqs, unsigned int state)
 331 {
 332         for_each_cpu(freqs->cpu, policy->cpus)
 333                 __cpufreq_notify_transition(policy, freqs, state);
 334 }
 335 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 336
 337 /* Do post notifications when there are chances that transition has failed */
 338 void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 339                 struct cpufreq_freqs *freqs, int transition_failed)
 340 {
 341         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 342         if (!transition_failed)
 343                 return;
 344
 345         swap(freqs->old, freqs->new);
 346         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 347         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 348 }
 349 EXPORT_SYMBOL_GPL(cpufreq_notify_post_transition);
 350
 351
 352 /*********************************************************************
 353  *                          SYSFS INTERFACE                          *
 354  *********************************************************************/
 355 ssize_t show_boost(struct kobject *kobj,
 356                                  struct attribute *attr, char *buf)
 357 {
 358         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 359 }
 360
 361 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 362                                   const char *buf, size_t count)
 363 {
 364         int ret, enable;
 365
 366         ret = sscanf(buf, "%d", &enable);
 367         if (ret != 1 || enable < 0 || enable > 1)
 368                 return -EINVAL;
 369
 370         if (cpufreq_boost_trigger_state(enable)) {
 371                 pr_err("%s: Cannot %s BOOST!\n", __func__,
 372                        enable ? "enable" : "disable");
 373                 return -EINVAL;
 374         }
 375
 376         pr_debug("%s: cpufreq BOOST %s\n", __func__,
 377                  enable ? "enabled" : "disabled");
 378
 379         return count;
 380 }
 381 define_one_global_rw(boost);
 382
 383 static struct cpufreq_governor *__find_governor(const char *str_governor)
 384 {
 385         struct cpufreq_governor *t;
 386
 387         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 388                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 389                         return t;
 390
 391         return NULL;
 392 }
 393
 394 /**
 395  * cpufreq_parse_governor - parse a governor string
 396  */
 397 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 398                                 struct cpufreq_governor **governor)
 399 {
 400         int err = -EINVAL;
 401
 402         if (!cpufreq_driver)
 403                 goto out;
 404
 405         if (cpufreq_driver->setpolicy) {
 406                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 407                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 408                         err = 0;
 409                 } else if (!strnicmp(str_governor, "powersave",
 410                                                 CPUFREQ_NAME_LEN)) {
 411                         *policy = CPUFREQ_POLICY_POWERSAVE;
 412                         err = 0;
 413                 }
 414         } else if (has_target()) {
 415                 struct cpufreq_governor *t;
 416
 417                 mutex_lock(&cpufreq_governor_mutex);
 418
 419                 t = __find_governor(str_governor);
 420
 421                 if (t == NULL) {
 422                         int ret;
 423
 424                         mutex_unlock(&cpufreq_governor_mutex);
 425                         ret = request_module("cpufreq_%s", str_governor);
 426                         mutex_lock(&cpufreq_governor_mutex);
 427
 428                         if (ret == 0)
 429                                 t = __find_governor(str_governor);
 430                 }
 431
 432                 if (t != NULL) {
 433                         *governor = t;
 434                         err = 0;
 435                 }
 436
 437                 mutex_unlock(&cpufreq_governor_mutex);
 438         }
 439 out:
 440         return err;
 441 }
 442
 443 /**
 444  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 445  * print out cpufreq information
 446  *
 447  * Write out information from cpufreq_driver->policy[cpu]; object must be
 448  * "unsigned int".
 449  */
 450
 451 #define show_one(file_name, object)                     \
 452 static ssize_t show_##file_name                         \
 453 (struct cpufreq_policy *policy, char *buf)              \
 454 {                                                       \
 455         return sprintf(buf, "%u\n", policy->object);    \
 456 }
 457
 458 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 459 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 460 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 461 show_one(scaling_min_freq, min);
 462 show_one(scaling_max_freq, max);
 463
 464 static ssize_t show_scaling_cur_freq(
 465         struct cpufreq_policy *policy, char *buf)
 466 {
 467         ssize_t ret;
 468
 469         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 470                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 471         else
 472                 ret = sprintf(buf, "%u\n", policy->cur);
 473         return ret;
 474 }
 475
 476 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 477                                 struct cpufreq_policy *new_policy);
 478
 479 /**
 480  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 481  */
 482 #define store_one(file_name, object)                    \
 483 static ssize_t store_##file_name                                        \
 484 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 485 {                                                                       \
 486         int ret;                                                        \
 487         struct cpufreq_policy new_policy;                               \
 488                                                                         \
 489         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 490         if (ret)                                                        \
 491                 return -EINVAL;                                         \
 492                                                                         \
 493         ret = sscanf(buf, "%u", &new_policy.object);                    \
 494         if (ret != 1)                                                   \
 495                 return -EINVAL;                                         \
 496                                                                         \
 497         ret = cpufreq_set_policy(policy, &new_policy);          \
 498         policy->user_policy.object = policy->object;                    \
 499                                                                         \
 500         return ret ? ret : count;                                       \
 501 }
 502
 503 store_one(scaling_min_freq, min);
 504 store_one(scaling_max_freq, max);
 505
 506 /**
 507  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 508  */
 509 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 510                                         char *buf)
 511 {
 512         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 513         if (!cur_freq)
 514                 return sprintf(buf, "<unknown>");
 515         return sprintf(buf, "%u\n", cur_freq);
 516 }
 517
 518 /**
 519  * show_scaling_governor - show the current policy for the specified CPU
 520  */
 521 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 522 {
 523         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 524                 return sprintf(buf, "powersave\n");
 525         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 526                 return sprintf(buf, "performance\n");
 527         else if (policy->governor)
 528                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 529                                 policy->governor->name);
 530         return -EINVAL;
 531 }
 532
 533 /**
 534  * store_scaling_governor - store policy for the specified CPU
 535  */
 536 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 537                                         const char *buf, size_t count)
 538 {
 539         int ret;
 540         char    str_governor[16];
 541         struct cpufreq_policy new_policy;
 542
 543         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 544         if (ret)
 545                 return ret;
 546
 547         ret = sscanf(buf, "%15s", str_governor);
 548         if (ret != 1)
 549                 return -EINVAL;
 550
 551         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 552                                                 &new_policy.governor))
 553                 return -EINVAL;
 554
 555         ret = cpufreq_set_policy(policy, &new_policy);
 556
 557         policy->user_policy.policy = policy->policy;
 558         policy->user_policy.governor = policy->governor;
 559
 560         if (ret)
 561                 return ret;
 562         else
 563                 return count;
 564 }
 565
 566 /**
 567  * show_scaling_driver - show the cpufreq driver currently loaded
 568  */
 569 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 570 {
 571         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 572 }
 573
 574 /**
 575  * show_scaling_available_governors - show the available CPUfreq governors
 576  */
 577 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 578                                                 char *buf)
 579 {
 580         ssize_t i = 0;
 581         struct cpufreq_governor *t;
 582
 583         if (!has_target()) {
 584                 i += sprintf(buf, "performance powersave");
 585                 goto out;
 586         }
 587
 588         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 589                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 590                     - (CPUFREQ_NAME_LEN + 2)))
 591                         goto out;
 592                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 593         }
 594 out:
 595         i += sprintf(&buf[i], "\n");
 596         return i;
 597 }
 598
 599 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 600 {
 601         ssize_t i = 0;
 602         unsigned int cpu;
 603
 604         for_each_cpu(cpu, mask) {
 605                 if (i)
 606                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 607                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 608                 if (i >= (PAGE_SIZE - 5))
 609                         break;
 610         }
 611         i += sprintf(&buf[i], "\n");
 612         return i;
 613 }
 614 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 615
 616 /**
 617  * show_related_cpus - show the CPUs affected by each transition even if
 618  * hw coordination is in use
 619  */
 620 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 621 {
 622         return cpufreq_show_cpus(policy->related_cpus, buf);
 623 }
 624
 625 /**
 626  * show_affected_cpus - show the CPUs affected by each transition
 627  */
 628 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 629 {
 630         return cpufreq_show_cpus(policy->cpus, buf);
 631 }
 632
 633 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 634                                         const char *buf, size_t count)
 635 {
 636         unsigned int freq = 0;
 637         unsigned int ret;
 638
 639         if (!policy->governor || !policy->governor->store_setspeed)
 640                 return -EINVAL;
 641
 642         ret = sscanf(buf, "%u", &freq);
 643         if (ret != 1)
 644                 return -EINVAL;
 645
 646         policy->governor->store_setspeed(policy, freq);
 647
 648         return count;
 649 }
 650
 651 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 652 {
 653         if (!policy->governor || !policy->governor->show_setspeed)
 654                 return sprintf(buf, "<unsupported>\n");
 655
 656         return policy->governor->show_setspeed(policy, buf);
 657 }
 658
 659 /**
 660  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 661  */
 662 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 663 {
 664         unsigned int limit;
 665         int ret;
 666         if (cpufreq_driver->bios_limit) {
 667                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 668                 if (!ret)
 669                         return sprintf(buf, "%u\n", limit);
 670         }
 671         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 672 }
 673
 674 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 675 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 676 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 677 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 678 cpufreq_freq_attr_ro(scaling_available_governors);
 679 cpufreq_freq_attr_ro(scaling_driver);
 680 cpufreq_freq_attr_ro(scaling_cur_freq);
 681 cpufreq_freq_attr_ro(bios_limit);
 682 cpufreq_freq_attr_ro(related_cpus);
 683 cpufreq_freq_attr_ro(affected_cpus);
 684 cpufreq_freq_attr_rw(scaling_min_freq);
 685 cpufreq_freq_attr_rw(scaling_max_freq);
 686 cpufreq_freq_attr_rw(scaling_governor);
 687 cpufreq_freq_attr_rw(scaling_setspeed);
 688
 689 static struct attribute *default_attrs[] = {
 690         &cpuinfo_min_freq.attr,
 691         &cpuinfo_max_freq.attr,
 692         &cpuinfo_transition_latency.attr,
 693         &scaling_min_freq.attr,
 694         &scaling_max_freq.attr,
 695         &affected_cpus.attr,
 696         &related_cpus.attr,
 697         &scaling_governor.attr,
 698         &scaling_driver.attr,
 699         &scaling_available_governors.attr,
 700         &scaling_setspeed.attr,
 701         NULL
 702 };
 703
 704 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 705 #define to_attr(a) container_of(a, struct freq_attr, attr)
 706
 707 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 708 {
 709         struct cpufreq_policy *policy = to_policy(kobj);
 710         struct freq_attr *fattr = to_attr(attr);
 711         ssize_t ret;
 712
 713         if (!down_read_trylock(&cpufreq_rwsem))
 714                 return -EINVAL;
 715
 716         down_read(&policy->rwsem);
 717
 718         if (fattr->show)
 719                 ret = fattr->show(policy, buf);
 720         else
 721                 ret = -EIO;
 722
 723         up_read(&policy->rwsem);
 724         up_read(&cpufreq_rwsem);
 725
 726         return ret;
 727 }
 728
 729 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 730                      const char *buf, size_t count)
 731 {
 732         struct cpufreq_policy *policy = to_policy(kobj);
 733         struct freq_attr *fattr = to_attr(attr);
 734         ssize_t ret = -EINVAL;
 735
 736         get_online_cpus();
 737
 738         if (!cpu_online(policy->cpu))
 739                 goto unlock;
 740
 741         if (!down_read_trylock(&cpufreq_rwsem))
 742                 goto unlock;
 743
 744         down_write(&policy->rwsem);
 745
 746         if (fattr->store)
 747                 ret = fattr->store(policy, buf, count);
 748         else
 749                 ret = -EIO;
 750
 751         up_write(&policy->rwsem);
 752
 753         up_read(&cpufreq_rwsem);
 754 unlock:
 755         put_online_cpus();
 756
 757         return ret;
 758 }
 759
 760 static void cpufreq_sysfs_release(struct kobject *kobj)
 761 {
 762         struct cpufreq_policy *policy = to_policy(kobj);
 763         pr_debug("last reference is dropped\n");
 764         complete(&policy->kobj_unregister);
 765 }
 766
 767 static const struct sysfs_ops sysfs_ops = {
 768         .show   = show,
 769         .store  = store,
 770 };
 771
 772 static struct kobj_type ktype_cpufreq = {
 773         .sysfs_ops      = &sysfs_ops,
 774         .default_attrs  = default_attrs,
 775         .release        = cpufreq_sysfs_release,
 776 };
 777
 778 struct kobject *cpufreq_global_kobject;
 779 EXPORT_SYMBOL(cpufreq_global_kobject);
 780
 781 static int cpufreq_global_kobject_usage;
 782
 783 int cpufreq_get_global_kobject(void)
 784 {
 785         if (!cpufreq_global_kobject_usage++)
 786                 return kobject_add(cpufreq_global_kobject,
 787                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 788
 789         return 0;
 790 }
 791 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 792
 793 void cpufreq_put_global_kobject(void)
 794 {
 795         if (!--cpufreq_global_kobject_usage)
 796                 kobject_del(cpufreq_global_kobject);
 797 }
 798 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 799
 800 int cpufreq_sysfs_create_file(const struct attribute *attr)
 801 {
 802         int ret = cpufreq_get_global_kobject();
 803
 804         if (!ret) {
 805                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 806                 if (ret)
 807                         cpufreq_put_global_kobject();
 808         }
 809
 810         return ret;
 811 }
 812 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 813
 814 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 815 {
 816         sysfs_remove_file(cpufreq_global_kobject, attr);
 817         cpufreq_put_global_kobject();
 818 }
 819 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 820
 821 /* symlink affected CPUs */
 822 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 823 {
 824         unsigned int j;
 825         int ret = 0;
 826
 827         for_each_cpu(j, policy->cpus) {
 828                 struct device *cpu_dev;
 829
 830                 if (j == policy->cpu)
 831                         continue;
 832
 833                 pr_debug("Adding link for CPU: %u\n", j);
 834                 cpu_dev = get_cpu_device(j);
 835                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 836                                         "cpufreq");
 837                 if (ret)
 838                         break;
 839         }
 840         return ret;
 841 }
 842
 843 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 844                                      struct device *dev)
 845 {
 846         struct freq_attr **drv_attr;
 847         int ret = 0;
 848
 849         /* prepare interface data */
 850         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 851                                    &dev->kobj, "cpufreq");
 852         if (ret)
 853                 return ret;
 854
 855         /* set up files for this cpu device */
 856         drv_attr = cpufreq_driver->attr;
 857         while ((drv_attr) && (*drv_attr)) {
 858                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 859                 if (ret)
 860                         goto err_out_kobj_put;
 861                 drv_attr++;
 862         }
 863         if (cpufreq_driver->get) {
 864                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 865                 if (ret)
 866                         goto err_out_kobj_put;
 867         }
 868
 869         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 870         if (ret)
 871                 goto err_out_kobj_put;
 872
 873         if (cpufreq_driver->bios_limit) {
 874                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 875                 if (ret)
 876                         goto err_out_kobj_put;
 877         }
 878
 879         ret = cpufreq_add_dev_symlink(policy);
 880         if (ret)
 881                 goto err_out_kobj_put;
 882
 883         return ret;
 884
 885 err_out_kobj_put:
 886         kobject_put(&policy->kobj);
 887         wait_for_completion(&policy->kobj_unregister);
 888         return ret;
 889 }
 890
 891 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 892 {
 893         struct cpufreq_policy new_policy;
 894         int ret = 0;
 895
 896         memcpy(&new_policy, policy, sizeof(*policy));
 897
 898         /* Use the default policy if its valid. */
 899         if (cpufreq_driver->setpolicy)
 900                 cpufreq_parse_governor(policy->governor->name,
 901                                         &new_policy.policy, NULL);
 902
 903         /* assure that the starting sequence is run in cpufreq_set_policy */
 904         policy->governor = NULL;
 905
 906         /* set default policy */
 907         ret = cpufreq_set_policy(policy, &new_policy);
 908         if (ret) {
 909                 pr_debug("setting policy failed\n");
 910                 if (cpufreq_driver->exit)
 911                         cpufreq_driver->exit(policy);
 912         }
 913 }
 914
 915 #ifdef CONFIG_HOTPLUG_CPU
 916 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 917                                   unsigned int cpu, struct device *dev)
 918 {
 919         int ret = 0;
 920         unsigned long flags;
 921
 922         if (has_target()) {
 923                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 924                 if (ret) {
 925                         pr_err("%s: Failed to stop governor\n", __func__);
 926                         return ret;
 927                 }
 928         }
 929
 930         down_write(&policy->rwsem);
 931
 932         write_lock_irqsave(&cpufreq_driver_lock, flags);
 933
 934         cpumask_set_cpu(cpu, policy->cpus);
 935         per_cpu(cpufreq_cpu_data, cpu) = policy;
 936         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 937
 938         up_write(&policy->rwsem);
 939
 940         if (has_target()) {
 941                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 942                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 943                         pr_err("%s: Failed to start governor\n", __func__);
 944                         return ret;
 945                 }
 946         }
 947
 948         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 949 }
 950 #endif
 951
 952 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 953 {
 954         struct cpufreq_policy *policy;
 955         unsigned long flags;
 956
 957         read_lock_irqsave(&cpufreq_driver_lock, flags);
 958
 959         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 960
 961         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 962
 963         return policy;
 964 }
 965
 966 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 967 {
 968         struct cpufreq_policy *policy;
 969
 970         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 971         if (!policy)
 972                 return NULL;
 973
 974         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 975                 goto err_free_policy;
 976
 977         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 978                 goto err_free_cpumask;
 979
 980         INIT_LIST_HEAD(&policy->policy_list);
 981         init_rwsem(&policy->rwsem);
 982
 983         return policy;
 984
 985 err_free_cpumask:
 986         free_cpumask_var(policy->cpus);
 987 err_free_policy:
 988         kfree(policy);
 989
 990         return NULL;
 991 }
 992
 993 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
 994 {
 995         struct kobject *kobj;
 996         struct completion *cmp;
 997
 998         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 999                         CPUFREQ_REMOVE_POLICY, policy);
1000
1001         down_read(&policy->rwsem);
1002         kobj = &policy->kobj;
1003         cmp = &policy->kobj_unregister;
1004         up_read(&policy->rwsem);
1005         kobject_put(kobj);
1006
1007         /*
1008          * We need to make sure that the underlying kobj is
1009          * actually not referenced anymore by anybody before we
1010          * proceed with unloading.
1011          */
1012         pr_debug("waiting for dropping of refcount\n");
1013         wait_for_completion(cmp);
1014         pr_debug("wait complete\n");
1015 }
1016
1017 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1018 {
1019         free_cpumask_var(policy->related_cpus);
1020         free_cpumask_var(policy->cpus);
1021         kfree(policy);
1022 }
1023
1024 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1025 {
1026         if (WARN_ON(cpu == policy->cpu))
1027                 return;
1028
1029         down_write(&policy->rwsem);
1030
1031         policy->last_cpu = policy->cpu;
1032         policy->cpu = cpu;
1033
1034         up_write(&policy->rwsem);
1035
1036         cpufreq_frequency_table_update_policy_cpu(policy);
1037         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1038                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1039 }
1040
1041 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
1042                              bool frozen)
1043 {
1044         unsigned int j, cpu = dev->id;
1045         int ret = -ENOMEM;
1046         struct cpufreq_policy *policy;
1047         unsigned long flags;
1048 #ifdef CONFIG_HOTPLUG_CPU
1049         struct cpufreq_policy *tpolicy;
1050         struct cpufreq_governor *gov;
1051 #endif
1052
1053         if (cpu_is_offline(cpu))
1054                 return 0;
1055
1056         pr_debug("adding CPU %u\n", cpu);
1057
1058 #ifdef CONFIG_SMP
1059         /* check whether a different CPU already registered this
1060          * CPU because it is in the same boat. */
1061         policy = cpufreq_cpu_get(cpu);
1062         if (unlikely(policy)) {
1063                 cpufreq_cpu_put(policy);
1064                 return 0;
1065         }
1066 #endif
1067
1068         if (!down_read_trylock(&cpufreq_rwsem))
1069                 return 0;
1070
1071 #ifdef CONFIG_HOTPLUG_CPU
1072         /* Check if this cpu was hot-unplugged earlier and has siblings */
1073         read_lock_irqsave(&cpufreq_driver_lock, flags);
1074         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1075                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1076                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1077                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1078                         up_read(&cpufreq_rwsem);
1079                         return ret;
1080                 }
1081         }
1082         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1083 #endif
1084
1085         /*
1086          * Restore the saved policy when doing light-weight init and fall back
1087          * to the full init if that fails.
1088          */
1089         policy = frozen ? cpufreq_policy_restore(cpu) : NULL;
1090         if (!policy) {
1091                 frozen = false;
1092                 policy = cpufreq_policy_alloc();
1093                 if (!policy)
1094                         goto nomem_out;
1095         }
1096
1097         /*
1098          * In the resume path, since we restore a saved policy, the assignment
1099          * to policy->cpu is like an update of the existing policy, rather than
1100          * the creation of a brand new one. So we need to perform this update
1101          * by invoking update_policy_cpu().
1102          */
1103         if (frozen && cpu != policy->cpu) {
1104                 update_policy_cpu(policy, cpu);
1105                 WARN_ON(kobject_move(&policy->kobj, &dev->kobj));
1106         } else {
1107                 policy->cpu = cpu;
1108         }
1109
1110         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1111         cpumask_copy(policy->cpus, cpumask_of(cpu));
1112
1113         init_completion(&policy->kobj_unregister);
1114         INIT_WORK(&policy->update, handle_update);
1115
1116         /* call driver. From then on the cpufreq must be able
1117          * to accept all calls to ->verify and ->setpolicy for this CPU
1118          */
1119         ret = cpufreq_driver->init(policy);
1120         if (ret) {
1121                 pr_debug("initialization failed\n");
1122                 goto err_set_policy_cpu;
1123         }
1124
1125         /* related cpus should atleast have policy->cpus */
1126         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1127
1128         /*
1129          * affected cpus must always be the one, which are online. We aren't
1130          * managing offline cpus here.
1131          */
1132         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1133
1134         if (!frozen) {
1135                 policy->user_policy.min = policy->min;
1136                 policy->user_policy.max = policy->max;
1137         }
1138
1139         down_write(&policy->rwsem);
1140         write_lock_irqsave(&cpufreq_driver_lock, flags);
1141         for_each_cpu(j, policy->cpus)
1142                 per_cpu(cpufreq_cpu_data, j) = policy;
1143         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1144
1145         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1146                 policy->cur = cpufreq_driver->get(policy->cpu);
1147                 if (!policy->cur) {
1148                         pr_err("%s: ->get() failed\n", __func__);
1149                         goto err_get_freq;
1150                 }
1151         }
1152
1153         /*
1154          * Sometimes boot loaders set CPU frequency to a value outside of
1155          * frequency table present with cpufreq core. In such cases CPU might be
1156          * unstable if it has to run on that frequency for long duration of time
1157          * and so its better to set it to a frequency which is specified in
1158          * freq-table. This also makes cpufreq stats inconsistent as
1159          * cpufreq-stats would fail to register because current frequency of CPU
1160          * isn't found in freq-table.
1161          *
1162          * Because we don't want this change to effect boot process badly, we go
1163          * for the next freq which is >= policy->cur ('cur' must be set by now,
1164          * otherwise we will end up setting freq to lowest of the table as 'cur'
1165          * is initialized to zero).
1166          *
1167          * We are passing target-freq as "policy->cur - 1" otherwise
1168          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1169          * equal to target-freq.
1170          */
1171         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1172             && has_target()) {
1173                 /* Are we running at unknown frequency ? */
1174                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1175                 if (ret == -EINVAL) {
1176                         /* Warn user and fix it */
1177                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1178                                 __func__, policy->cpu, policy->cur);
1179                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1180                                 CPUFREQ_RELATION_L);
1181
1182                         /*
1183                          * Reaching here after boot in a few seconds may not
1184                          * mean that system will remain stable at "unknown"
1185                          * frequency for longer duration. Hence, a BUG_ON().
1186                          */
1187                         BUG_ON(ret);
1188                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1189                                 __func__, policy->cpu, policy->cur);
1190                 }
1191         }
1192
1193         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1194                                      CPUFREQ_START, policy);
1195
1196 #ifdef CONFIG_HOTPLUG_CPU
1197         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1198         if (gov) {
1199                 policy->governor = gov;
1200                 pr_debug("Restoring governor %s for cpu %d\n",
1201                        policy->governor->name, cpu);
1202         }
1203 #endif
1204
1205         if (!frozen) {
1206                 ret = cpufreq_add_dev_interface(policy, dev);
1207                 if (ret)
1208                         goto err_out_unregister;
1209                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1210                                 CPUFREQ_CREATE_POLICY, policy);
1211         }
1212
1213         write_lock_irqsave(&cpufreq_driver_lock, flags);
1214         list_add(&policy->policy_list, &cpufreq_policy_list);
1215         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1216
1217         cpufreq_init_policy(policy);
1218
1219         if (!frozen) {
1220                 policy->user_policy.policy = policy->policy;
1221                 policy->user_policy.governor = policy->governor;
1222         }
1223         up_write(&policy->rwsem);
1224
1225         kobject_uevent(&policy->kobj, KOBJ_ADD);
1226         up_read(&cpufreq_rwsem);
1227
1228         pr_debug("initialization complete\n");
1229
1230         return 0;
1231
1232 err_out_unregister:
1233 err_get_freq:
1234         write_lock_irqsave(&cpufreq_driver_lock, flags);
1235         for_each_cpu(j, policy->cpus)
1236                 per_cpu(cpufreq_cpu_data, j) = NULL;
1237         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1238
1239         up_write(&policy->rwsem);
1240
1241         if (cpufreq_driver->exit)
1242                 cpufreq_driver->exit(policy);
1243 err_set_policy_cpu:
1244         if (frozen) {
1245                 /* Do not leave stale fallback data behind. */
1246                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1247                 cpufreq_policy_put_kobj(policy);
1248         }
1249         cpufreq_policy_free(policy);
1250
1251 nomem_out:
1252         up_read(&cpufreq_rwsem);
1253
1254         return ret;
1255 }
1256
1257 /**
1258  * cpufreq_add_dev - add a CPU device
1259  *
1260  * Adds the cpufreq interface for a CPU device.
1261  *
1262  * The Oracle says: try running cpufreq registration/unregistration concurrently
1263  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1264  * mess up, but more thorough testing is needed. - Mathieu
1265  */
1266 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1267 {
1268         return __cpufreq_add_dev(dev, sif, false);
1269 }
1270
1271 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1272                                            unsigned int old_cpu)
1273 {
1274         struct device *cpu_dev;
1275         int ret;
1276
1277         /* first sibling now owns the new sysfs dir */
1278         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1279
1280         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1281         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1282         if (ret) {
1283                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1284
1285                 down_write(&policy->rwsem);
1286                 cpumask_set_cpu(old_cpu, policy->cpus);
1287                 up_write(&policy->rwsem);
1288
1289                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1290                                         "cpufreq");
1291
1292                 return -EINVAL;
1293         }
1294
1295         return cpu_dev->id;
1296 }
1297
1298 static int __cpufreq_remove_dev_prepare(struct device *dev,
1299                                         struct subsys_interface *sif,
1300                                         bool frozen)
1301 {
1302         unsigned int cpu = dev->id, cpus;
1303         int new_cpu, ret;
1304         unsigned long flags;
1305         struct cpufreq_policy *policy;
1306
1307         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1308
1309         write_lock_irqsave(&cpufreq_driver_lock, flags);
1310
1311         policy = per_cpu(cpufreq_cpu_data, cpu);
1312
1313         /* Save the policy somewhere when doing a light-weight tear-down */
1314         if (frozen)
1315                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1316
1317         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1318
1319         if (!policy) {
1320                 pr_debug("%s: No cpu_data found\n", __func__);
1321                 return -EINVAL;
1322         }
1323
1324         if (has_target()) {
1325                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1326                 if (ret) {
1327                         pr_err("%s: Failed to stop governor\n", __func__);
1328                         return ret;
1329                 }
1330         }
1331
1332 #ifdef CONFIG_HOTPLUG_CPU
1333         if (!cpufreq_driver->setpolicy)
1334                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1335                         policy->governor->name, CPUFREQ_NAME_LEN);
1336 #endif
1337
1338         down_read(&policy->rwsem);
1339         cpus = cpumask_weight(policy->cpus);
1340         up_read(&policy->rwsem);
1341
1342         if (cpu != policy->cpu) {
1343                 sysfs_remove_link(&dev->kobj, "cpufreq");
1344         } else if (cpus > 1) {
1345                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1346                 if (new_cpu >= 0) {
1347                         update_policy_cpu(policy, new_cpu);
1348
1349                         if (!frozen) {
1350                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1351                                                 __func__, new_cpu, cpu);
1352                         }
1353                 }
1354         }
1355
1356         return 0;
1357 }
1358
1359 static int __cpufreq_remove_dev_finish(struct device *dev,
1360                                        struct subsys_interface *sif,
1361                                        bool frozen)
1362 {
1363         unsigned int cpu = dev->id, cpus;
1364         int ret;
1365         unsigned long flags;
1366         struct cpufreq_policy *policy;
1367
1368         read_lock_irqsave(&cpufreq_driver_lock, flags);
1369         policy = per_cpu(cpufreq_cpu_data, cpu);
1370         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1371
1372         if (!policy) {
1373                 pr_debug("%s: No cpu_data found\n", __func__);
1374                 return -EINVAL;
1375         }
1376
1377         down_write(&policy->rwsem);
1378         cpus = cpumask_weight(policy->cpus);
1379
1380         if (cpus > 1)
1381                 cpumask_clear_cpu(cpu, policy->cpus);
1382         up_write(&policy->rwsem);
1383
1384         /* If cpu is last user of policy, free policy */
1385         if (cpus == 1) {
1386                 if (has_target()) {
1387                         ret = __cpufreq_governor(policy,
1388                                         CPUFREQ_GOV_POLICY_EXIT);
1389                         if (ret) {
1390                                 pr_err("%s: Failed to exit governor\n",
1391                                                 __func__);
1392                                 return ret;
1393                         }
1394                 }
1395
1396                 if (!frozen)
1397                         cpufreq_policy_put_kobj(policy);
1398
1399                 /*
1400                  * Perform the ->exit() even during light-weight tear-down,
1401                  * since this is a core component, and is essential for the
1402                  * subsequent light-weight ->init() to succeed.
1403                  */
1404                 if (cpufreq_driver->exit)
1405                         cpufreq_driver->exit(policy);
1406
1407                 /* Remove policy from list of active policies */
1408                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1409                 list_del(&policy->policy_list);
1410                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1411
1412                 if (!frozen)
1413                         cpufreq_policy_free(policy);
1414         } else {
1415                 if (has_target()) {
1416                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1417                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1418                                 pr_err("%s: Failed to start governor\n",
1419                                                 __func__);
1420                                 return ret;
1421                         }
1422                 }
1423         }
1424
1425         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1426         return 0;
1427 }
1428
1429 /**
1430  * cpufreq_remove_dev - remove a CPU device
1431  *
1432  * Removes the cpufreq interface for a CPU device.
1433  */
1434 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1435 {
1436         unsigned int cpu = dev->id;
1437         int ret;
1438
1439         if (cpu_is_offline(cpu))
1440                 return 0;
1441
1442         ret = __cpufreq_remove_dev_prepare(dev, sif, false);
1443
1444         if (!ret)
1445                 ret = __cpufreq_remove_dev_finish(dev, sif, false);
1446
1447         return ret;
1448 }
1449
1450 static void handle_update(struct work_struct *work)
1451 {
1452         struct cpufreq_policy *policy =
1453                 container_of(work, struct cpufreq_policy, update);
1454         unsigned int cpu = policy->cpu;
1455         pr_debug("handle_update for cpu %u called\n", cpu);
1456         cpufreq_update_policy(cpu);
1457 }
1458
1459 /**
1460  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1461  *      in deep trouble.
1462  *      @cpu: cpu number
1463  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1464  *      @new_freq: CPU frequency the CPU actually runs at
1465  *
1466  *      We adjust to current frequency first, and need to clean up later.
1467  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1468  */
1469 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1470                                 unsigned int new_freq)
1471 {
1472         struct cpufreq_policy *policy;
1473         struct cpufreq_freqs freqs;
1474         unsigned long flags;
1475
1476         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1477                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1478
1479         freqs.old = old_freq;
1480         freqs.new = new_freq;
1481
1482         read_lock_irqsave(&cpufreq_driver_lock, flags);
1483         policy = per_cpu(cpufreq_cpu_data, cpu);
1484         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1485
1486         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1487         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1488 }
1489
1490 /**
1491  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1492  * @cpu: CPU number
1493  *
1494  * This is the last known freq, without actually getting it from the driver.
1495  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1496  */
1497 unsigned int cpufreq_quick_get(unsigned int cpu)
1498 {
1499         struct cpufreq_policy *policy;
1500         unsigned int ret_freq = 0;
1501
1502         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1503                 return cpufreq_driver->get(cpu);
1504
1505         policy = cpufreq_cpu_get(cpu);
1506         if (policy) {
1507                 ret_freq = policy->cur;
1508                 cpufreq_cpu_put(policy);
1509         }
1510
1511         return ret_freq;
1512 }
1513 EXPORT_SYMBOL(cpufreq_quick_get);
1514
1515 /**
1516  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1517  * @cpu: CPU number
1518  *
1519  * Just return the max possible frequency for a given CPU.
1520  */
1521 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1522 {
1523         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1524         unsigned int ret_freq = 0;
1525
1526         if (policy) {
1527                 ret_freq = policy->max;
1528                 cpufreq_cpu_put(policy);
1529         }
1530
1531         return ret_freq;
1532 }
1533 EXPORT_SYMBOL(cpufreq_quick_get_max);
1534
1535 static unsigned int __cpufreq_get(unsigned int cpu)
1536 {
1537         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1538         unsigned int ret_freq = 0;
1539
1540         if (!cpufreq_driver->get)
1541                 return ret_freq;
1542
1543         ret_freq = cpufreq_driver->get(cpu);
1544
1545         if (ret_freq && policy->cur &&
1546                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1547                 /* verify no discrepancy between actual and
1548                                         saved value exists */
1549                 if (unlikely(ret_freq != policy->cur)) {
1550                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1551                         schedule_work(&policy->update);
1552                 }
1553         }
1554
1555         return ret_freq;
1556 }
1557
1558 /**
1559  * cpufreq_get - get the current CPU frequency (in kHz)
1560  * @cpu: CPU number
1561  *
1562  * Get the CPU current (static) CPU frequency
1563  */
1564 unsigned int cpufreq_get(unsigned int cpu)
1565 {
1566         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1567         unsigned int ret_freq = 0;
1568
1569         if (policy) {
1570                 down_read(&policy->rwsem);
1571                 ret_freq = __cpufreq_get(cpu);
1572                 up_read(&policy->rwsem);
1573
1574                 cpufreq_cpu_put(policy);
1575         }
1576
1577         return ret_freq;
1578 }
1579 EXPORT_SYMBOL(cpufreq_get);
1580
1581 static struct subsys_interface cpufreq_interface = {
1582         .name           = "cpufreq",
1583         .subsys         = &cpu_subsys,
1584         .add_dev        = cpufreq_add_dev,
1585         .remove_dev     = cpufreq_remove_dev,
1586 };
1587
1588 /**
1589  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1590  *
1591  * This function is only executed for the boot processor.  The other CPUs
1592  * have been put offline by means of CPU hotplug.
1593  */
1594 static int cpufreq_bp_suspend(void)
1595 {
1596         int ret = 0;
1597
1598         int cpu = smp_processor_id();
1599         struct cpufreq_policy *policy;
1600
1601         pr_debug("suspending cpu %u\n", cpu);
1602
1603         /* If there's no policy for the boot CPU, we have nothing to do. */
1604         policy = cpufreq_cpu_get(cpu);
1605         if (!policy)
1606                 return 0;
1607
1608         if (cpufreq_driver->suspend) {
1609                 ret = cpufreq_driver->suspend(policy);
1610                 if (ret)
1611                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1612                                         "step on CPU %u\n", policy->cpu);
1613         }
1614
1615         cpufreq_cpu_put(policy);
1616         return ret;
1617 }
1618
1619 /**
1620  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1621  *
1622  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1623  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1624  *          restored. It will verify that the current freq is in sync with
1625  *          what we believe it to be. This is a bit later than when it
1626  *          should be, but nonethteless it's better than calling
1627  *          cpufreq_driver->get() here which might re-enable interrupts...
1628  *
1629  * This function is only executed for the boot CPU.  The other CPUs have not
1630  * been turned on yet.
1631  */
1632 static void cpufreq_bp_resume(void)
1633 {
1634         int ret = 0;
1635
1636         int cpu = smp_processor_id();
1637         struct cpufreq_policy *policy;
1638
1639         pr_debug("resuming cpu %u\n", cpu);
1640
1641         /* If there's no policy for the boot CPU, we have nothing to do. */
1642         policy = cpufreq_cpu_get(cpu);
1643         if (!policy)
1644                 return;
1645
1646         if (cpufreq_driver->resume) {
1647                 ret = cpufreq_driver->resume(policy);
1648                 if (ret) {
1649                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1650                                         "step on CPU %u\n", policy->cpu);
1651                         goto fail;
1652                 }
1653         }
1654
1655         schedule_work(&policy->update);
1656
1657 fail:
1658         cpufreq_cpu_put(policy);
1659 }
1660
1661 static struct syscore_ops cpufreq_syscore_ops = {
1662         .suspend        = cpufreq_bp_suspend,
1663         .resume         = cpufreq_bp_resume,
1664 };
1665
1666 /**
1667  *      cpufreq_get_current_driver - return current driver's name
1668  *
1669  *      Return the name string of the currently loaded cpufreq driver
1670  *      or NULL, if none.
1671  */
1672 const char *cpufreq_get_current_driver(void)
1673 {
1674         if (cpufreq_driver)
1675                 return cpufreq_driver->name;
1676
1677         return NULL;
1678 }
1679 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1680
1681 /*********************************************************************
1682  *                     NOTIFIER LISTS INTERFACE                      *
1683  *********************************************************************/
1684
1685 /**
1686  *      cpufreq_register_notifier - register a driver with cpufreq
1687  *      @nb: notifier function to register
1688  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1689  *
1690  *      Add a driver to one of two lists: either a list of drivers that
1691  *      are notified about clock rate changes (once before and once after
1692  *      the transition), or a list of drivers that are notified about
1693  *      changes in cpufreq policy.
1694  *
1695  *      This function may sleep, and has the same return conditions as
1696  *      blocking_notifier_chain_register.
1697  */
1698 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1699 {
1700         int ret;
1701
1702         if (cpufreq_disabled())
1703                 return -EINVAL;
1704
1705         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1706
1707         switch (list) {
1708         case CPUFREQ_TRANSITION_NOTIFIER:
1709                 ret = srcu_notifier_chain_register(
1710                                 &cpufreq_transition_notifier_list, nb);
1711                 break;
1712         case CPUFREQ_POLICY_NOTIFIER:
1713                 ret = blocking_notifier_chain_register(
1714                                 &cpufreq_policy_notifier_list, nb);
1715                 break;
1716         default:
1717                 ret = -EINVAL;
1718         }
1719
1720         return ret;
1721 }
1722 EXPORT_SYMBOL(cpufreq_register_notifier);
1723
1724 /**
1725  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1726  *      @nb: notifier block to be unregistered
1727  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1728  *
1729  *      Remove a driver from the CPU frequency notifier list.
1730  *
1731  *      This function may sleep, and has the same return conditions as
1732  *      blocking_notifier_chain_unregister.
1733  */
1734 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1735 {
1736         int ret;
1737
1738         if (cpufreq_disabled())
1739                 return -EINVAL;
1740
1741         switch (list) {
1742         case CPUFREQ_TRANSITION_NOTIFIER:
1743                 ret = srcu_notifier_chain_unregister(
1744                                 &cpufreq_transition_notifier_list, nb);
1745                 break;
1746         case CPUFREQ_POLICY_NOTIFIER:
1747                 ret = blocking_notifier_chain_unregister(
1748                                 &cpufreq_policy_notifier_list, nb);
1749                 break;
1750         default:
1751                 ret = -EINVAL;
1752         }
1753
1754         return ret;
1755 }
1756 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1757
1758
1759 /*********************************************************************
1760  *                              GOVERNORS                            *
1761  *********************************************************************/
1762
1763 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1764                             unsigned int target_freq,
1765                             unsigned int relation)
1766 {
1767         int retval = -EINVAL;
1768         unsigned int old_target_freq = target_freq;
1769
1770         if (cpufreq_disabled())
1771                 return -ENODEV;
1772
1773         /* Make sure that target_freq is within supported range */
1774         if (target_freq > policy->max)
1775                 target_freq = policy->max;
1776         if (target_freq < policy->min)
1777                 target_freq = policy->min;
1778
1779         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1780                         policy->cpu, target_freq, relation, old_target_freq);
1781
1782         /*
1783          * This might look like a redundant call as we are checking it again
1784          * after finding index. But it is left intentionally for cases where
1785          * exactly same freq is called again and so we can save on few function
1786          * calls.
1787          */
1788         if (target_freq == policy->cur)
1789                 return 0;
1790
1791         if (cpufreq_driver->target)
1792                 retval = cpufreq_driver->target(policy, target_freq, relation);
1793         else if (cpufreq_driver->target_index) {
1794                 struct cpufreq_frequency_table *freq_table;
1795                 struct cpufreq_freqs freqs;
1796                 bool notify;
1797                 int index;
1798
1799                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1800                 if (unlikely(!freq_table)) {
1801                         pr_err("%s: Unable to find freq_table\n", __func__);
1802                         goto out;
1803                 }
1804
1805                 retval = cpufreq_frequency_table_target(policy, freq_table,
1806                                 target_freq, relation, &index);
1807                 if (unlikely(retval)) {
1808                         pr_err("%s: Unable to find matching freq\n", __func__);
1809                         goto out;
1810                 }
1811
1812                 if (freq_table[index].frequency == policy->cur) {
1813                         retval = 0;
1814                         goto out;
1815                 }
1816
1817                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1818
1819                 if (notify) {
1820                         freqs.old = policy->cur;
1821                         freqs.new = freq_table[index].frequency;
1822                         freqs.flags = 0;
1823
1824                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1825                                         __func__, policy->cpu, freqs.old,
1826                                         freqs.new);
1827
1828                         cpufreq_notify_transition(policy, &freqs,
1829                                         CPUFREQ_PRECHANGE);
1830                 }
1831
1832                 retval = cpufreq_driver->target_index(policy, index);
1833                 if (retval)
1834                         pr_err("%s: Failed to change cpu frequency: %d\n",
1835                                         __func__, retval);
1836
1837                 if (notify)
1838                         cpufreq_notify_post_transition(policy, &freqs, retval);
1839         }
1840
1841 out:
1842         return retval;
1843 }
1844 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1845
1846 int cpufreq_driver_target(struct cpufreq_policy *policy,
1847                           unsigned int target_freq,
1848                           unsigned int relation)
1849 {
1850         int ret = -EINVAL;
1851
1852         down_write(&policy->rwsem);
1853
1854         ret = __cpufreq_driver_target(policy, target_freq, relation);
1855
1856         up_write(&policy->rwsem);
1857
1858         return ret;
1859 }
1860 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1861
1862 /*
1863  * when "event" is CPUFREQ_GOV_LIMITS
1864  */
1865
1866 static int __cpufreq_governor(struct cpufreq_policy *policy,
1867                                         unsigned int event)
1868 {
1869         int ret;
1870
1871         /* Only must be defined when default governor is known to have latency
1872            restrictions, like e.g. conservative or ondemand.
1873            That this is the case is already ensured in Kconfig
1874         */
1875 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1876         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1877 #else
1878         struct cpufreq_governor *gov = NULL;
1879 #endif
1880
1881         if (policy->governor->max_transition_latency &&
1882             policy->cpuinfo.transition_latency >
1883             policy->governor->max_transition_latency) {
1884                 if (!gov)
1885                         return -EINVAL;
1886                 else {
1887                         printk(KERN_WARNING "%s governor failed, too long"
1888                                " transition latency of HW, fallback"
1889                                " to %s governor\n",
1890                                policy->governor->name,
1891                                gov->name);
1892                         policy->governor = gov;
1893                 }
1894         }
1895
1896         if (event == CPUFREQ_GOV_POLICY_INIT)
1897                 if (!try_module_get(policy->governor->owner))
1898                         return -EINVAL;
1899
1900         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1901                                                 policy->cpu, event);
1902
1903         mutex_lock(&cpufreq_governor_lock);
1904         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1905             || (!policy->governor_enabled
1906             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1907                 mutex_unlock(&cpufreq_governor_lock);
1908                 return -EBUSY;
1909         }
1910
1911         if (event == CPUFREQ_GOV_STOP)
1912                 policy->governor_enabled = false;
1913         else if (event == CPUFREQ_GOV_START)
1914                 policy->governor_enabled = true;
1915
1916         mutex_unlock(&cpufreq_governor_lock);
1917
1918         ret = policy->governor->governor(policy, event);
1919
1920         if (!ret) {
1921                 if (event == CPUFREQ_GOV_POLICY_INIT)
1922                         policy->governor->initialized++;
1923                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1924                         policy->governor->initialized--;
1925         } else {
1926                 /* Restore original values */
1927                 mutex_lock(&cpufreq_governor_lock);
1928                 if (event == CPUFREQ_GOV_STOP)
1929                         policy->governor_enabled = true;
1930                 else if (event == CPUFREQ_GOV_START)
1931                         policy->governor_enabled = false;
1932                 mutex_unlock(&cpufreq_governor_lock);
1933         }
1934
1935         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1936                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1937                 module_put(policy->governor->owner);
1938
1939         return ret;
1940 }
1941
1942 int cpufreq_register_governor(struct cpufreq_governor *governor)
1943 {
1944         int err;
1945
1946         if (!governor)
1947                 return -EINVAL;
1948
1949         if (cpufreq_disabled())
1950                 return -ENODEV;
1951
1952         mutex_lock(&cpufreq_governor_mutex);
1953
1954         governor->initialized = 0;
1955         err = -EBUSY;
1956         if (__find_governor(governor->name) == NULL) {
1957                 err = 0;
1958                 list_add(&governor->governor_list, &cpufreq_governor_list);
1959         }
1960
1961         mutex_unlock(&cpufreq_governor_mutex);
1962         return err;
1963 }
1964 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1965
1966 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1967 {
1968 #ifdef CONFIG_HOTPLUG_CPU
1969         int cpu;
1970 #endif
1971
1972         if (!governor)
1973                 return;
1974
1975         if (cpufreq_disabled())
1976                 return;
1977
1978 #ifdef CONFIG_HOTPLUG_CPU
1979         for_each_present_cpu(cpu) {
1980                 if (cpu_online(cpu))
1981                         continue;
1982                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1983                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1984         }
1985 #endif
1986
1987         mutex_lock(&cpufreq_governor_mutex);
1988         list_del(&governor->governor_list);
1989         mutex_unlock(&cpufreq_governor_mutex);
1990         return;
1991 }
1992 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1993
1994
1995 /*********************************************************************
1996  *                          POLICY INTERFACE                         *
1997  *********************************************************************/
1998
1999 /**
2000  * cpufreq_get_policy - get the current cpufreq_policy
2001  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2002  *      is written
2003  *
2004  * Reads the current cpufreq policy.
2005  */
2006 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2007 {
2008         struct cpufreq_policy *cpu_policy;
2009         if (!policy)
2010                 return -EINVAL;
2011
2012         cpu_policy = cpufreq_cpu_get(cpu);
2013         if (!cpu_policy)
2014                 return -EINVAL;
2015
2016         memcpy(policy, cpu_policy, sizeof(*policy));
2017
2018         cpufreq_cpu_put(cpu_policy);
2019         return 0;
2020 }
2021 EXPORT_SYMBOL(cpufreq_get_policy);
2022
2023 /*
2024  * policy : current policy.
2025  * new_policy: policy to be set.
2026  */
2027 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2028                                 struct cpufreq_policy *new_policy)
2029 {
2030         int ret = 0, failed = 1;
2031
2032         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
2033                 new_policy->min, new_policy->max);
2034
2035         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2036
2037         if (new_policy->min > policy->max || new_policy->max < policy->min) {
2038                 ret = -EINVAL;
2039                 goto error_out;
2040         }
2041
2042         /* verify the cpu speed can be set within this limit */
2043         ret = cpufreq_driver->verify(new_policy);
2044         if (ret)
2045                 goto error_out;
2046
2047         /* adjust if necessary - all reasons */
2048         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2049                         CPUFREQ_ADJUST, new_policy);
2050
2051         /* adjust if necessary - hardware incompatibility*/
2052         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2053                         CPUFREQ_INCOMPATIBLE, new_policy);
2054
2055         /*
2056          * verify the cpu speed can be set within this limit, which might be
2057          * different to the first one
2058          */
2059         ret = cpufreq_driver->verify(new_policy);
2060         if (ret)
2061                 goto error_out;
2062
2063         /* notification of the new policy */
2064         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2065                         CPUFREQ_NOTIFY, new_policy);
2066
2067         policy->min = new_policy->min;
2068         policy->max = new_policy->max;
2069
2070         pr_debug("new min and max freqs are %u - %u kHz\n",
2071                                         policy->min, policy->max);
2072
2073         if (cpufreq_driver->setpolicy) {
2074                 policy->policy = new_policy->policy;
2075                 pr_debug("setting range\n");
2076                 ret = cpufreq_driver->setpolicy(new_policy);
2077         } else {
2078                 if (new_policy->governor != policy->governor) {
2079                         /* save old, working values */
2080                         struct cpufreq_governor *old_gov = policy->governor;
2081
2082                         pr_debug("governor switch\n");
2083
2084                         /* end old governor */
2085                         if (policy->governor) {
2086                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2087                                 up_write(&policy->rwsem);
2088                                 __cpufreq_governor(policy,
2089                                                 CPUFREQ_GOV_POLICY_EXIT);
2090                                 down_write(&policy->rwsem);
2091                         }
2092
2093                         /* start new governor */
2094                         policy->governor = new_policy->governor;
2095                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2096                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
2097                                         failed = 0;
2098                                 } else {
2099                                         up_write(&policy->rwsem);
2100                                         __cpufreq_governor(policy,
2101                                                         CPUFREQ_GOV_POLICY_EXIT);
2102                                         down_write(&policy->rwsem);
2103                                 }
2104                         }
2105
2106                         if (failed) {
2107                                 /* new governor failed, so re-start old one */
2108                                 pr_debug("starting governor %s failed\n",
2109                                                         policy->governor->name);
2110                                 if (old_gov) {
2111                                         policy->governor = old_gov;
2112                                         __cpufreq_governor(policy,
2113                                                         CPUFREQ_GOV_POLICY_INIT);
2114                                         __cpufreq_governor(policy,
2115                                                            CPUFREQ_GOV_START);
2116                                 }
2117                                 ret = -EINVAL;
2118                                 goto error_out;
2119                         }
2120                         /* might be a policy change, too, so fall through */
2121                 }
2122                 pr_debug("governor: change or update limits\n");
2123                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2124         }
2125
2126 error_out:
2127         return ret;
2128 }
2129
2130 /**
2131  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2132  *      @cpu: CPU which shall be re-evaluated
2133  *
2134  *      Useful for policy notifiers which have different necessities
2135  *      at different times.
2136  */
2137 int cpufreq_update_policy(unsigned int cpu)
2138 {
2139         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2140         struct cpufreq_policy new_policy;
2141         int ret;
2142
2143         if (!policy) {
2144                 ret = -ENODEV;
2145                 goto no_policy;
2146         }
2147
2148         down_write(&policy->rwsem);
2149
2150         pr_debug("updating policy for CPU %u\n", cpu);
2151         memcpy(&new_policy, policy, sizeof(*policy));
2152         new_policy.min = policy->user_policy.min;
2153         new_policy.max = policy->user_policy.max;
2154         new_policy.policy = policy->user_policy.policy;
2155         new_policy.governor = policy->user_policy.governor;
2156
2157         /*
2158          * BIOS might change freq behind our back
2159          * -> ask driver for current freq and notify governors about a change
2160          */
2161         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2162                 new_policy.cur = cpufreq_driver->get(cpu);
2163                 if (!policy->cur) {
2164                         pr_debug("Driver did not initialize current freq");
2165                         policy->cur = new_policy.cur;
2166                 } else {
2167                         if (policy->cur != new_policy.cur && has_target())
2168                                 cpufreq_out_of_sync(cpu, policy->cur,
2169                                                                 new_policy.cur);
2170                 }
2171         }
2172
2173         ret = cpufreq_set_policy(policy, &new_policy);
2174
2175         up_write(&policy->rwsem);
2176
2177         cpufreq_cpu_put(policy);
2178 no_policy:
2179         return ret;
2180 }
2181 EXPORT_SYMBOL(cpufreq_update_policy);
2182
2183 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2184                                         unsigned long action, void *hcpu)
2185 {
2186         unsigned int cpu = (unsigned long)hcpu;
2187         struct device *dev;
2188         bool frozen = false;
2189
2190         dev = get_cpu_device(cpu);
2191         if (dev) {
2192
2193                 if (action & CPU_TASKS_FROZEN)
2194                         frozen = true;
2195
2196                 switch (action & ~CPU_TASKS_FROZEN) {
2197                 case CPU_ONLINE:
2198                         __cpufreq_add_dev(dev, NULL, frozen);
2199                         cpufreq_update_policy(cpu);
2200                         break;
2201
2202                 case CPU_DOWN_PREPARE:
2203                         __cpufreq_remove_dev_prepare(dev, NULL, frozen);
2204                         break;
2205
2206                 case CPU_POST_DEAD:
2207                         __cpufreq_remove_dev_finish(dev, NULL, frozen);
2208                         break;
2209
2210                 case CPU_DOWN_FAILED:
2211                         __cpufreq_add_dev(dev, NULL, frozen);
2212                         break;
2213                 }
2214         }
2215         return NOTIFY_OK;
2216 }
2217
2218 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2219         .notifier_call = cpufreq_cpu_callback,
2220 };
2221
2222 /*********************************************************************
2223  *               BOOST                                               *
2224  *********************************************************************/
2225 static int cpufreq_boost_set_sw(int state)
2226 {
2227         struct cpufreq_frequency_table *freq_table;
2228         struct cpufreq_policy *policy;
2229         int ret = -EINVAL;
2230
2231         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2232                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2233                 if (freq_table) {
2234                         ret = cpufreq_frequency_table_cpuinfo(policy,
2235                                                         freq_table);
2236                         if (ret) {
2237                                 pr_err("%s: Policy frequency update failed\n",
2238                                        __func__);
2239                                 break;
2240                         }
2241                         policy->user_policy.max = policy->max;
2242                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2243                 }
2244         }
2245
2246         return ret;
2247 }
2248
2249 int cpufreq_boost_trigger_state(int state)
2250 {
2251         unsigned long flags;
2252         int ret = 0;
2253
2254         if (cpufreq_driver->boost_enabled == state)
2255                 return 0;
2256
2257         write_lock_irqsave(&cpufreq_driver_lock, flags);
2258         cpufreq_driver->boost_enabled = state;
2259         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2260
2261         ret = cpufreq_driver->set_boost(state);
2262         if (ret) {
2263                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2264                 cpufreq_driver->boost_enabled = !state;
2265                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2266
2267                 pr_err("%s: Cannot %s BOOST\n", __func__,
2268                        state ? "enable" : "disable");
2269         }
2270
2271         return ret;
2272 }
2273
2274 int cpufreq_boost_supported(void)
2275 {
2276         if (likely(cpufreq_driver))
2277                 return cpufreq_driver->boost_supported;
2278
2279         return 0;
2280 }
2281 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2282
2283 int cpufreq_boost_enabled(void)
2284 {
2285         return cpufreq_driver->boost_enabled;
2286 }
2287 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2288
2289 /*********************************************************************
2290  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2291  *********************************************************************/
2292
2293 /**
2294  * cpufreq_register_driver - register a CPU Frequency driver
2295  * @driver_data: A struct cpufreq_driver containing the values#
2296  * submitted by the CPU Frequency driver.
2297  *
2298  * Registers a CPU Frequency driver to this core code. This code
2299  * returns zero on success, -EBUSY when another driver got here first
2300  * (and isn't unregistered in the meantime).
2301  *
2302  */
2303 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2304 {
2305         unsigned long flags;
2306         int ret;
2307
2308         if (cpufreq_disabled())
2309                 return -ENODEV;
2310
2311         if (!driver_data || !driver_data->verify || !driver_data->init ||
2312             !(driver_data->setpolicy || driver_data->target_index ||
2313                     driver_data->target))
2314                 return -EINVAL;
2315
2316         pr_debug("trying to register driver %s\n", driver_data->name);
2317
2318         if (driver_data->setpolicy)
2319                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2320
2321         write_lock_irqsave(&cpufreq_driver_lock, flags);
2322         if (cpufreq_driver) {
2323                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2324                 return -EEXIST;
2325         }
2326         cpufreq_driver = driver_data;
2327         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2328
2329         if (cpufreq_boost_supported()) {
2330                 /*
2331                  * Check if driver provides function to enable boost -
2332                  * if not, use cpufreq_boost_set_sw as default
2333                  */
2334                 if (!cpufreq_driver->set_boost)
2335                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2336
2337                 ret = cpufreq_sysfs_create_file(&boost.attr);
2338                 if (ret) {
2339                         pr_err("%s: cannot register global BOOST sysfs file\n",
2340                                 __func__);
2341                         goto err_null_driver;
2342                 }
2343         }
2344
2345         ret = subsys_interface_register(&cpufreq_interface);
2346         if (ret)
2347                 goto err_boost_unreg;
2348
2349         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2350                 int i;
2351                 ret = -ENODEV;
2352
2353                 /* check for at least one working CPU */
2354                 for (i = 0; i < nr_cpu_ids; i++)
2355                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2356                                 ret = 0;
2357                                 break;
2358                         }
2359
2360                 /* if all ->init() calls failed, unregister */
2361                 if (ret) {
2362                         pr_debug("no CPU initialized for driver %s\n",
2363                                                         driver_data->name);
2364                         goto err_if_unreg;
2365                 }
2366         }
2367
2368         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2369         pr_debug("driver %s up and running\n", driver_data->name);
2370
2371         return 0;
2372 err_if_unreg:
2373         subsys_interface_unregister(&cpufreq_interface);
2374 err_boost_unreg:
2375         if (cpufreq_boost_supported())
2376                 cpufreq_sysfs_remove_file(&boost.attr);
2377 err_null_driver:
2378         write_lock_irqsave(&cpufreq_driver_lock, flags);
2379         cpufreq_driver = NULL;
2380         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2381         return ret;
2382 }
2383 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2384
2385 /**
2386  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2387  *
2388  * Unregister the current CPUFreq driver. Only call this if you have
2389  * the right to do so, i.e. if you have succeeded in initialising before!
2390  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2391  * currently not initialised.
2392  */
2393 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2394 {
2395         unsigned long flags;
2396
2397         if (!cpufreq_driver || (driver != cpufreq_driver))
2398                 return -EINVAL;
2399
2400         pr_debug("unregistering driver %s\n", driver->name);
2401
2402         subsys_interface_unregister(&cpufreq_interface);
2403         if (cpufreq_boost_supported())
2404                 cpufreq_sysfs_remove_file(&boost.attr);
2405
2406         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2407
2408         down_write(&cpufreq_rwsem);
2409         write_lock_irqsave(&cpufreq_driver_lock, flags);
2410
2411         cpufreq_driver = NULL;
2412
2413         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2414         up_write(&cpufreq_rwsem);
2415
2416         return 0;
2417 }
2418 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2419
2420 static int __init cpufreq_core_init(void)
2421 {
2422         if (cpufreq_disabled())
2423                 return -ENODEV;
2424
2425         cpufreq_global_kobject = kobject_create();
2426         BUG_ON(!cpufreq_global_kobject);
2427         register_syscore_ops(&cpufreq_syscore_ops);
2428
2429         return 0;
2430 }
2431 core_initcall(cpufreq_core_init);