drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 static inline bool has_target(void)
  51 {
  52         return cpufreq_driver->target_index || cpufreq_driver->target;
  53 }
  54
  55 /*
  56  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  57  * sections
  58  */
  59 static DECLARE_RWSEM(cpufreq_rwsem);
  60
  61 /* internal prototypes */
  62 static int __cpufreq_governor(struct cpufreq_policy *policy,
  63                 unsigned int event);
  64 static unsigned int __cpufreq_get(unsigned int cpu);
  65 static void handle_update(struct work_struct *work);
  66
  67 /**
  68  * Two notifier lists: the "policy" list is involved in the
  69  * validation process for a new CPU frequency policy; the
  70  * "transition" list for kernel code that needs to handle
  71  * changes to devices when the CPU clock speed changes.
  72  * The mutex locks both lists.
  73  */
  74 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  75 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  76
  77 static bool init_cpufreq_transition_notifier_list_called;
  78 static int __init init_cpufreq_transition_notifier_list(void)
  79 {
  80         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  81         init_cpufreq_transition_notifier_list_called = true;
  82         return 0;
  83 }
  84 pure_initcall(init_cpufreq_transition_notifier_list);
  85
  86 static int off __read_mostly;
  87 static int cpufreq_disabled(void)
  88 {
  89         return off;
  90 }
  91 void disable_cpufreq(void)
  92 {
  93         off = 1;
  94 }
  95 static LIST_HEAD(cpufreq_governor_list);
  96 static DEFINE_MUTEX(cpufreq_governor_mutex);
  97
  98 bool have_governor_per_policy(void)
  99 {
 100         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 101 }
 102 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 103
 104 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 105 {
 106         if (have_governor_per_policy())
 107                 return &policy->kobj;
 108         else
 109                 return cpufreq_global_kobject;
 110 }
 111 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 112
 113 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 114 {
 115         u64 idle_time;
 116         u64 cur_wall_time;
 117         u64 busy_time;
 118
 119         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 120
 121         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 122         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 127
 128         idle_time = cur_wall_time - busy_time;
 129         if (wall)
 130                 *wall = cputime_to_usecs(cur_wall_time);
 131
 132         return cputime_to_usecs(idle_time);
 133 }
 134
 135 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 136 {
 137         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 138
 139         if (idle_time == -1ULL)
 140                 return get_cpu_idle_time_jiffy(cpu, wall);
 141         else if (!io_busy)
 142                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 143
 144         return idle_time;
 145 }
 146 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 147
 148 /*
 149  * This is a generic cpufreq init() routine which can be used by cpufreq
 150  * drivers of SMP systems. It will do following:
 151  * - validate & show freq table passed
 152  * - set policies transition latency
 153  * - policy->cpus with all possible CPUs
 154  */
 155 int cpufreq_generic_init(struct cpufreq_policy *policy,
 156                 struct cpufreq_frequency_table *table,
 157                 unsigned int transition_latency)
 158 {
 159         int ret;
 160
 161         ret = cpufreq_table_validate_and_show(policy, table);
 162         if (ret) {
 163                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 164                 return ret;
 165         }
 166
 167         policy->cpuinfo.transition_latency = transition_latency;
 168
 169         /*
 170          * The driver only supports the SMP configuartion where all processors
 171          * share the clock and voltage and clock.
 172          */
 173         cpumask_setall(policy->cpus);
 174
 175         return 0;
 176 }
 177 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 178
 179 unsigned int cpufreq_generic_get(unsigned int cpu)
 180 {
 181         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 182
 183         if (!policy || IS_ERR(policy->clk)) {
 184                 pr_err("%s: No %s associated to cpu: %d\n", __func__,
 185                                 policy ? "clk" : "policy", cpu);
 186                 return 0;
 187         }
 188
 189         return clk_get_rate(policy->clk) / 1000;
 190 }
 191 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 192
 193 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 194 {
 195         struct cpufreq_policy *policy = NULL;
 196         unsigned long flags;
 197
 198         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 199                 return NULL;
 200
 201         if (!down_read_trylock(&cpufreq_rwsem))
 202                 return NULL;
 203
 204         /* get the cpufreq driver */
 205         read_lock_irqsave(&cpufreq_driver_lock, flags);
 206
 207         if (cpufreq_driver) {
 208                 /* get the CPU */
 209                 policy = per_cpu(cpufreq_cpu_data, cpu);
 210                 if (policy)
 211                         kobject_get(&policy->kobj);
 212         }
 213
 214         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 215
 216         if (!policy)
 217                 up_read(&cpufreq_rwsem);
 218
 219         return policy;
 220 }
 221 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 222
 223 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 224 {
 225         if (cpufreq_disabled())
 226                 return;
 227
 228         kobject_put(&policy->kobj);
 229         up_read(&cpufreq_rwsem);
 230 }
 231 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 232
 233 /*********************************************************************
 234  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 235  *********************************************************************/
 236
 237 /**
 238  * adjust_jiffies - adjust the system "loops_per_jiffy"
 239  *
 240  * This function alters the system "loops_per_jiffy" for the clock
 241  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 242  * systems as each CPU might be scaled differently. So, use the arch
 243  * per-CPU loops_per_jiffy value wherever possible.
 244  */
 245 #ifndef CONFIG_SMP
 246 static unsigned long l_p_j_ref;
 247 static unsigned int l_p_j_ref_freq;
 248
 249 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 250 {
 251         if (ci->flags & CPUFREQ_CONST_LOOPS)
 252                 return;
 253
 254         if (!l_p_j_ref_freq) {
 255                 l_p_j_ref = loops_per_jiffy;
 256                 l_p_j_ref_freq = ci->old;
 257                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 258                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 259         }
 260         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 261                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 262                                                                 ci->new);
 263                 pr_debug("scaling loops_per_jiffy to %lu "
 264                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 265         }
 266 }
 267 #else
 268 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 269 {
 270         return;
 271 }
 272 #endif
 273
 274 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 275                 struct cpufreq_freqs *freqs, unsigned int state)
 276 {
 277         BUG_ON(irqs_disabled());
 278
 279         if (cpufreq_disabled())
 280                 return;
 281
 282         freqs->flags = cpufreq_driver->flags;
 283         pr_debug("notification %u of frequency transition to %u kHz\n",
 284                 state, freqs->new);
 285
 286         switch (state) {
 287
 288         case CPUFREQ_PRECHANGE:
 289                 /* detect if the driver reported a value as "old frequency"
 290                  * which is not equal to what the cpufreq core thinks is
 291                  * "old frequency".
 292                  */
 293                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 294                         if ((policy) && (policy->cpu == freqs->cpu) &&
 295                             (policy->cur) && (policy->cur != freqs->old)) {
 296                                 pr_debug("Warning: CPU frequency is"
 297                                         " %u, cpufreq assumed %u kHz.\n",
 298                                         freqs->old, policy->cur);
 299                                 freqs->old = policy->cur;
 300                         }
 301                 }
 302                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 303                                 CPUFREQ_PRECHANGE, freqs);
 304                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 305                 break;
 306
 307         case CPUFREQ_POSTCHANGE:
 308                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 309                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 310                         (unsigned long)freqs->cpu);
 311                 trace_cpu_frequency(freqs->new, freqs->cpu);
 312                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 313                                 CPUFREQ_POSTCHANGE, freqs);
 314                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 315                         policy->cur = freqs->new;
 316                 break;
 317         }
 318 }
 319
 320 /**
 321  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 322  * on frequency transition.
 323  *
 324  * This function calls the transition notifiers and the "adjust_jiffies"
 325  * function. It is called twice on all CPU frequency changes that have
 326  * external effects.
 327  */
 328 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 329                 struct cpufreq_freqs *freqs, unsigned int state)
 330 {
 331         for_each_cpu(freqs->cpu, policy->cpus)
 332                 __cpufreq_notify_transition(policy, freqs, state);
 333 }
 334 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 335
 336 /* Do post notifications when there are chances that transition has failed */
 337 void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 338                 struct cpufreq_freqs *freqs, int transition_failed)
 339 {
 340         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 341         if (!transition_failed)
 342                 return;
 343
 344         swap(freqs->old, freqs->new);
 345         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 346         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 347 }
 348 EXPORT_SYMBOL_GPL(cpufreq_notify_post_transition);
 349
 350
 351 /*********************************************************************
 352  *                          SYSFS INTERFACE                          *
 353  *********************************************************************/
 354 ssize_t show_boost(struct kobject *kobj,
 355                                  struct attribute *attr, char *buf)
 356 {
 357         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 358 }
 359
 360 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 361                                   const char *buf, size_t count)
 362 {
 363         int ret, enable;
 364
 365         ret = sscanf(buf, "%d", &enable);
 366         if (ret != 1 || enable < 0 || enable > 1)
 367                 return -EINVAL;
 368
 369         if (cpufreq_boost_trigger_state(enable)) {
 370                 pr_err("%s: Cannot %s BOOST!\n", __func__,
 371                        enable ? "enable" : "disable");
 372                 return -EINVAL;
 373         }
 374
 375         pr_debug("%s: cpufreq BOOST %s\n", __func__,
 376                  enable ? "enabled" : "disabled");
 377
 378         return count;
 379 }
 380 define_one_global_rw(boost);
 381
 382 static struct cpufreq_governor *__find_governor(const char *str_governor)
 383 {
 384         struct cpufreq_governor *t;
 385
 386         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 387                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 388                         return t;
 389
 390         return NULL;
 391 }
 392
 393 /**
 394  * cpufreq_parse_governor - parse a governor string
 395  */
 396 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 397                                 struct cpufreq_governor **governor)
 398 {
 399         int err = -EINVAL;
 400
 401         if (!cpufreq_driver)
 402                 goto out;
 403
 404         if (cpufreq_driver->setpolicy) {
 405                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 406                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 407                         err = 0;
 408                 } else if (!strnicmp(str_governor, "powersave",
 409                                                 CPUFREQ_NAME_LEN)) {
 410                         *policy = CPUFREQ_POLICY_POWERSAVE;
 411                         err = 0;
 412                 }
 413         } else if (has_target()) {
 414                 struct cpufreq_governor *t;
 415
 416                 mutex_lock(&cpufreq_governor_mutex);
 417
 418                 t = __find_governor(str_governor);
 419
 420                 if (t == NULL) {
 421                         int ret;
 422
 423                         mutex_unlock(&cpufreq_governor_mutex);
 424                         ret = request_module("cpufreq_%s", str_governor);
 425                         mutex_lock(&cpufreq_governor_mutex);
 426
 427                         if (ret == 0)
 428                                 t = __find_governor(str_governor);
 429                 }
 430
 431                 if (t != NULL) {
 432                         *governor = t;
 433                         err = 0;
 434                 }
 435
 436                 mutex_unlock(&cpufreq_governor_mutex);
 437         }
 438 out:
 439         return err;
 440 }
 441
 442 /**
 443  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 444  * print out cpufreq information
 445  *
 446  * Write out information from cpufreq_driver->policy[cpu]; object must be
 447  * "unsigned int".
 448  */
 449
 450 #define show_one(file_name, object)                     \
 451 static ssize_t show_##file_name                         \
 452 (struct cpufreq_policy *policy, char *buf)              \
 453 {                                                       \
 454         return sprintf(buf, "%u\n", policy->object);    \
 455 }
 456
 457 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 458 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 459 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 460 show_one(scaling_min_freq, min);
 461 show_one(scaling_max_freq, max);
 462
 463 static ssize_t show_scaling_cur_freq(
 464         struct cpufreq_policy *policy, char *buf)
 465 {
 466         ssize_t ret;
 467
 468         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 469                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 470         else
 471                 ret = sprintf(buf, "%u\n", policy->cur);
 472         return ret;
 473 }
 474
 475 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 476                                 struct cpufreq_policy *new_policy);
 477
 478 /**
 479  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 480  */
 481 #define store_one(file_name, object)                    \
 482 static ssize_t store_##file_name                                        \
 483 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 484 {                                                                       \
 485         int ret;                                                        \
 486         struct cpufreq_policy new_policy;                               \
 487                                                                         \
 488         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 489         if (ret)                                                        \
 490                 return -EINVAL;                                         \
 491                                                                         \
 492         ret = sscanf(buf, "%u", &new_policy.object);                    \
 493         if (ret != 1)                                                   \
 494                 return -EINVAL;                                         \
 495                                                                         \
 496         ret = cpufreq_set_policy(policy, &new_policy);          \
 497         policy->user_policy.object = policy->object;                    \
 498                                                                         \
 499         return ret ? ret : count;                                       \
 500 }
 501
 502 store_one(scaling_min_freq, min);
 503 store_one(scaling_max_freq, max);
 504
 505 /**
 506  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 507  */
 508 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 509                                         char *buf)
 510 {
 511         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 512         if (!cur_freq)
 513                 return sprintf(buf, "<unknown>");
 514         return sprintf(buf, "%u\n", cur_freq);
 515 }
 516
 517 /**
 518  * show_scaling_governor - show the current policy for the specified CPU
 519  */
 520 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 521 {
 522         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 523                 return sprintf(buf, "powersave\n");
 524         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 525                 return sprintf(buf, "performance\n");
 526         else if (policy->governor)
 527                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 528                                 policy->governor->name);
 529         return -EINVAL;
 530 }
 531
 532 /**
 533  * store_scaling_governor - store policy for the specified CPU
 534  */
 535 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 536                                         const char *buf, size_t count)
 537 {
 538         int ret;
 539         char    str_governor[16];
 540         struct cpufreq_policy new_policy;
 541
 542         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 543         if (ret)
 544                 return ret;
 545
 546         ret = sscanf(buf, "%15s", str_governor);
 547         if (ret != 1)
 548                 return -EINVAL;
 549
 550         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 551                                                 &new_policy.governor))
 552                 return -EINVAL;
 553
 554         ret = cpufreq_set_policy(policy, &new_policy);
 555
 556         policy->user_policy.policy = policy->policy;
 557         policy->user_policy.governor = policy->governor;
 558
 559         if (ret)
 560                 return ret;
 561         else
 562                 return count;
 563 }
 564
 565 /**
 566  * show_scaling_driver - show the cpufreq driver currently loaded
 567  */
 568 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 569 {
 570         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 571 }
 572
 573 /**
 574  * show_scaling_available_governors - show the available CPUfreq governors
 575  */
 576 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 577                                                 char *buf)
 578 {
 579         ssize_t i = 0;
 580         struct cpufreq_governor *t;
 581
 582         if (!has_target()) {
 583                 i += sprintf(buf, "performance powersave");
 584                 goto out;
 585         }
 586
 587         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 588                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 589                     - (CPUFREQ_NAME_LEN + 2)))
 590                         goto out;
 591                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 592         }
 593 out:
 594         i += sprintf(&buf[i], "\n");
 595         return i;
 596 }
 597
 598 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 599 {
 600         ssize_t i = 0;
 601         unsigned int cpu;
 602
 603         for_each_cpu(cpu, mask) {
 604                 if (i)
 605                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 606                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 607                 if (i >= (PAGE_SIZE - 5))
 608                         break;
 609         }
 610         i += sprintf(&buf[i], "\n");
 611         return i;
 612 }
 613 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 614
 615 /**
 616  * show_related_cpus - show the CPUs affected by each transition even if
 617  * hw coordination is in use
 618  */
 619 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 620 {
 621         return cpufreq_show_cpus(policy->related_cpus, buf);
 622 }
 623
 624 /**
 625  * show_affected_cpus - show the CPUs affected by each transition
 626  */
 627 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 628 {
 629         return cpufreq_show_cpus(policy->cpus, buf);
 630 }
 631
 632 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 633                                         const char *buf, size_t count)
 634 {
 635         unsigned int freq = 0;
 636         unsigned int ret;
 637
 638         if (!policy->governor || !policy->governor->store_setspeed)
 639                 return -EINVAL;
 640
 641         ret = sscanf(buf, "%u", &freq);
 642         if (ret != 1)
 643                 return -EINVAL;
 644
 645         policy->governor->store_setspeed(policy, freq);
 646
 647         return count;
 648 }
 649
 650 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 651 {
 652         if (!policy->governor || !policy->governor->show_setspeed)
 653                 return sprintf(buf, "<unsupported>\n");
 654
 655         return policy->governor->show_setspeed(policy, buf);
 656 }
 657
 658 /**
 659  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 660  */
 661 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 662 {
 663         unsigned int limit;
 664         int ret;
 665         if (cpufreq_driver->bios_limit) {
 666                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 667                 if (!ret)
 668                         return sprintf(buf, "%u\n", limit);
 669         }
 670         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 671 }
 672
 673 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 674 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 675 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 676 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 677 cpufreq_freq_attr_ro(scaling_available_governors);
 678 cpufreq_freq_attr_ro(scaling_driver);
 679 cpufreq_freq_attr_ro(scaling_cur_freq);
 680 cpufreq_freq_attr_ro(bios_limit);
 681 cpufreq_freq_attr_ro(related_cpus);
 682 cpufreq_freq_attr_ro(affected_cpus);
 683 cpufreq_freq_attr_rw(scaling_min_freq);
 684 cpufreq_freq_attr_rw(scaling_max_freq);
 685 cpufreq_freq_attr_rw(scaling_governor);
 686 cpufreq_freq_attr_rw(scaling_setspeed);
 687
 688 static struct attribute *default_attrs[] = {
 689         &cpuinfo_min_freq.attr,
 690         &cpuinfo_max_freq.attr,
 691         &cpuinfo_transition_latency.attr,
 692         &scaling_min_freq.attr,
 693         &scaling_max_freq.attr,
 694         &affected_cpus.attr,
 695         &related_cpus.attr,
 696         &scaling_governor.attr,
 697         &scaling_driver.attr,
 698         &scaling_available_governors.attr,
 699         &scaling_setspeed.attr,
 700         NULL
 701 };
 702
 703 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 704 #define to_attr(a) container_of(a, struct freq_attr, attr)
 705
 706 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 707 {
 708         struct cpufreq_policy *policy = to_policy(kobj);
 709         struct freq_attr *fattr = to_attr(attr);
 710         ssize_t ret;
 711
 712         if (!down_read_trylock(&cpufreq_rwsem))
 713                 return -EINVAL;
 714
 715         down_read(&policy->rwsem);
 716
 717         if (fattr->show)
 718                 ret = fattr->show(policy, buf);
 719         else
 720                 ret = -EIO;
 721
 722         up_read(&policy->rwsem);
 723         up_read(&cpufreq_rwsem);
 724
 725         return ret;
 726 }
 727
 728 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 729                      const char *buf, size_t count)
 730 {
 731         struct cpufreq_policy *policy = to_policy(kobj);
 732         struct freq_attr *fattr = to_attr(attr);
 733         ssize_t ret = -EINVAL;
 734
 735         get_online_cpus();
 736
 737         if (!cpu_online(policy->cpu))
 738                 goto unlock;
 739
 740         if (!down_read_trylock(&cpufreq_rwsem))
 741                 goto unlock;
 742
 743         down_write(&policy->rwsem);
 744
 745         if (fattr->store)
 746                 ret = fattr->store(policy, buf, count);
 747         else
 748                 ret = -EIO;
 749
 750         up_write(&policy->rwsem);
 751
 752         up_read(&cpufreq_rwsem);
 753 unlock:
 754         put_online_cpus();
 755
 756         return ret;
 757 }
 758
 759 static void cpufreq_sysfs_release(struct kobject *kobj)
 760 {
 761         struct cpufreq_policy *policy = to_policy(kobj);
 762         pr_debug("last reference is dropped\n");
 763         complete(&policy->kobj_unregister);
 764 }
 765
 766 static const struct sysfs_ops sysfs_ops = {
 767         .show   = show,
 768         .store  = store,
 769 };
 770
 771 static struct kobj_type ktype_cpufreq = {
 772         .sysfs_ops      = &sysfs_ops,
 773         .default_attrs  = default_attrs,
 774         .release        = cpufreq_sysfs_release,
 775 };
 776
 777 struct kobject *cpufreq_global_kobject;
 778 EXPORT_SYMBOL(cpufreq_global_kobject);
 779
 780 static int cpufreq_global_kobject_usage;
 781
 782 int cpufreq_get_global_kobject(void)
 783 {
 784         if (!cpufreq_global_kobject_usage++)
 785                 return kobject_add(cpufreq_global_kobject,
 786                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 787
 788         return 0;
 789 }
 790 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 791
 792 void cpufreq_put_global_kobject(void)
 793 {
 794         if (!--cpufreq_global_kobject_usage)
 795                 kobject_del(cpufreq_global_kobject);
 796 }
 797 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 798
 799 int cpufreq_sysfs_create_file(const struct attribute *attr)
 800 {
 801         int ret = cpufreq_get_global_kobject();
 802
 803         if (!ret) {
 804                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 805                 if (ret)
 806                         cpufreq_put_global_kobject();
 807         }
 808
 809         return ret;
 810 }
 811 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 812
 813 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 814 {
 815         sysfs_remove_file(cpufreq_global_kobject, attr);
 816         cpufreq_put_global_kobject();
 817 }
 818 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 819
 820 /* symlink affected CPUs */
 821 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 822 {
 823         unsigned int j;
 824         int ret = 0;
 825
 826         for_each_cpu(j, policy->cpus) {
 827                 struct device *cpu_dev;
 828
 829                 if (j == policy->cpu)
 830                         continue;
 831
 832                 pr_debug("Adding link for CPU: %u\n", j);
 833                 cpu_dev = get_cpu_device(j);
 834                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 835                                         "cpufreq");
 836                 if (ret)
 837                         break;
 838         }
 839         return ret;
 840 }
 841
 842 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 843                                      struct device *dev)
 844 {
 845         struct freq_attr **drv_attr;
 846         int ret = 0;
 847
 848         /* prepare interface data */
 849         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 850                                    &dev->kobj, "cpufreq");
 851         if (ret)
 852                 return ret;
 853
 854         /* set up files for this cpu device */
 855         drv_attr = cpufreq_driver->attr;
 856         while ((drv_attr) && (*drv_attr)) {
 857                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 858                 if (ret)
 859                         goto err_out_kobj_put;
 860                 drv_attr++;
 861         }
 862         if (cpufreq_driver->get) {
 863                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 864                 if (ret)
 865                         goto err_out_kobj_put;
 866         }
 867
 868         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 869         if (ret)
 870                 goto err_out_kobj_put;
 871
 872         if (cpufreq_driver->bios_limit) {
 873                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 874                 if (ret)
 875                         goto err_out_kobj_put;
 876         }
 877
 878         ret = cpufreq_add_dev_symlink(policy);
 879         if (ret)
 880                 goto err_out_kobj_put;
 881
 882         return ret;
 883
 884 err_out_kobj_put:
 885         kobject_put(&policy->kobj);
 886         wait_for_completion(&policy->kobj_unregister);
 887         return ret;
 888 }
 889
 890 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 891 {
 892         struct cpufreq_policy new_policy;
 893         int ret = 0;
 894
 895         memcpy(&new_policy, policy, sizeof(*policy));
 896
 897         /* Use the default policy if its valid. */
 898         if (cpufreq_driver->setpolicy)
 899                 cpufreq_parse_governor(policy->governor->name,
 900                                         &new_policy.policy, NULL);
 901
 902         /* assure that the starting sequence is run in cpufreq_set_policy */
 903         policy->governor = NULL;
 904
 905         /* set default policy */
 906         ret = cpufreq_set_policy(policy, &new_policy);
 907         if (ret) {
 908                 pr_debug("setting policy failed\n");
 909                 if (cpufreq_driver->exit)
 910                         cpufreq_driver->exit(policy);
 911         }
 912 }
 913
 914 #ifdef CONFIG_HOTPLUG_CPU
 915 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 916                                   unsigned int cpu, struct device *dev)
 917 {
 918         int ret = 0;
 919         unsigned long flags;
 920
 921         if (has_target()) {
 922                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 923                 if (ret) {
 924                         pr_err("%s: Failed to stop governor\n", __func__);
 925                         return ret;
 926                 }
 927         }
 928
 929         down_write(&policy->rwsem);
 930
 931         write_lock_irqsave(&cpufreq_driver_lock, flags);
 932
 933         cpumask_set_cpu(cpu, policy->cpus);
 934         per_cpu(cpufreq_cpu_data, cpu) = policy;
 935         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 936
 937         up_write(&policy->rwsem);
 938
 939         if (has_target()) {
 940                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 941                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 942                         pr_err("%s: Failed to start governor\n", __func__);
 943                         return ret;
 944                 }
 945         }
 946
 947         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 948 }
 949 #endif
 950
 951 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 952 {
 953         struct cpufreq_policy *policy;
 954         unsigned long flags;
 955
 956         read_lock_irqsave(&cpufreq_driver_lock, flags);
 957
 958         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 959
 960         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 961
 962         return policy;
 963 }
 964
 965 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 966 {
 967         struct cpufreq_policy *policy;
 968
 969         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 970         if (!policy)
 971                 return NULL;
 972
 973         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 974                 goto err_free_policy;
 975
 976         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 977                 goto err_free_cpumask;
 978
 979         INIT_LIST_HEAD(&policy->policy_list);
 980         init_rwsem(&policy->rwsem);
 981
 982         return policy;
 983
 984 err_free_cpumask:
 985         free_cpumask_var(policy->cpus);
 986 err_free_policy:
 987         kfree(policy);
 988
 989         return NULL;
 990 }
 991
 992 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
 993 {
 994         struct kobject *kobj;
 995         struct completion *cmp;
 996
 997         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 998                         CPUFREQ_REMOVE_POLICY, policy);
 999
1000         down_read(&policy->rwsem);
1001         kobj = &policy->kobj;
1002         cmp = &policy->kobj_unregister;
1003         up_read(&policy->rwsem);
1004         kobject_put(kobj);
1005
1006         /*
1007          * We need to make sure that the underlying kobj is
1008          * actually not referenced anymore by anybody before we
1009          * proceed with unloading.
1010          */
1011         pr_debug("waiting for dropping of refcount\n");
1012         wait_for_completion(cmp);
1013         pr_debug("wait complete\n");
1014 }
1015
1016 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1017 {
1018         free_cpumask_var(policy->related_cpus);
1019         free_cpumask_var(policy->cpus);
1020         kfree(policy);
1021 }
1022
1023 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1024 {
1025         if (WARN_ON(cpu == policy->cpu))
1026                 return;
1027
1028         down_write(&policy->rwsem);
1029
1030         policy->last_cpu = policy->cpu;
1031         policy->cpu = cpu;
1032
1033         up_write(&policy->rwsem);
1034
1035         cpufreq_frequency_table_update_policy_cpu(policy);
1036         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1037                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1038 }
1039
1040 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
1041                              bool frozen)
1042 {
1043         unsigned int j, cpu = dev->id;
1044         int ret = -ENOMEM;
1045         struct cpufreq_policy *policy;
1046         unsigned long flags;
1047 #ifdef CONFIG_HOTPLUG_CPU
1048         struct cpufreq_policy *tpolicy;
1049         struct cpufreq_governor *gov;
1050 #endif
1051
1052         if (cpu_is_offline(cpu))
1053                 return 0;
1054
1055         pr_debug("adding CPU %u\n", cpu);
1056
1057 #ifdef CONFIG_SMP
1058         /* check whether a different CPU already registered this
1059          * CPU because it is in the same boat. */
1060         policy = cpufreq_cpu_get(cpu);
1061         if (unlikely(policy)) {
1062                 cpufreq_cpu_put(policy);
1063                 return 0;
1064         }
1065 #endif
1066
1067         if (!down_read_trylock(&cpufreq_rwsem))
1068                 return 0;
1069
1070 #ifdef CONFIG_HOTPLUG_CPU
1071         /* Check if this cpu was hot-unplugged earlier and has siblings */
1072         read_lock_irqsave(&cpufreq_driver_lock, flags);
1073         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1074                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1075                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1076                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1077                         up_read(&cpufreq_rwsem);
1078                         return ret;
1079                 }
1080         }
1081         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1082 #endif
1083
1084         /*
1085          * Restore the saved policy when doing light-weight init and fall back
1086          * to the full init if that fails.
1087          */
1088         policy = frozen ? cpufreq_policy_restore(cpu) : NULL;
1089         if (!policy) {
1090                 frozen = false;
1091                 policy = cpufreq_policy_alloc();
1092                 if (!policy)
1093                         goto nomem_out;
1094         }
1095
1096         /*
1097          * In the resume path, since we restore a saved policy, the assignment
1098          * to policy->cpu is like an update of the existing policy, rather than
1099          * the creation of a brand new one. So we need to perform this update
1100          * by invoking update_policy_cpu().
1101          */
1102         if (frozen && cpu != policy->cpu) {
1103                 update_policy_cpu(policy, cpu);
1104                 WARN_ON(kobject_move(&policy->kobj, &dev->kobj));
1105         } else {
1106                 policy->cpu = cpu;
1107         }
1108
1109         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1110         cpumask_copy(policy->cpus, cpumask_of(cpu));
1111
1112         init_completion(&policy->kobj_unregister);
1113         INIT_WORK(&policy->update, handle_update);
1114
1115         /* call driver. From then on the cpufreq must be able
1116          * to accept all calls to ->verify and ->setpolicy for this CPU
1117          */
1118         ret = cpufreq_driver->init(policy);
1119         if (ret) {
1120                 pr_debug("initialization failed\n");
1121                 goto err_set_policy_cpu;
1122         }
1123
1124         /* related cpus should atleast have policy->cpus */
1125         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1126
1127         /*
1128          * affected cpus must always be the one, which are online. We aren't
1129          * managing offline cpus here.
1130          */
1131         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1132
1133         if (!frozen) {
1134                 policy->user_policy.min = policy->min;
1135                 policy->user_policy.max = policy->max;
1136         }
1137
1138         down_write(&policy->rwsem);
1139         write_lock_irqsave(&cpufreq_driver_lock, flags);
1140         for_each_cpu(j, policy->cpus)
1141                 per_cpu(cpufreq_cpu_data, j) = policy;
1142         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1143
1144         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1145                 policy->cur = cpufreq_driver->get(policy->cpu);
1146                 if (!policy->cur) {
1147                         pr_err("%s: ->get() failed\n", __func__);
1148                         goto err_get_freq;
1149                 }
1150         }
1151
1152         /*
1153          * Sometimes boot loaders set CPU frequency to a value outside of
1154          * frequency table present with cpufreq core. In such cases CPU might be
1155          * unstable if it has to run on that frequency for long duration of time
1156          * and so its better to set it to a frequency which is specified in
1157          * freq-table. This also makes cpufreq stats inconsistent as
1158          * cpufreq-stats would fail to register because current frequency of CPU
1159          * isn't found in freq-table.
1160          *
1161          * Because we don't want this change to effect boot process badly, we go
1162          * for the next freq which is >= policy->cur ('cur' must be set by now,
1163          * otherwise we will end up setting freq to lowest of the table as 'cur'
1164          * is initialized to zero).
1165          *
1166          * We are passing target-freq as "policy->cur - 1" otherwise
1167          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1168          * equal to target-freq.
1169          */
1170         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1171             && has_target()) {
1172                 /* Are we running at unknown frequency ? */
1173                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1174                 if (ret == -EINVAL) {
1175                         /* Warn user and fix it */
1176                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1177                                 __func__, policy->cpu, policy->cur);
1178                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1179                                 CPUFREQ_RELATION_L);
1180
1181                         /*
1182                          * Reaching here after boot in a few seconds may not
1183                          * mean that system will remain stable at "unknown"
1184                          * frequency for longer duration. Hence, a BUG_ON().
1185                          */
1186                         BUG_ON(ret);
1187                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1188                                 __func__, policy->cpu, policy->cur);
1189                 }
1190         }
1191
1192         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1193                                      CPUFREQ_START, policy);
1194
1195 #ifdef CONFIG_HOTPLUG_CPU
1196         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1197         if (gov) {
1198                 policy->governor = gov;
1199                 pr_debug("Restoring governor %s for cpu %d\n",
1200                        policy->governor->name, cpu);
1201         }
1202 #endif
1203
1204         if (!frozen) {
1205                 ret = cpufreq_add_dev_interface(policy, dev);
1206                 if (ret)
1207                         goto err_out_unregister;
1208                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1209                                 CPUFREQ_CREATE_POLICY, policy);
1210         }
1211
1212         write_lock_irqsave(&cpufreq_driver_lock, flags);
1213         list_add(&policy->policy_list, &cpufreq_policy_list);
1214         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1215
1216         cpufreq_init_policy(policy);
1217
1218         if (!frozen) {
1219                 policy->user_policy.policy = policy->policy;
1220                 policy->user_policy.governor = policy->governor;
1221         }
1222         up_write(&policy->rwsem);
1223
1224         kobject_uevent(&policy->kobj, KOBJ_ADD);
1225         up_read(&cpufreq_rwsem);
1226
1227         pr_debug("initialization complete\n");
1228
1229         return 0;
1230
1231 err_out_unregister:
1232 err_get_freq:
1233         write_lock_irqsave(&cpufreq_driver_lock, flags);
1234         for_each_cpu(j, policy->cpus)
1235                 per_cpu(cpufreq_cpu_data, j) = NULL;
1236         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1237
1238         up_write(&policy->rwsem);
1239
1240         if (cpufreq_driver->exit)
1241                 cpufreq_driver->exit(policy);
1242 err_set_policy_cpu:
1243         if (frozen) {
1244                 /* Do not leave stale fallback data behind. */
1245                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1246                 cpufreq_policy_put_kobj(policy);
1247         }
1248         cpufreq_policy_free(policy);
1249
1250 nomem_out:
1251         up_read(&cpufreq_rwsem);
1252
1253         return ret;
1254 }
1255
1256 /**
1257  * cpufreq_add_dev - add a CPU device
1258  *
1259  * Adds the cpufreq interface for a CPU device.
1260  *
1261  * The Oracle says: try running cpufreq registration/unregistration concurrently
1262  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1263  * mess up, but more thorough testing is needed. - Mathieu
1264  */
1265 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1266 {
1267         return __cpufreq_add_dev(dev, sif, false);
1268 }
1269
1270 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1271                                            unsigned int old_cpu)
1272 {
1273         struct device *cpu_dev;
1274         int ret;
1275
1276         /* first sibling now owns the new sysfs dir */
1277         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1278
1279         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1280         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1281         if (ret) {
1282                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1283
1284                 down_write(&policy->rwsem);
1285                 cpumask_set_cpu(old_cpu, policy->cpus);
1286                 up_write(&policy->rwsem);
1287
1288                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1289                                         "cpufreq");
1290
1291                 return -EINVAL;
1292         }
1293
1294         return cpu_dev->id;
1295 }
1296
1297 static int __cpufreq_remove_dev_prepare(struct device *dev,
1298                                         struct subsys_interface *sif,
1299                                         bool frozen)
1300 {
1301         unsigned int cpu = dev->id, cpus;
1302         int new_cpu, ret;
1303         unsigned long flags;
1304         struct cpufreq_policy *policy;
1305
1306         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1307
1308         write_lock_irqsave(&cpufreq_driver_lock, flags);
1309
1310         policy = per_cpu(cpufreq_cpu_data, cpu);
1311
1312         /* Save the policy somewhere when doing a light-weight tear-down */
1313         if (frozen)
1314                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1315
1316         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1317
1318         if (!policy) {
1319                 pr_debug("%s: No cpu_data found\n", __func__);
1320                 return -EINVAL;
1321         }
1322
1323         if (has_target()) {
1324                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1325                 if (ret) {
1326                         pr_err("%s: Failed to stop governor\n", __func__);
1327                         return ret;
1328                 }
1329         }
1330
1331 #ifdef CONFIG_HOTPLUG_CPU
1332         if (!cpufreq_driver->setpolicy)
1333                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1334                         policy->governor->name, CPUFREQ_NAME_LEN);
1335 #endif
1336
1337         down_read(&policy->rwsem);
1338         cpus = cpumask_weight(policy->cpus);
1339         up_read(&policy->rwsem);
1340
1341         if (cpu != policy->cpu) {
1342                 sysfs_remove_link(&dev->kobj, "cpufreq");
1343         } else if (cpus > 1) {
1344                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1345                 if (new_cpu >= 0) {
1346                         update_policy_cpu(policy, new_cpu);
1347
1348                         if (!frozen) {
1349                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1350                                                 __func__, new_cpu, cpu);
1351                         }
1352                 }
1353         }
1354
1355         return 0;
1356 }
1357
1358 static int __cpufreq_remove_dev_finish(struct device *dev,
1359                                        struct subsys_interface *sif,
1360                                        bool frozen)
1361 {
1362         unsigned int cpu = dev->id, cpus;
1363         int ret;
1364         unsigned long flags;
1365         struct cpufreq_policy *policy;
1366
1367         read_lock_irqsave(&cpufreq_driver_lock, flags);
1368         policy = per_cpu(cpufreq_cpu_data, cpu);
1369         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1370
1371         if (!policy) {
1372                 pr_debug("%s: No cpu_data found\n", __func__);
1373                 return -EINVAL;
1374         }
1375
1376         down_write(&policy->rwsem);
1377         cpus = cpumask_weight(policy->cpus);
1378
1379         if (cpus > 1)
1380                 cpumask_clear_cpu(cpu, policy->cpus);
1381         up_write(&policy->rwsem);
1382
1383         /* If cpu is last user of policy, free policy */
1384         if (cpus == 1) {
1385                 if (has_target()) {
1386                         ret = __cpufreq_governor(policy,
1387                                         CPUFREQ_GOV_POLICY_EXIT);
1388                         if (ret) {
1389                                 pr_err("%s: Failed to exit governor\n",
1390                                                 __func__);
1391                                 return ret;
1392                         }
1393                 }
1394
1395                 if (!frozen)
1396                         cpufreq_policy_put_kobj(policy);
1397
1398                 /*
1399                  * Perform the ->exit() even during light-weight tear-down,
1400                  * since this is a core component, and is essential for the
1401                  * subsequent light-weight ->init() to succeed.
1402                  */
1403                 if (cpufreq_driver->exit)
1404                         cpufreq_driver->exit(policy);
1405
1406                 /* Remove policy from list of active policies */
1407                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1408                 list_del(&policy->policy_list);
1409                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1410
1411                 if (!frozen)
1412                         cpufreq_policy_free(policy);
1413         } else {
1414                 if (has_target()) {
1415                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1416                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1417                                 pr_err("%s: Failed to start governor\n",
1418                                                 __func__);
1419                                 return ret;
1420                         }
1421                 }
1422         }
1423
1424         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1425         return 0;
1426 }
1427
1428 /**
1429  * cpufreq_remove_dev - remove a CPU device
1430  *
1431  * Removes the cpufreq interface for a CPU device.
1432  */
1433 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1434 {
1435         unsigned int cpu = dev->id;
1436         int ret;
1437
1438         if (cpu_is_offline(cpu))
1439                 return 0;
1440
1441         ret = __cpufreq_remove_dev_prepare(dev, sif, false);
1442
1443         if (!ret)
1444                 ret = __cpufreq_remove_dev_finish(dev, sif, false);
1445
1446         return ret;
1447 }
1448
1449 static void handle_update(struct work_struct *work)
1450 {
1451         struct cpufreq_policy *policy =
1452                 container_of(work, struct cpufreq_policy, update);
1453         unsigned int cpu = policy->cpu;
1454         pr_debug("handle_update for cpu %u called\n", cpu);
1455         cpufreq_update_policy(cpu);
1456 }
1457
1458 /**
1459  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1460  *      in deep trouble.
1461  *      @cpu: cpu number
1462  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1463  *      @new_freq: CPU frequency the CPU actually runs at
1464  *
1465  *      We adjust to current frequency first, and need to clean up later.
1466  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1467  */
1468 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1469                                 unsigned int new_freq)
1470 {
1471         struct cpufreq_policy *policy;
1472         struct cpufreq_freqs freqs;
1473         unsigned long flags;
1474
1475         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1476                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1477
1478         freqs.old = old_freq;
1479         freqs.new = new_freq;
1480
1481         read_lock_irqsave(&cpufreq_driver_lock, flags);
1482         policy = per_cpu(cpufreq_cpu_data, cpu);
1483         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1484
1485         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1486         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1487 }
1488
1489 /**
1490  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1491  * @cpu: CPU number
1492  *
1493  * This is the last known freq, without actually getting it from the driver.
1494  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1495  */
1496 unsigned int cpufreq_quick_get(unsigned int cpu)
1497 {
1498         struct cpufreq_policy *policy;
1499         unsigned int ret_freq = 0;
1500
1501         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1502                 return cpufreq_driver->get(cpu);
1503
1504         policy = cpufreq_cpu_get(cpu);
1505         if (policy) {
1506                 ret_freq = policy->cur;
1507                 cpufreq_cpu_put(policy);
1508         }
1509
1510         return ret_freq;
1511 }
1512 EXPORT_SYMBOL(cpufreq_quick_get);
1513
1514 /**
1515  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1516  * @cpu: CPU number
1517  *
1518  * Just return the max possible frequency for a given CPU.
1519  */
1520 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1521 {
1522         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1523         unsigned int ret_freq = 0;
1524
1525         if (policy) {
1526                 ret_freq = policy->max;
1527                 cpufreq_cpu_put(policy);
1528         }
1529
1530         return ret_freq;
1531 }
1532 EXPORT_SYMBOL(cpufreq_quick_get_max);
1533
1534 static unsigned int __cpufreq_get(unsigned int cpu)
1535 {
1536         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1537         unsigned int ret_freq = 0;
1538
1539         if (!cpufreq_driver->get)
1540                 return ret_freq;
1541
1542         ret_freq = cpufreq_driver->get(cpu);
1543
1544         if (ret_freq && policy->cur &&
1545                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1546                 /* verify no discrepancy between actual and
1547                                         saved value exists */
1548                 if (unlikely(ret_freq != policy->cur)) {
1549                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1550                         schedule_work(&policy->update);
1551                 }
1552         }
1553
1554         return ret_freq;
1555 }
1556
1557 /**
1558  * cpufreq_get - get the current CPU frequency (in kHz)
1559  * @cpu: CPU number
1560  *
1561  * Get the CPU current (static) CPU frequency
1562  */
1563 unsigned int cpufreq_get(unsigned int cpu)
1564 {
1565         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1566         unsigned int ret_freq = 0;
1567
1568         if (policy) {
1569                 down_read(&policy->rwsem);
1570                 ret_freq = __cpufreq_get(cpu);
1571                 up_read(&policy->rwsem);
1572
1573                 cpufreq_cpu_put(policy);
1574         }
1575
1576         return ret_freq;
1577 }
1578 EXPORT_SYMBOL(cpufreq_get);
1579
1580 static struct subsys_interface cpufreq_interface = {
1581         .name           = "cpufreq",
1582         .subsys         = &cpu_subsys,
1583         .add_dev        = cpufreq_add_dev,
1584         .remove_dev     = cpufreq_remove_dev,
1585 };
1586
1587 /**
1588  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1589  *
1590  * This function is only executed for the boot processor.  The other CPUs
1591  * have been put offline by means of CPU hotplug.
1592  */
1593 static int cpufreq_bp_suspend(void)
1594 {
1595         int ret = 0;
1596
1597         int cpu = smp_processor_id();
1598         struct cpufreq_policy *policy;
1599
1600         pr_debug("suspending cpu %u\n", cpu);
1601
1602         /* If there's no policy for the boot CPU, we have nothing to do. */
1603         policy = cpufreq_cpu_get(cpu);
1604         if (!policy)
1605                 return 0;
1606
1607         if (cpufreq_driver->suspend) {
1608                 ret = cpufreq_driver->suspend(policy);
1609                 if (ret)
1610                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1611                                         "step on CPU %u\n", policy->cpu);
1612         }
1613
1614         cpufreq_cpu_put(policy);
1615         return ret;
1616 }
1617
1618 /**
1619  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1620  *
1621  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1622  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1623  *          restored. It will verify that the current freq is in sync with
1624  *          what we believe it to be. This is a bit later than when it
1625  *          should be, but nonethteless it's better than calling
1626  *          cpufreq_driver->get() here which might re-enable interrupts...
1627  *
1628  * This function is only executed for the boot CPU.  The other CPUs have not
1629  * been turned on yet.
1630  */
1631 static void cpufreq_bp_resume(void)
1632 {
1633         int ret = 0;
1634
1635         int cpu = smp_processor_id();
1636         struct cpufreq_policy *policy;
1637
1638         pr_debug("resuming cpu %u\n", cpu);
1639
1640         /* If there's no policy for the boot CPU, we have nothing to do. */
1641         policy = cpufreq_cpu_get(cpu);
1642         if (!policy)
1643                 return;
1644
1645         if (cpufreq_driver->resume) {
1646                 ret = cpufreq_driver->resume(policy);
1647                 if (ret) {
1648                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1649                                         "step on CPU %u\n", policy->cpu);
1650                         goto fail;
1651                 }
1652         }
1653
1654         schedule_work(&policy->update);
1655
1656 fail:
1657         cpufreq_cpu_put(policy);
1658 }
1659
1660 static struct syscore_ops cpufreq_syscore_ops = {
1661         .suspend        = cpufreq_bp_suspend,
1662         .resume         = cpufreq_bp_resume,
1663 };
1664
1665 /**
1666  *      cpufreq_get_current_driver - return current driver's name
1667  *
1668  *      Return the name string of the currently loaded cpufreq driver
1669  *      or NULL, if none.
1670  */
1671 const char *cpufreq_get_current_driver(void)
1672 {
1673         if (cpufreq_driver)
1674                 return cpufreq_driver->name;
1675
1676         return NULL;
1677 }
1678 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1679
1680 /*********************************************************************
1681  *                     NOTIFIER LISTS INTERFACE                      *
1682  *********************************************************************/
1683
1684 /**
1685  *      cpufreq_register_notifier - register a driver with cpufreq
1686  *      @nb: notifier function to register
1687  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1688  *
1689  *      Add a driver to one of two lists: either a list of drivers that
1690  *      are notified about clock rate changes (once before and once after
1691  *      the transition), or a list of drivers that are notified about
1692  *      changes in cpufreq policy.
1693  *
1694  *      This function may sleep, and has the same return conditions as
1695  *      blocking_notifier_chain_register.
1696  */
1697 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1698 {
1699         int ret;
1700
1701         if (cpufreq_disabled())
1702                 return -EINVAL;
1703
1704         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1705
1706         switch (list) {
1707         case CPUFREQ_TRANSITION_NOTIFIER:
1708                 ret = srcu_notifier_chain_register(
1709                                 &cpufreq_transition_notifier_list, nb);
1710                 break;
1711         case CPUFREQ_POLICY_NOTIFIER:
1712                 ret = blocking_notifier_chain_register(
1713                                 &cpufreq_policy_notifier_list, nb);
1714                 break;
1715         default:
1716                 ret = -EINVAL;
1717         }
1718
1719         return ret;
1720 }
1721 EXPORT_SYMBOL(cpufreq_register_notifier);
1722
1723 /**
1724  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1725  *      @nb: notifier block to be unregistered
1726  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1727  *
1728  *      Remove a driver from the CPU frequency notifier list.
1729  *
1730  *      This function may sleep, and has the same return conditions as
1731  *      blocking_notifier_chain_unregister.
1732  */
1733 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1734 {
1735         int ret;
1736
1737         if (cpufreq_disabled())
1738                 return -EINVAL;
1739
1740         switch (list) {
1741         case CPUFREQ_TRANSITION_NOTIFIER:
1742                 ret = srcu_notifier_chain_unregister(
1743                                 &cpufreq_transition_notifier_list, nb);
1744                 break;
1745         case CPUFREQ_POLICY_NOTIFIER:
1746                 ret = blocking_notifier_chain_unregister(
1747                                 &cpufreq_policy_notifier_list, nb);
1748                 break;
1749         default:
1750                 ret = -EINVAL;
1751         }
1752
1753         return ret;
1754 }
1755 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1756
1757
1758 /*********************************************************************
1759  *                              GOVERNORS                            *
1760  *********************************************************************/
1761
1762 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1763                             unsigned int target_freq,
1764                             unsigned int relation)
1765 {
1766         int retval = -EINVAL;
1767         unsigned int old_target_freq = target_freq;
1768
1769         if (cpufreq_disabled())
1770                 return -ENODEV;
1771
1772         /* Make sure that target_freq is within supported range */
1773         if (target_freq > policy->max)
1774                 target_freq = policy->max;
1775         if (target_freq < policy->min)
1776                 target_freq = policy->min;
1777
1778         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1779                         policy->cpu, target_freq, relation, old_target_freq);
1780
1781         /*
1782          * This might look like a redundant call as we are checking it again
1783          * after finding index. But it is left intentionally for cases where
1784          * exactly same freq is called again and so we can save on few function
1785          * calls.
1786          */
1787         if (target_freq == policy->cur)
1788                 return 0;
1789
1790         if (cpufreq_driver->target)
1791                 retval = cpufreq_driver->target(policy, target_freq, relation);
1792         else if (cpufreq_driver->target_index) {
1793                 struct cpufreq_frequency_table *freq_table;
1794                 struct cpufreq_freqs freqs;
1795                 bool notify;
1796                 int index;
1797
1798                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1799                 if (unlikely(!freq_table)) {
1800                         pr_err("%s: Unable to find freq_table\n", __func__);
1801                         goto out;
1802                 }
1803
1804                 retval = cpufreq_frequency_table_target(policy, freq_table,
1805                                 target_freq, relation, &index);
1806                 if (unlikely(retval)) {
1807                         pr_err("%s: Unable to find matching freq\n", __func__);
1808                         goto out;
1809                 }
1810
1811                 if (freq_table[index].frequency == policy->cur) {
1812                         retval = 0;
1813                         goto out;
1814                 }
1815
1816                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1817
1818                 if (notify) {
1819                         freqs.old = policy->cur;
1820                         freqs.new = freq_table[index].frequency;
1821                         freqs.flags = 0;
1822
1823                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1824                                         __func__, policy->cpu, freqs.old,
1825                                         freqs.new);
1826
1827                         cpufreq_notify_transition(policy, &freqs,
1828                                         CPUFREQ_PRECHANGE);
1829                 }
1830
1831                 retval = cpufreq_driver->target_index(policy, index);
1832                 if (retval)
1833                         pr_err("%s: Failed to change cpu frequency: %d\n",
1834                                         __func__, retval);
1835
1836                 if (notify)
1837                         cpufreq_notify_post_transition(policy, &freqs, retval);
1838         }
1839
1840 out:
1841         return retval;
1842 }
1843 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1844
1845 int cpufreq_driver_target(struct cpufreq_policy *policy,
1846                           unsigned int target_freq,
1847                           unsigned int relation)
1848 {
1849         int ret = -EINVAL;
1850
1851         down_write(&policy->rwsem);
1852
1853         ret = __cpufreq_driver_target(policy, target_freq, relation);
1854
1855         up_write(&policy->rwsem);
1856
1857         return ret;
1858 }
1859 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1860
1861 /*
1862  * when "event" is CPUFREQ_GOV_LIMITS
1863  */
1864
1865 static int __cpufreq_governor(struct cpufreq_policy *policy,
1866                                         unsigned int event)
1867 {
1868         int ret;
1869
1870         /* Only must be defined when default governor is known to have latency
1871            restrictions, like e.g. conservative or ondemand.
1872            That this is the case is already ensured in Kconfig
1873         */
1874 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1875         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1876 #else
1877         struct cpufreq_governor *gov = NULL;
1878 #endif
1879
1880         if (policy->governor->max_transition_latency &&
1881             policy->cpuinfo.transition_latency >
1882             policy->governor->max_transition_latency) {
1883                 if (!gov)
1884                         return -EINVAL;
1885                 else {
1886                         printk(KERN_WARNING "%s governor failed, too long"
1887                                " transition latency of HW, fallback"
1888                                " to %s governor\n",
1889                                policy->governor->name,
1890                                gov->name);
1891                         policy->governor = gov;
1892                 }
1893         }
1894
1895         if (event == CPUFREQ_GOV_POLICY_INIT)
1896                 if (!try_module_get(policy->governor->owner))
1897                         return -EINVAL;
1898
1899         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1900                                                 policy->cpu, event);
1901
1902         mutex_lock(&cpufreq_governor_lock);
1903         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1904             || (!policy->governor_enabled
1905             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1906                 mutex_unlock(&cpufreq_governor_lock);
1907                 return -EBUSY;
1908         }
1909
1910         if (event == CPUFREQ_GOV_STOP)
1911                 policy->governor_enabled = false;
1912         else if (event == CPUFREQ_GOV_START)
1913                 policy->governor_enabled = true;
1914
1915         mutex_unlock(&cpufreq_governor_lock);
1916
1917         ret = policy->governor->governor(policy, event);
1918
1919         if (!ret) {
1920                 if (event == CPUFREQ_GOV_POLICY_INIT)
1921                         policy->governor->initialized++;
1922                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1923                         policy->governor->initialized--;
1924         } else {
1925                 /* Restore original values */
1926                 mutex_lock(&cpufreq_governor_lock);
1927                 if (event == CPUFREQ_GOV_STOP)
1928                         policy->governor_enabled = true;
1929                 else if (event == CPUFREQ_GOV_START)
1930                         policy->governor_enabled = false;
1931                 mutex_unlock(&cpufreq_governor_lock);
1932         }
1933
1934         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1935                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1936                 module_put(policy->governor->owner);
1937
1938         return ret;
1939 }
1940
1941 int cpufreq_register_governor(struct cpufreq_governor *governor)
1942 {
1943         int err;
1944
1945         if (!governor)
1946                 return -EINVAL;
1947
1948         if (cpufreq_disabled())
1949                 return -ENODEV;
1950
1951         mutex_lock(&cpufreq_governor_mutex);
1952
1953         governor->initialized = 0;
1954         err = -EBUSY;
1955         if (__find_governor(governor->name) == NULL) {
1956                 err = 0;
1957                 list_add(&governor->governor_list, &cpufreq_governor_list);
1958         }
1959
1960         mutex_unlock(&cpufreq_governor_mutex);
1961         return err;
1962 }
1963 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1964
1965 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1966 {
1967 #ifdef CONFIG_HOTPLUG_CPU
1968         int cpu;
1969 #endif
1970
1971         if (!governor)
1972                 return;
1973
1974         if (cpufreq_disabled())
1975                 return;
1976
1977 #ifdef CONFIG_HOTPLUG_CPU
1978         for_each_present_cpu(cpu) {
1979                 if (cpu_online(cpu))
1980                         continue;
1981                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1982                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1983         }
1984 #endif
1985
1986         mutex_lock(&cpufreq_governor_mutex);
1987         list_del(&governor->governor_list);
1988         mutex_unlock(&cpufreq_governor_mutex);
1989         return;
1990 }
1991 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1992
1993
1994 /*********************************************************************
1995  *                          POLICY INTERFACE                         *
1996  *********************************************************************/
1997
1998 /**
1999  * cpufreq_get_policy - get the current cpufreq_policy
2000  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2001  *      is written
2002  *
2003  * Reads the current cpufreq policy.
2004  */
2005 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2006 {
2007         struct cpufreq_policy *cpu_policy;
2008         if (!policy)
2009                 return -EINVAL;
2010
2011         cpu_policy = cpufreq_cpu_get(cpu);
2012         if (!cpu_policy)
2013                 return -EINVAL;
2014
2015         memcpy(policy, cpu_policy, sizeof(*policy));
2016
2017         cpufreq_cpu_put(cpu_policy);
2018         return 0;
2019 }
2020 EXPORT_SYMBOL(cpufreq_get_policy);
2021
2022 /*
2023  * policy : current policy.
2024  * new_policy: policy to be set.
2025  */
2026 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2027                                 struct cpufreq_policy *new_policy)
2028 {
2029         int ret = 0, failed = 1;
2030
2031         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
2032                 new_policy->min, new_policy->max);
2033
2034         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2035
2036         if (new_policy->min > policy->max || new_policy->max < policy->min) {
2037                 ret = -EINVAL;
2038                 goto error_out;
2039         }
2040
2041         /* verify the cpu speed can be set within this limit */
2042         ret = cpufreq_driver->verify(new_policy);
2043         if (ret)
2044                 goto error_out;
2045
2046         /* adjust if necessary - all reasons */
2047         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2048                         CPUFREQ_ADJUST, new_policy);
2049
2050         /* adjust if necessary - hardware incompatibility*/
2051         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2052                         CPUFREQ_INCOMPATIBLE, new_policy);
2053
2054         /*
2055          * verify the cpu speed can be set within this limit, which might be
2056          * different to the first one
2057          */
2058         ret = cpufreq_driver->verify(new_policy);
2059         if (ret)
2060                 goto error_out;
2061
2062         /* notification of the new policy */
2063         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2064                         CPUFREQ_NOTIFY, new_policy);
2065
2066         policy->min = new_policy->min;
2067         policy->max = new_policy->max;
2068
2069         pr_debug("new min and max freqs are %u - %u kHz\n",
2070                                         policy->min, policy->max);
2071
2072         if (cpufreq_driver->setpolicy) {
2073                 policy->policy = new_policy->policy;
2074                 pr_debug("setting range\n");
2075                 ret = cpufreq_driver->setpolicy(new_policy);
2076         } else {
2077                 if (new_policy->governor != policy->governor) {
2078                         /* save old, working values */
2079                         struct cpufreq_governor *old_gov = policy->governor;
2080
2081                         pr_debug("governor switch\n");
2082
2083                         /* end old governor */
2084                         if (policy->governor) {
2085                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2086                                 up_write(&policy->rwsem);
2087                                 __cpufreq_governor(policy,
2088                                                 CPUFREQ_GOV_POLICY_EXIT);
2089                                 down_write(&policy->rwsem);
2090                         }
2091
2092                         /* start new governor */
2093                         policy->governor = new_policy->governor;
2094                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2095                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
2096                                         failed = 0;
2097                                 } else {
2098                                         up_write(&policy->rwsem);
2099                                         __cpufreq_governor(policy,
2100                                                         CPUFREQ_GOV_POLICY_EXIT);
2101                                         down_write(&policy->rwsem);
2102                                 }
2103                         }
2104
2105                         if (failed) {
2106                                 /* new governor failed, so re-start old one */
2107                                 pr_debug("starting governor %s failed\n",
2108                                                         policy->governor->name);
2109                                 if (old_gov) {
2110                                         policy->governor = old_gov;
2111                                         __cpufreq_governor(policy,
2112                                                         CPUFREQ_GOV_POLICY_INIT);
2113                                         __cpufreq_governor(policy,
2114                                                            CPUFREQ_GOV_START);
2115                                 }
2116                                 ret = -EINVAL;
2117                                 goto error_out;
2118                         }
2119                         /* might be a policy change, too, so fall through */
2120                 }
2121                 pr_debug("governor: change or update limits\n");
2122                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2123         }
2124
2125 error_out:
2126         return ret;
2127 }
2128
2129 /**
2130  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2131  *      @cpu: CPU which shall be re-evaluated
2132  *
2133  *      Useful for policy notifiers which have different necessities
2134  *      at different times.
2135  */
2136 int cpufreq_update_policy(unsigned int cpu)
2137 {
2138         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2139         struct cpufreq_policy new_policy;
2140         int ret;
2141
2142         if (!policy) {
2143                 ret = -ENODEV;
2144                 goto no_policy;
2145         }
2146
2147         down_write(&policy->rwsem);
2148
2149         pr_debug("updating policy for CPU %u\n", cpu);
2150         memcpy(&new_policy, policy, sizeof(*policy));
2151         new_policy.min = policy->user_policy.min;
2152         new_policy.max = policy->user_policy.max;
2153         new_policy.policy = policy->user_policy.policy;
2154         new_policy.governor = policy->user_policy.governor;
2155
2156         /*
2157          * BIOS might change freq behind our back
2158          * -> ask driver for current freq and notify governors about a change
2159          */
2160         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2161                 new_policy.cur = cpufreq_driver->get(cpu);
2162                 if (!policy->cur) {
2163                         pr_debug("Driver did not initialize current freq");
2164                         policy->cur = new_policy.cur;
2165                 } else {
2166                         if (policy->cur != new_policy.cur && has_target())
2167                                 cpufreq_out_of_sync(cpu, policy->cur,
2168                                                                 new_policy.cur);
2169                 }
2170         }
2171
2172         ret = cpufreq_set_policy(policy, &new_policy);
2173
2174         up_write(&policy->rwsem);
2175
2176         cpufreq_cpu_put(policy);
2177 no_policy:
2178         return ret;
2179 }
2180 EXPORT_SYMBOL(cpufreq_update_policy);
2181
2182 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2183                                         unsigned long action, void *hcpu)
2184 {
2185         unsigned int cpu = (unsigned long)hcpu;
2186         struct device *dev;
2187         bool frozen = false;
2188
2189         dev = get_cpu_device(cpu);
2190         if (dev) {
2191
2192                 if (action & CPU_TASKS_FROZEN)
2193                         frozen = true;
2194
2195                 switch (action & ~CPU_TASKS_FROZEN) {
2196                 case CPU_ONLINE:
2197                         __cpufreq_add_dev(dev, NULL, frozen);
2198                         cpufreq_update_policy(cpu);
2199                         break;
2200
2201                 case CPU_DOWN_PREPARE:
2202                         __cpufreq_remove_dev_prepare(dev, NULL, frozen);
2203                         break;
2204
2205                 case CPU_POST_DEAD:
2206                         __cpufreq_remove_dev_finish(dev, NULL, frozen);
2207                         break;
2208
2209                 case CPU_DOWN_FAILED:
2210                         __cpufreq_add_dev(dev, NULL, frozen);
2211                         break;
2212                 }
2213         }
2214         return NOTIFY_OK;
2215 }
2216
2217 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2218         .notifier_call = cpufreq_cpu_callback,
2219 };
2220
2221 /*********************************************************************
2222  *               BOOST                                               *
2223  *********************************************************************/
2224 static int cpufreq_boost_set_sw(int state)
2225 {
2226         struct cpufreq_frequency_table *freq_table;
2227         struct cpufreq_policy *policy;
2228         int ret = -EINVAL;
2229
2230         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2231                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2232                 if (freq_table) {
2233                         ret = cpufreq_frequency_table_cpuinfo(policy,
2234                                                         freq_table);
2235                         if (ret) {
2236                                 pr_err("%s: Policy frequency update failed\n",
2237                                        __func__);
2238                                 break;
2239                         }
2240                         policy->user_policy.max = policy->max;
2241                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2242                 }
2243         }
2244
2245         return ret;
2246 }
2247
2248 int cpufreq_boost_trigger_state(int state)
2249 {
2250         unsigned long flags;
2251         int ret = 0;
2252
2253         if (cpufreq_driver->boost_enabled == state)
2254                 return 0;
2255
2256         write_lock_irqsave(&cpufreq_driver_lock, flags);
2257         cpufreq_driver->boost_enabled = state;
2258         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2259
2260         ret = cpufreq_driver->set_boost(state);
2261         if (ret) {
2262                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2263                 cpufreq_driver->boost_enabled = !state;
2264                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2265
2266                 pr_err("%s: Cannot %s BOOST\n", __func__,
2267                        state ? "enable" : "disable");
2268         }
2269
2270         return ret;
2271 }
2272
2273 int cpufreq_boost_supported(void)
2274 {
2275         if (likely(cpufreq_driver))
2276                 return cpufreq_driver->boost_supported;
2277
2278         return 0;
2279 }
2280 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2281
2282 int cpufreq_boost_enabled(void)
2283 {
2284         return cpufreq_driver->boost_enabled;
2285 }
2286 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2287
2288 /*********************************************************************
2289  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2290  *********************************************************************/
2291
2292 /**
2293  * cpufreq_register_driver - register a CPU Frequency driver
2294  * @driver_data: A struct cpufreq_driver containing the values#
2295  * submitted by the CPU Frequency driver.
2296  *
2297  * Registers a CPU Frequency driver to this core code. This code
2298  * returns zero on success, -EBUSY when another driver got here first
2299  * (and isn't unregistered in the meantime).
2300  *
2301  */
2302 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2303 {
2304         unsigned long flags;
2305         int ret;
2306
2307         if (cpufreq_disabled())
2308                 return -ENODEV;
2309
2310         if (!driver_data || !driver_data->verify || !driver_data->init ||
2311             !(driver_data->setpolicy || driver_data->target_index ||
2312                     driver_data->target))
2313                 return -EINVAL;
2314
2315         pr_debug("trying to register driver %s\n", driver_data->name);
2316
2317         if (driver_data->setpolicy)
2318                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2319
2320         write_lock_irqsave(&cpufreq_driver_lock, flags);
2321         if (cpufreq_driver) {
2322                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2323                 return -EEXIST;
2324         }
2325         cpufreq_driver = driver_data;
2326         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2327
2328         if (cpufreq_boost_supported()) {
2329                 /*
2330                  * Check if driver provides function to enable boost -
2331                  * if not, use cpufreq_boost_set_sw as default
2332                  */
2333                 if (!cpufreq_driver->set_boost)
2334                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2335
2336                 ret = cpufreq_sysfs_create_file(&boost.attr);
2337                 if (ret) {
2338                         pr_err("%s: cannot register global BOOST sysfs file\n",
2339                                 __func__);
2340                         goto err_null_driver;
2341                 }
2342         }
2343
2344         ret = subsys_interface_register(&cpufreq_interface);
2345         if (ret)
2346                 goto err_boost_unreg;
2347
2348         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2349                 int i;
2350                 ret = -ENODEV;
2351
2352                 /* check for at least one working CPU */
2353                 for (i = 0; i < nr_cpu_ids; i++)
2354                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2355                                 ret = 0;
2356                                 break;
2357                         }
2358
2359                 /* if all ->init() calls failed, unregister */
2360                 if (ret) {
2361                         pr_debug("no CPU initialized for driver %s\n",
2362                                                         driver_data->name);
2363                         goto err_if_unreg;
2364                 }
2365         }
2366
2367         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2368         pr_debug("driver %s up and running\n", driver_data->name);
2369
2370         return 0;
2371 err_if_unreg:
2372         subsys_interface_unregister(&cpufreq_interface);
2373 err_boost_unreg:
2374         if (cpufreq_boost_supported())
2375                 cpufreq_sysfs_remove_file(&boost.attr);
2376 err_null_driver:
2377         write_lock_irqsave(&cpufreq_driver_lock, flags);
2378         cpufreq_driver = NULL;
2379         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2380         return ret;
2381 }
2382 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2383
2384 /**
2385  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2386  *
2387  * Unregister the current CPUFreq driver. Only call this if you have
2388  * the right to do so, i.e. if you have succeeded in initialising before!
2389  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2390  * currently not initialised.
2391  */
2392 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2393 {
2394         unsigned long flags;
2395
2396         if (!cpufreq_driver || (driver != cpufreq_driver))
2397                 return -EINVAL;
2398
2399         pr_debug("unregistering driver %s\n", driver->name);
2400
2401         subsys_interface_unregister(&cpufreq_interface);
2402         if (cpufreq_boost_supported())
2403                 cpufreq_sysfs_remove_file(&boost.attr);
2404
2405         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2406
2407         down_write(&cpufreq_rwsem);
2408         write_lock_irqsave(&cpufreq_driver_lock, flags);
2409
2410         cpufreq_driver = NULL;
2411
2412         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2413         up_write(&cpufreq_rwsem);
2414
2415         return 0;
2416 }
2417 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2418
2419 static int __init cpufreq_core_init(void)
2420 {
2421         if (cpufreq_disabled())
2422                 return -ENODEV;
2423
2424         cpufreq_global_kobject = kobject_create();
2425         BUG_ON(!cpufreq_global_kobject);
2426         register_syscore_ops(&cpufreq_syscore_ops);
2427
2428         return 0;
2429 }
2430 core_initcall(cpufreq_core_init);