2 * Generic OPP Interface
4 * Copyright (C) 2009-2010 Texas Instruments Incorporated.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/cpu.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/device.h>
20 #include <linux/list.h>
21 #include <linux/rculist.h>
22 #include <linux/rcupdate.h>
23 #include <linux/pm_opp.h>
25 #include <linux/export.h>
28 * Internal data structure organization with the OPP layer library is as
31 * |- device 1 (represents voltage domain 1)
32 * | |- opp 1 (availability, freq, voltage)
36 * |- device 2 (represents the next voltage domain)
38 * `- device m (represents mth voltage domain)
39 * device 1, 2.. are represented by dev_opp structure while each opp
40 * is represented by the opp structure.
44 * struct dev_pm_opp - Generic OPP description structure
45 * @node: opp list node. The nodes are maintained throughout the lifetime
46 * of boot. It is expected only an optimal set of OPPs are
47 * added to the library by the SoC framework.
48 * RCU usage: opp list is traversed with RCU locks. node
49 * modification is possible realtime, hence the modifications
50 * are protected by the dev_opp_list_lock for integrity.
51 * IMPORTANT: the opp nodes should be maintained in increasing
53 * @dynamic: not-created from static DT entries.
54 * @available: true/false - marks if this OPP as available or not
55 * @turbo: true if turbo (boost) OPP
56 * @rate: Frequency in hertz
57 * @u_volt: Target voltage in microvolts corresponding to this OPP
58 * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP
59 * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP
60 * @u_amp: Maximum current drawn by the device in microamperes
61 * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
62 * frequency from any other OPP's frequency.
63 * @dev_opp: points back to the device_opp struct this opp belongs to
64 * @rcu_head: RCU callback head used for deferred freeing
65 * @np: OPP's device node.
67 * This structure stores the OPP information for a given device.
70 struct list_head node;
78 unsigned long u_volt_min;
79 unsigned long u_volt_max;
81 unsigned long clock_latency_ns;
83 struct device_opp *dev_opp;
84 struct rcu_head rcu_head;
86 struct device_node *np;
90 * struct device_list_opp - devices managed by 'struct device_opp'
92 * @dev: device to which the struct object belongs
93 * @rcu_head: RCU callback head used for deferred freeing
95 * This is an internal data structure maintaining the list of devices that are
96 * managed by 'struct device_opp'.
98 struct device_list_opp {
99 struct list_head node;
100 const struct device *dev;
101 struct rcu_head rcu_head;
105 * struct device_opp - Device opp structure
106 * @node: list node - contains the devices with OPPs that
107 * have been registered. Nodes once added are not modified in this
109 * RCU usage: nodes are not modified in the list of device_opp,
110 * however addition is possible and is secured by dev_opp_list_lock
111 * @srcu_head: notifier head to notify the OPP availability changes.
112 * @rcu_head: RCU callback head used for deferred freeing
113 * @dev_list: list of devices that share these OPPs
114 * @opp_list: list of opps
115 * @np: struct device_node pointer for opp's DT node.
116 * @shared_opp: OPP is shared between multiple devices.
118 * This is an internal data structure maintaining the link to opps attached to
119 * a device. This structure is not meant to be shared to users as it is
120 * meant for book keeping and private to OPP library.
122 * Because the opp structures can be used from both rcu and srcu readers, we
123 * need to wait for the grace period of both of them before freeing any
124 * resources. And so we have used kfree_rcu() from within call_srcu() handlers.
127 struct list_head node;
129 struct srcu_notifier_head srcu_head;
130 struct rcu_head rcu_head;
131 struct list_head dev_list;
132 struct list_head opp_list;
134 struct device_node *np;
135 unsigned long clock_latency_ns_max;
137 struct dev_pm_opp *suspend_opp;
141 * The root of the list of all devices. All device_opp structures branch off
142 * from here, with each device_opp containing the list of opp it supports in
143 * various states of availability.
145 static LIST_HEAD(dev_opp_list);
146 /* Lock to allow exclusive modification to the device and opp lists */
147 static DEFINE_MUTEX(dev_opp_list_lock);
149 #define opp_rcu_lockdep_assert() \
151 rcu_lockdep_assert(rcu_read_lock_held() || \
152 lockdep_is_held(&dev_opp_list_lock), \
153 "Missing rcu_read_lock() or " \
154 "dev_opp_list_lock protection"); \
157 static struct device_list_opp *_find_list_dev(const struct device *dev,
158 struct device_opp *dev_opp)
160 struct device_list_opp *list_dev;
162 list_for_each_entry(list_dev, &dev_opp->dev_list, node)
163 if (list_dev->dev == dev)
169 static struct device_opp *_managed_opp(const struct device_node *np)
171 struct device_opp *dev_opp;
173 list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) {
174 if (dev_opp->np == np) {
176 * Multiple devices can point to the same OPP table and
177 * so will have same node-pointer, np.
179 * But the OPPs will be considered as shared only if the
180 * OPP table contains a "opp-shared" property.
182 return dev_opp->shared_opp ? dev_opp : NULL;
190 * _find_device_opp() - find device_opp struct using device pointer
191 * @dev: device pointer used to lookup device OPPs
193 * Search list of device OPPs for one containing matching device. Does a RCU
194 * reader operation to grab the pointer needed.
196 * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or
197 * -EINVAL based on type of error.
199 * Locking: This function must be called under rcu_read_lock(). device_opp
200 * is a RCU protected pointer. This means that device_opp is valid as long
201 * as we are under RCU lock.
203 static struct device_opp *_find_device_opp(struct device *dev)
205 struct device_opp *dev_opp;
207 if (IS_ERR_OR_NULL(dev)) {
208 pr_err("%s: Invalid parameters\n", __func__);
209 return ERR_PTR(-EINVAL);
212 list_for_each_entry_rcu(dev_opp, &dev_opp_list, node)
213 if (_find_list_dev(dev, dev_opp))
216 return ERR_PTR(-ENODEV);
220 * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp
221 * @opp: opp for which voltage has to be returned for
223 * Return: voltage in micro volt corresponding to the opp, else
226 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
227 * protected pointer. This means that opp which could have been fetched by
228 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
229 * under RCU lock. The pointer returned by the opp_find_freq family must be
230 * used in the same section as the usage of this function with the pointer
231 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
234 unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp)
236 struct dev_pm_opp *tmp_opp;
239 opp_rcu_lockdep_assert();
241 tmp_opp = rcu_dereference(opp);
242 if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available)
243 pr_err("%s: Invalid parameters\n", __func__);
249 EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage);
252 * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp
253 * @opp: opp for which frequency has to be returned for
255 * Return: frequency in hertz corresponding to the opp, else
258 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
259 * protected pointer. This means that opp which could have been fetched by
260 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
261 * under RCU lock. The pointer returned by the opp_find_freq family must be
262 * used in the same section as the usage of this function with the pointer
263 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
266 unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp)
268 struct dev_pm_opp *tmp_opp;
271 opp_rcu_lockdep_assert();
273 tmp_opp = rcu_dereference(opp);
274 if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available)
275 pr_err("%s: Invalid parameters\n", __func__);
281 EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq);
284 * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not
285 * @opp: opp for which turbo mode is being verified
287 * Turbo OPPs are not for normal use, and can be enabled (under certain
288 * conditions) for short duration of times to finish high throughput work
289 * quickly. Running on them for longer times may overheat the chip.
291 * Return: true if opp is turbo opp, else false.
293 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
294 * protected pointer. This means that opp which could have been fetched by
295 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
296 * under RCU lock. The pointer returned by the opp_find_freq family must be
297 * used in the same section as the usage of this function with the pointer
298 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
301 bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp)
303 struct dev_pm_opp *tmp_opp;
305 opp_rcu_lockdep_assert();
307 tmp_opp = rcu_dereference(opp);
308 if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) {
309 pr_err("%s: Invalid parameters\n", __func__);
313 return tmp_opp->turbo;
315 EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo);
318 * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds
319 * @dev: device for which we do this operation
321 * Return: This function returns the max clock latency in nanoseconds.
323 * Locking: This function takes rcu_read_lock().
325 unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev)
327 struct device_opp *dev_opp;
328 unsigned long clock_latency_ns;
332 dev_opp = _find_device_opp(dev);
334 clock_latency_ns = 0;
336 clock_latency_ns = dev_opp->clock_latency_ns_max;
339 return clock_latency_ns;
341 EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency);
344 * dev_pm_opp_get_suspend_opp() - Get suspend opp
345 * @dev: device for which we do this operation
347 * Return: This function returns pointer to the suspend opp if it is
348 * defined, otherwise it returns NULL.
350 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
351 * protected pointer. The reason for the same is that the opp pointer which is
352 * returned will remain valid for use with opp_get_{voltage, freq} only while
353 * under the locked area. The pointer returned must be used prior to unlocking
354 * with rcu_read_unlock() to maintain the integrity of the pointer.
356 struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev)
358 struct device_opp *dev_opp;
359 struct dev_pm_opp *opp;
361 opp_rcu_lockdep_assert();
363 dev_opp = _find_device_opp(dev);
367 opp = dev_opp->suspend_opp;
371 EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp);
374 * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list
375 * @dev: device for which we do this operation
377 * Return: This function returns the number of available opps if there are any,
378 * else returns 0 if none or the corresponding error value.
380 * Locking: This function takes rcu_read_lock().
382 int dev_pm_opp_get_opp_count(struct device *dev)
384 struct device_opp *dev_opp;
385 struct dev_pm_opp *temp_opp;
390 dev_opp = _find_device_opp(dev);
391 if (IS_ERR(dev_opp)) {
392 count = PTR_ERR(dev_opp);
393 dev_err(dev, "%s: device OPP not found (%d)\n",
398 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
399 if (temp_opp->available)
407 EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count);
410 * dev_pm_opp_find_freq_exact() - search for an exact frequency
411 * @dev: device for which we do this operation
412 * @freq: frequency to search for
413 * @available: true/false - match for available opp
415 * Return: Searches for exact match in the opp list and returns pointer to the
416 * matching opp if found, else returns ERR_PTR in case of error and should
417 * be handled using IS_ERR. Error return values can be:
418 * EINVAL: for bad pointer
419 * ERANGE: no match found for search
420 * ENODEV: if device not found in list of registered devices
422 * Note: available is a modifier for the search. if available=true, then the
423 * match is for exact matching frequency and is available in the stored OPP
424 * table. if false, the match is for exact frequency which is not available.
426 * This provides a mechanism to enable an opp which is not available currently
427 * or the opposite as well.
429 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
430 * protected pointer. The reason for the same is that the opp pointer which is
431 * returned will remain valid for use with opp_get_{voltage, freq} only while
432 * under the locked area. The pointer returned must be used prior to unlocking
433 * with rcu_read_unlock() to maintain the integrity of the pointer.
435 struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev,
439 struct device_opp *dev_opp;
440 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
442 opp_rcu_lockdep_assert();
444 dev_opp = _find_device_opp(dev);
445 if (IS_ERR(dev_opp)) {
446 int r = PTR_ERR(dev_opp);
447 dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r);
451 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
452 if (temp_opp->available == available &&
453 temp_opp->rate == freq) {
461 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact);
464 * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq
465 * @dev: device for which we do this operation
466 * @freq: Start frequency
468 * Search for the matching ceil *available* OPP from a starting freq
471 * Return: matching *opp and refreshes *freq accordingly, else returns
472 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
474 * EINVAL: for bad pointer
475 * ERANGE: no match found for search
476 * ENODEV: if device not found in list of registered devices
478 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
479 * protected pointer. The reason for the same is that the opp pointer which is
480 * returned will remain valid for use with opp_get_{voltage, freq} only while
481 * under the locked area. The pointer returned must be used prior to unlocking
482 * with rcu_read_unlock() to maintain the integrity of the pointer.
484 struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev,
487 struct device_opp *dev_opp;
488 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
490 opp_rcu_lockdep_assert();
493 dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq);
494 return ERR_PTR(-EINVAL);
497 dev_opp = _find_device_opp(dev);
499 return ERR_CAST(dev_opp);
501 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
502 if (temp_opp->available && temp_opp->rate >= *freq) {
511 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil);
514 * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq
515 * @dev: device for which we do this operation
516 * @freq: Start frequency
518 * Search for the matching floor *available* OPP from a starting freq
521 * Return: matching *opp and refreshes *freq accordingly, else returns
522 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
524 * EINVAL: for bad pointer
525 * ERANGE: no match found for search
526 * ENODEV: if device not found in list of registered devices
528 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
529 * protected pointer. The reason for the same is that the opp pointer which is
530 * returned will remain valid for use with opp_get_{voltage, freq} only while
531 * under the locked area. The pointer returned must be used prior to unlocking
532 * with rcu_read_unlock() to maintain the integrity of the pointer.
534 struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev,
537 struct device_opp *dev_opp;
538 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
540 opp_rcu_lockdep_assert();
543 dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq);
544 return ERR_PTR(-EINVAL);
547 dev_opp = _find_device_opp(dev);
549 return ERR_CAST(dev_opp);
551 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
552 if (temp_opp->available) {
553 /* go to the next node, before choosing prev */
554 if (temp_opp->rate > *freq)
565 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor);
567 /* List-dev Helpers */
568 static void _kfree_list_dev_rcu(struct rcu_head *head)
570 struct device_list_opp *list_dev;
572 list_dev = container_of(head, struct device_list_opp, rcu_head);
573 kfree_rcu(list_dev, rcu_head);
576 static void _remove_list_dev(struct device_list_opp *list_dev,
577 struct device_opp *dev_opp)
579 list_del(&list_dev->node);
580 call_srcu(&dev_opp->srcu_head.srcu, &list_dev->rcu_head,
581 _kfree_list_dev_rcu);
584 static struct device_list_opp *_add_list_dev(const struct device *dev,
585 struct device_opp *dev_opp)
587 struct device_list_opp *list_dev;
589 list_dev = kzalloc(sizeof(*list_dev), GFP_KERNEL);
593 /* Initialize list-dev */
595 list_add_rcu(&list_dev->node, &dev_opp->dev_list);
601 * _add_device_opp() - Find device OPP table or allocate a new one
602 * @dev: device for which we do this operation
604 * It tries to find an existing table first, if it couldn't find one, it
605 * allocates a new OPP table and returns that.
607 * Return: valid device_opp pointer if success, else NULL.
609 static struct device_opp *_add_device_opp(struct device *dev)
611 struct device_opp *dev_opp;
612 struct device_list_opp *list_dev;
614 /* Check for existing list for 'dev' first */
615 dev_opp = _find_device_opp(dev);
616 if (!IS_ERR(dev_opp))
620 * Allocate a new device OPP table. In the infrequent case where a new
621 * device is needed to be added, we pay this penalty.
623 dev_opp = kzalloc(sizeof(*dev_opp), GFP_KERNEL);
627 INIT_LIST_HEAD(&dev_opp->dev_list);
629 list_dev = _add_list_dev(dev, dev_opp);
635 srcu_init_notifier_head(&dev_opp->srcu_head);
636 INIT_LIST_HEAD(&dev_opp->opp_list);
638 /* Secure the device list modification */
639 list_add_rcu(&dev_opp->node, &dev_opp_list);
644 * _kfree_device_rcu() - Free device_opp RCU handler
647 static void _kfree_device_rcu(struct rcu_head *head)
649 struct device_opp *device_opp = container_of(head, struct device_opp, rcu_head);
651 kfree_rcu(device_opp, rcu_head);
655 * _remove_device_opp() - Removes a device OPP table
656 * @dev_opp: device OPP table to be removed.
658 * Removes/frees device OPP table it it doesn't contain any OPPs.
660 static void _remove_device_opp(struct device_opp *dev_opp)
662 struct device_list_opp *list_dev;
664 if (!list_empty(&dev_opp->opp_list))
667 list_dev = list_first_entry(&dev_opp->dev_list, struct device_list_opp,
670 _remove_list_dev(list_dev, dev_opp);
672 /* dev_list must be empty now */
673 WARN_ON(!list_empty(&dev_opp->dev_list));
675 list_del_rcu(&dev_opp->node);
676 call_srcu(&dev_opp->srcu_head.srcu, &dev_opp->rcu_head,
681 * _kfree_opp_rcu() - Free OPP RCU handler
684 static void _kfree_opp_rcu(struct rcu_head *head)
686 struct dev_pm_opp *opp = container_of(head, struct dev_pm_opp, rcu_head);
688 kfree_rcu(opp, rcu_head);
692 * _opp_remove() - Remove an OPP from a table definition
693 * @dev_opp: points back to the device_opp struct this opp belongs to
694 * @opp: pointer to the OPP to remove
695 * @notify: OPP_EVENT_REMOVE notification should be sent or not
697 * This function removes an opp definition from the opp list.
699 * Locking: The internal device_opp and opp structures are RCU protected.
700 * It is assumed that the caller holds required mutex for an RCU updater
703 static void _opp_remove(struct device_opp *dev_opp,
704 struct dev_pm_opp *opp, bool notify)
707 * Notify the changes in the availability of the operable
708 * frequency/voltage list.
711 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_REMOVE, opp);
712 list_del_rcu(&opp->node);
713 call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu);
715 _remove_device_opp(dev_opp);
719 * dev_pm_opp_remove() - Remove an OPP from OPP list
720 * @dev: device for which we do this operation
721 * @freq: OPP to remove with matching 'freq'
723 * This function removes an opp from the opp list.
725 * Locking: The internal device_opp and opp structures are RCU protected.
726 * Hence this function internally uses RCU updater strategy with mutex locks
727 * to keep the integrity of the internal data structures. Callers should ensure
728 * that this function is *NOT* called under RCU protection or in contexts where
729 * mutex cannot be locked.
731 void dev_pm_opp_remove(struct device *dev, unsigned long freq)
733 struct dev_pm_opp *opp;
734 struct device_opp *dev_opp;
737 /* Hold our list modification lock here */
738 mutex_lock(&dev_opp_list_lock);
740 dev_opp = _find_device_opp(dev);
744 list_for_each_entry(opp, &dev_opp->opp_list, node) {
745 if (opp->rate == freq) {
752 dev_warn(dev, "%s: Couldn't find OPP with freq: %lu\n",
757 _opp_remove(dev_opp, opp, true);
759 mutex_unlock(&dev_opp_list_lock);
761 EXPORT_SYMBOL_GPL(dev_pm_opp_remove);
763 static struct dev_pm_opp *_allocate_opp(struct device *dev,
764 struct device_opp **dev_opp)
766 struct dev_pm_opp *opp;
768 /* allocate new OPP node */
769 opp = kzalloc(sizeof(*opp), GFP_KERNEL);
773 INIT_LIST_HEAD(&opp->node);
775 *dev_opp = _add_device_opp(dev);
784 static int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
785 struct device_opp *dev_opp)
787 struct dev_pm_opp *opp;
788 struct list_head *head = &dev_opp->opp_list;
791 * Insert new OPP in order of increasing frequency and discard if
794 * Need to use &dev_opp->opp_list in the condition part of the 'for'
795 * loop, don't replace it with head otherwise it will become an infinite
798 list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) {
799 if (new_opp->rate > opp->rate) {
804 if (new_opp->rate < opp->rate)
808 dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
809 __func__, opp->rate, opp->u_volt, opp->available,
810 new_opp->rate, new_opp->u_volt, new_opp->available);
812 return opp->available && new_opp->u_volt == opp->u_volt ?
816 new_opp->dev_opp = dev_opp;
817 list_add_rcu(&new_opp->node, head);
823 * _opp_add_dynamic() - Allocate a dynamic OPP.
824 * @dev: device for which we do this operation
825 * @freq: Frequency in Hz for this OPP
826 * @u_volt: Voltage in uVolts for this OPP
827 * @dynamic: Dynamically added OPPs.
829 * This function adds an opp definition to the opp list and returns status.
830 * The opp is made available by default and it can be controlled using
831 * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove.
833 * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and
834 * freed by of_free_opp_table.
836 * Locking: The internal device_opp and opp structures are RCU protected.
837 * Hence this function internally uses RCU updater strategy with mutex locks
838 * to keep the integrity of the internal data structures. Callers should ensure
839 * that this function is *NOT* called under RCU protection or in contexts where
840 * mutex cannot be locked.
844 * Duplicate OPPs (both freq and volt are same) and opp->available
845 * -EEXIST Freq are same and volt are different OR
846 * Duplicate OPPs (both freq and volt are same) and !opp->available
847 * -ENOMEM Memory allocation failure
849 static int _opp_add_dynamic(struct device *dev, unsigned long freq,
850 long u_volt, bool dynamic)
852 struct device_opp *dev_opp;
853 struct dev_pm_opp *new_opp;
856 /* Hold our list modification lock here */
857 mutex_lock(&dev_opp_list_lock);
859 new_opp = _allocate_opp(dev, &dev_opp);
865 /* populate the opp table */
866 new_opp->rate = freq;
867 new_opp->u_volt = u_volt;
868 new_opp->available = true;
869 new_opp->dynamic = dynamic;
871 ret = _opp_add(dev, new_opp, dev_opp);
875 mutex_unlock(&dev_opp_list_lock);
878 * Notify the changes in the availability of the operable
879 * frequency/voltage list.
881 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp);
885 _opp_remove(dev_opp, new_opp, false);
887 mutex_unlock(&dev_opp_list_lock);
891 /* TODO: Support multiple regulators */
892 static int opp_get_microvolt(struct dev_pm_opp *opp, struct device *dev)
894 u32 microvolt[3] = {0};
897 count = of_property_count_u32_elems(opp->np, "opp-microvolt");
901 /* There can be one or three elements here */
902 if (count != 1 && count != 3) {
903 dev_err(dev, "%s: Invalid number of elements in opp-microvolt property (%d)\n",
908 ret = of_property_read_u32_array(opp->np, "opp-microvolt", microvolt,
911 dev_err(dev, "%s: error parsing opp-microvolt: %d\n", __func__,
916 opp->u_volt = microvolt[0];
917 opp->u_volt_min = microvolt[1];
918 opp->u_volt_max = microvolt[2];
924 * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings)
925 * @dev: device for which we do this operation
928 * This function adds an opp definition to the opp list and returns status. The
929 * opp can be controlled using dev_pm_opp_enable/disable functions and may be
930 * removed by dev_pm_opp_remove.
932 * Locking: The internal device_opp and opp structures are RCU protected.
933 * Hence this function internally uses RCU updater strategy with mutex locks
934 * to keep the integrity of the internal data structures. Callers should ensure
935 * that this function is *NOT* called under RCU protection or in contexts where
936 * mutex cannot be locked.
940 * Duplicate OPPs (both freq and volt are same) and opp->available
941 * -EEXIST Freq are same and volt are different OR
942 * Duplicate OPPs (both freq and volt are same) and !opp->available
943 * -ENOMEM Memory allocation failure
944 * -EINVAL Failed parsing the OPP node
946 static int _opp_add_static_v2(struct device *dev, struct device_node *np)
948 struct device_opp *dev_opp;
949 struct dev_pm_opp *new_opp;
954 /* Hold our list modification lock here */
955 mutex_lock(&dev_opp_list_lock);
957 new_opp = _allocate_opp(dev, &dev_opp);
963 ret = of_property_read_u64(np, "opp-hz", &rate);
965 dev_err(dev, "%s: opp-hz not found\n", __func__);
970 * Rate is defined as an unsigned long in clk API, and so casting
971 * explicitly to its type. Must be fixed once rate is 64 bit
972 * guaranteed in clk API.
974 new_opp->rate = (unsigned long)rate;
975 new_opp->turbo = of_property_read_bool(np, "turbo-mode");
978 new_opp->dynamic = false;
979 new_opp->available = true;
981 if (!of_property_read_u32(np, "clock-latency-ns", &val))
982 new_opp->clock_latency_ns = val;
984 ret = opp_get_microvolt(new_opp, dev);
988 if (!of_property_read_u32(new_opp->np, "opp-microamp", &val))
989 new_opp->u_amp = val;
991 ret = _opp_add(dev, new_opp, dev_opp);
995 /* OPP to select on device suspend */
996 if (of_property_read_bool(np, "opp-suspend")) {
997 if (dev_opp->suspend_opp)
998 dev_warn(dev, "%s: Multiple suspend OPPs found (%lu %lu)\n",
999 __func__, dev_opp->suspend_opp->rate,
1002 dev_opp->suspend_opp = new_opp;
1005 if (new_opp->clock_latency_ns > dev_opp->clock_latency_ns_max)
1006 dev_opp->clock_latency_ns_max = new_opp->clock_latency_ns;
1008 mutex_unlock(&dev_opp_list_lock);
1010 pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n",
1011 __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt,
1012 new_opp->u_volt_min, new_opp->u_volt_max,
1013 new_opp->clock_latency_ns);
1016 * Notify the changes in the availability of the operable
1017 * frequency/voltage list.
1019 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp);
1023 _opp_remove(dev_opp, new_opp, false);
1025 mutex_unlock(&dev_opp_list_lock);
1030 * dev_pm_opp_add() - Add an OPP table from a table definitions
1031 * @dev: device for which we do this operation
1032 * @freq: Frequency in Hz for this OPP
1033 * @u_volt: Voltage in uVolts for this OPP
1035 * This function adds an opp definition to the opp list and returns status.
1036 * The opp is made available by default and it can be controlled using
1037 * dev_pm_opp_enable/disable functions.
1039 * Locking: The internal device_opp and opp structures are RCU protected.
1040 * Hence this function internally uses RCU updater strategy with mutex locks
1041 * to keep the integrity of the internal data structures. Callers should ensure
1042 * that this function is *NOT* called under RCU protection or in contexts where
1043 * mutex cannot be locked.
1047 * Duplicate OPPs (both freq and volt are same) and opp->available
1048 * -EEXIST Freq are same and volt are different OR
1049 * Duplicate OPPs (both freq and volt are same) and !opp->available
1050 * -ENOMEM Memory allocation failure
1052 int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt)
1054 return _opp_add_dynamic(dev, freq, u_volt, true);
1056 EXPORT_SYMBOL_GPL(dev_pm_opp_add);
1059 * _opp_set_availability() - helper to set the availability of an opp
1060 * @dev: device for which we do this operation
1061 * @freq: OPP frequency to modify availability
1062 * @availability_req: availability status requested for this opp
1064 * Set the availability of an OPP with an RCU operation, opp_{enable,disable}
1065 * share a common logic which is isolated here.
1067 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1068 * copy operation, returns 0 if no modifcation was done OR modification was
1071 * Locking: The internal device_opp and opp structures are RCU protected.
1072 * Hence this function internally uses RCU updater strategy with mutex locks to
1073 * keep the integrity of the internal data structures. Callers should ensure
1074 * that this function is *NOT* called under RCU protection or in contexts where
1075 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1077 static int _opp_set_availability(struct device *dev, unsigned long freq,
1078 bool availability_req)
1080 struct device_opp *dev_opp;
1081 struct dev_pm_opp *new_opp, *tmp_opp, *opp = ERR_PTR(-ENODEV);
1084 /* keep the node allocated */
1085 new_opp = kmalloc(sizeof(*new_opp), GFP_KERNEL);
1089 mutex_lock(&dev_opp_list_lock);
1091 /* Find the device_opp */
1092 dev_opp = _find_device_opp(dev);
1093 if (IS_ERR(dev_opp)) {
1094 r = PTR_ERR(dev_opp);
1095 dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r);
1099 /* Do we have the frequency? */
1100 list_for_each_entry(tmp_opp, &dev_opp->opp_list, node) {
1101 if (tmp_opp->rate == freq) {
1111 /* Is update really needed? */
1112 if (opp->available == availability_req)
1114 /* copy the old data over */
1117 /* plug in new node */
1118 new_opp->available = availability_req;
1120 list_replace_rcu(&opp->node, &new_opp->node);
1121 mutex_unlock(&dev_opp_list_lock);
1122 call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu);
1124 /* Notify the change of the OPP availability */
1125 if (availability_req)
1126 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ENABLE,
1129 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_DISABLE,
1135 mutex_unlock(&dev_opp_list_lock);
1141 * dev_pm_opp_enable() - Enable a specific OPP
1142 * @dev: device for which we do this operation
1143 * @freq: OPP frequency to enable
1145 * Enables a provided opp. If the operation is valid, this returns 0, else the
1146 * corresponding error value. It is meant to be used for users an OPP available
1147 * after being temporarily made unavailable with dev_pm_opp_disable.
1149 * Locking: The internal device_opp and opp structures are RCU protected.
1150 * Hence this function indirectly uses RCU and mutex locks to keep the
1151 * integrity of the internal data structures. Callers should ensure that
1152 * this function is *NOT* called under RCU protection or in contexts where
1153 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1155 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1156 * copy operation, returns 0 if no modifcation was done OR modification was
1159 int dev_pm_opp_enable(struct device *dev, unsigned long freq)
1161 return _opp_set_availability(dev, freq, true);
1163 EXPORT_SYMBOL_GPL(dev_pm_opp_enable);
1166 * dev_pm_opp_disable() - Disable a specific OPP
1167 * @dev: device for which we do this operation
1168 * @freq: OPP frequency to disable
1170 * Disables a provided opp. If the operation is valid, this returns
1171 * 0, else the corresponding error value. It is meant to be a temporary
1172 * control by users to make this OPP not available until the circumstances are
1173 * right to make it available again (with a call to dev_pm_opp_enable).
1175 * Locking: The internal device_opp and opp structures are RCU protected.
1176 * Hence this function indirectly uses RCU and mutex locks to keep the
1177 * integrity of the internal data structures. Callers should ensure that
1178 * this function is *NOT* called under RCU protection or in contexts where
1179 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1181 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1182 * copy operation, returns 0 if no modifcation was done OR modification was
1185 int dev_pm_opp_disable(struct device *dev, unsigned long freq)
1187 return _opp_set_availability(dev, freq, false);
1189 EXPORT_SYMBOL_GPL(dev_pm_opp_disable);
1192 * dev_pm_opp_get_notifier() - find notifier_head of the device with opp
1193 * @dev: device pointer used to lookup device OPPs.
1195 * Return: pointer to notifier head if found, otherwise -ENODEV or
1196 * -EINVAL based on type of error casted as pointer. value must be checked
1197 * with IS_ERR to determine valid pointer or error result.
1199 * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU
1200 * protected pointer. The reason for the same is that the opp pointer which is
1201 * returned will remain valid for use with opp_get_{voltage, freq} only while
1202 * under the locked area. The pointer returned must be used prior to unlocking
1203 * with rcu_read_unlock() to maintain the integrity of the pointer.
1205 struct srcu_notifier_head *dev_pm_opp_get_notifier(struct device *dev)
1207 struct device_opp *dev_opp = _find_device_opp(dev);
1209 if (IS_ERR(dev_opp))
1210 return ERR_CAST(dev_opp); /* matching type */
1212 return &dev_opp->srcu_head;
1214 EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier);
1218 * of_free_opp_table() - Free OPP table entries created from static DT entries
1219 * @dev: device pointer used to lookup device OPPs.
1221 * Free OPPs created using static entries present in DT.
1223 * Locking: The internal device_opp and opp structures are RCU protected.
1224 * Hence this function indirectly uses RCU updater strategy with mutex locks
1225 * to keep the integrity of the internal data structures. Callers should ensure
1226 * that this function is *NOT* called under RCU protection or in contexts where
1227 * mutex cannot be locked.
1229 void of_free_opp_table(struct device *dev)
1231 struct device_opp *dev_opp;
1232 struct dev_pm_opp *opp, *tmp;
1234 /* Hold our list modification lock here */
1235 mutex_lock(&dev_opp_list_lock);
1237 /* Check for existing list for 'dev' */
1238 dev_opp = _find_device_opp(dev);
1239 if (IS_ERR(dev_opp)) {
1240 int error = PTR_ERR(dev_opp);
1242 if (error != -ENODEV)
1243 WARN(1, "%s: dev_opp: %d\n",
1244 IS_ERR_OR_NULL(dev) ?
1245 "Invalid device" : dev_name(dev),
1250 /* Find if dev_opp manages a single device */
1251 if (list_is_singular(&dev_opp->dev_list)) {
1252 /* Free static OPPs */
1253 list_for_each_entry_safe(opp, tmp, &dev_opp->opp_list, node) {
1255 _opp_remove(dev_opp, opp, true);
1258 _remove_list_dev(_find_list_dev(dev, dev_opp), dev_opp);
1262 mutex_unlock(&dev_opp_list_lock);
1264 EXPORT_SYMBOL_GPL(of_free_opp_table);
1266 void of_cpumask_free_opp_table(cpumask_var_t cpumask)
1268 struct device *cpu_dev;
1271 WARN_ON(cpumask_empty(cpumask));
1273 for_each_cpu(cpu, cpumask) {
1274 cpu_dev = get_cpu_device(cpu);
1276 pr_err("%s: failed to get cpu%d device\n", __func__,
1281 of_free_opp_table(cpu_dev);
1284 EXPORT_SYMBOL_GPL(of_cpumask_free_opp_table);
1286 /* Returns opp descriptor node from its phandle. Caller must do of_node_put() */
1287 static struct device_node *
1288 _of_get_opp_desc_node_from_prop(struct device *dev, const struct property *prop)
1290 struct device_node *opp_np;
1292 opp_np = of_find_node_by_phandle(be32_to_cpup(prop->value));
1294 dev_err(dev, "%s: Prop: %s contains invalid opp desc phandle\n",
1295 __func__, prop->name);
1296 return ERR_PTR(-EINVAL);
1302 /* Returns opp descriptor node for a device. Caller must do of_node_put() */
1303 static struct device_node *_of_get_opp_desc_node(struct device *dev)
1305 const struct property *prop;
1307 prop = of_find_property(dev->of_node, "operating-points-v2", NULL);
1309 return ERR_PTR(-ENODEV);
1311 return ERR_PTR(-ENODATA);
1314 * TODO: Support for multiple OPP tables.
1316 * There should be only ONE phandle present in "operating-points-v2"
1319 if (prop->length != sizeof(__be32)) {
1320 dev_err(dev, "%s: Invalid opp desc phandle\n", __func__);
1321 return ERR_PTR(-EINVAL);
1324 return _of_get_opp_desc_node_from_prop(dev, prop);
1327 /* Initializes OPP tables based on new bindings */
1328 static int _of_init_opp_table_v2(struct device *dev,
1329 const struct property *prop)
1331 struct device_node *opp_np, *np;
1332 struct device_opp *dev_opp;
1333 int ret = 0, count = 0;
1339 opp_np = _of_get_opp_desc_node_from_prop(dev, prop);
1341 return PTR_ERR(opp_np);
1343 dev_opp = _managed_opp(opp_np);
1345 /* OPPs are already managed */
1346 if (!_add_list_dev(dev, dev_opp))
1351 /* We have opp-list node now, iterate over it and add OPPs */
1352 for_each_available_child_of_node(opp_np, np) {
1355 ret = _opp_add_static_v2(dev, np);
1357 dev_err(dev, "%s: Failed to add OPP, %d\n", __func__,
1363 /* There should be one of more OPP defined */
1364 if (WARN_ON(!count)) {
1369 dev_opp = _find_device_opp(dev);
1370 if (WARN_ON(IS_ERR(dev_opp))) {
1371 ret = PTR_ERR(dev_opp);
1375 dev_opp->np = opp_np;
1376 dev_opp->shared_opp = of_property_read_bool(opp_np, "opp-shared");
1378 of_node_put(opp_np);
1382 of_free_opp_table(dev);
1384 of_node_put(opp_np);
1389 /* Initializes OPP tables based on old-deprecated bindings */
1390 static int _of_init_opp_table_v1(struct device *dev)
1392 const struct property *prop;
1396 prop = of_find_property(dev->of_node, "operating-points", NULL);
1403 * Each OPP is a set of tuples consisting of frequency and
1404 * voltage like <freq-kHz vol-uV>.
1406 nr = prop->length / sizeof(u32);
1408 dev_err(dev, "%s: Invalid OPP list\n", __func__);
1414 unsigned long freq = be32_to_cpup(val++) * 1000;
1415 unsigned long volt = be32_to_cpup(val++);
1417 if (_opp_add_dynamic(dev, freq, volt, false))
1418 dev_warn(dev, "%s: Failed to add OPP %ld\n",
1427 * of_init_opp_table() - Initialize opp table from device tree
1428 * @dev: device pointer used to lookup device OPPs.
1430 * Register the initial OPP table with the OPP library for given device.
1432 * Locking: The internal device_opp and opp structures are RCU protected.
1433 * Hence this function indirectly uses RCU updater strategy with mutex locks
1434 * to keep the integrity of the internal data structures. Callers should ensure
1435 * that this function is *NOT* called under RCU protection or in contexts where
1436 * mutex cannot be locked.
1440 * Duplicate OPPs (both freq and volt are same) and opp->available
1441 * -EEXIST Freq are same and volt are different OR
1442 * Duplicate OPPs (both freq and volt are same) and !opp->available
1443 * -ENOMEM Memory allocation failure
1444 * -ENODEV when 'operating-points' property is not found or is invalid data
1446 * -ENODATA when empty 'operating-points' property is found
1447 * -EINVAL when invalid entries are found in opp-v2 table
1449 int of_init_opp_table(struct device *dev)
1451 const struct property *prop;
1454 * OPPs have two version of bindings now. The older one is deprecated,
1455 * try for the new binding first.
1457 prop = of_find_property(dev->of_node, "operating-points-v2", NULL);
1460 * Try old-deprecated bindings for backward compatibility with
1463 return _of_init_opp_table_v1(dev);
1466 return _of_init_opp_table_v2(dev, prop);
1468 EXPORT_SYMBOL_GPL(of_init_opp_table);
1470 int of_cpumask_init_opp_table(cpumask_var_t cpumask)
1472 struct device *cpu_dev;
1475 WARN_ON(cpumask_empty(cpumask));
1477 for_each_cpu(cpu, cpumask) {
1478 cpu_dev = get_cpu_device(cpu);
1480 pr_err("%s: failed to get cpu%d device\n", __func__,
1485 ret = of_init_opp_table(cpu_dev);
1487 pr_err("%s: couldn't find opp table for cpu:%d, %d\n",
1488 __func__, cpu, ret);
1490 /* Free all other OPPs */
1491 of_cpumask_free_opp_table(cpumask);
1498 EXPORT_SYMBOL_GPL(of_cpumask_init_opp_table);
1500 /* Required only for V1 bindings, as v2 can manage it from DT itself */
1501 int set_cpus_sharing_opps(struct device *cpu_dev, cpumask_var_t cpumask)
1503 struct device_list_opp *list_dev;
1504 struct device_opp *dev_opp;
1510 dev_opp = _find_device_opp(cpu_dev);
1511 if (IS_ERR(dev_opp)) {
1513 goto out_rcu_read_unlock;
1516 for_each_cpu(cpu, cpumask) {
1517 if (cpu == cpu_dev->id)
1520 dev = get_cpu_device(cpu);
1522 dev_err(cpu_dev, "%s: failed to get cpu%d device\n",
1527 list_dev = _add_list_dev(dev, dev_opp);
1529 dev_err(dev, "%s: failed to add list-dev for cpu%d device\n",
1534 out_rcu_read_unlock:
1539 EXPORT_SYMBOL_GPL(set_cpus_sharing_opps);
1542 * Works only for OPP v2 bindings.
1544 * cpumask should be already set to mask of cpu_dev->id.
1545 * Returns -ENOENT if operating-points-v2 bindings aren't supported.
1547 int of_get_cpus_sharing_opps(struct device *cpu_dev, cpumask_var_t cpumask)
1549 struct device_node *np, *tmp_np;
1550 struct device *tcpu_dev;
1553 /* Get OPP descriptor node */
1554 np = _of_get_opp_desc_node(cpu_dev);
1556 dev_dbg(cpu_dev, "%s: Couldn't find opp node: %ld\n", __func__,
1561 /* OPPs are shared ? */
1562 if (!of_property_read_bool(np, "opp-shared"))
1565 for_each_possible_cpu(cpu) {
1566 if (cpu == cpu_dev->id)
1569 tcpu_dev = get_cpu_device(cpu);
1571 dev_err(cpu_dev, "%s: failed to get cpu%d device\n",
1577 /* Get OPP descriptor node */
1578 tmp_np = _of_get_opp_desc_node(tcpu_dev);
1579 if (IS_ERR(tmp_np)) {
1580 dev_err(tcpu_dev, "%s: Couldn't find opp node: %ld\n",
1581 __func__, PTR_ERR(tmp_np));
1582 ret = PTR_ERR(tmp_np);
1586 /* CPUs are sharing opp node */
1588 cpumask_set_cpu(cpu, cpumask);
1590 of_node_put(tmp_np);
1597 EXPORT_SYMBOL_GPL(of_get_cpus_sharing_opps);