drivers/base/power/main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * drivers/base/power/main.c - Where the driver meets power management.
   4  *
   5  * Copyright (c) 2003 Patrick Mochel
   6  * Copyright (c) 2003 Open Source Development Lab
   7  *
   8  * The driver model core calls device_pm_add() when a device is registered.
   9  * This will initialize the embedded device_pm_info object in the device
  10  * and add it to the list of power-controlled devices. sysfs entries for
  11  * controlling device power management will also be added.
  12  *
  13  * A separate list is used for keeping track of power info, because the power
  14  * domain dependencies may differ from the ancestral dependencies that the
  15  * subsystem list maintains.
  16  */
  17
  18 #define pr_fmt(fmt) "PM: " fmt
  19 #define dev_fmt pr_fmt
  20
  21 #include <linux/device.h>
  22 #include <linux/export.h>
  23 #include <linux/mutex.h>
  24 #include <linux/pm.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/pm-trace.h>
  27 #include <linux/pm_wakeirq.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/sched.h>
  30 #include <linux/sched/debug.h>
  31 #include <linux/async.h>
  32 #include <linux/suspend.h>
  33 #include <trace/events/power.h>
  34 #include <linux/cpufreq.h>
  35 #include <linux/devfreq.h>
  36 #include <linux/timer.h>
  37
  38 #include "../base.h"
  39 #include "power.h"
  40
  41 typedef int (*pm_callback_t)(struct device *);
  42
  43 #define list_for_each_entry_rcu_locked(pos, head, member) \
  44         list_for_each_entry_rcu(pos, head, member, \
  45                         device_links_read_lock_held())
  46
  47 /*
  48  * The entries in the dpm_list list are in a depth first order, simply
  49  * because children are guaranteed to be discovered after parents, and
  50  * are inserted at the back of the list on discovery.
  51  *
  52  * Since device_pm_add() may be called with a device lock held,
  53  * we must never try to acquire a device lock while holding
  54  * dpm_list_mutex.
  55  */
  56
  57 LIST_HEAD(dpm_list);
  58 static LIST_HEAD(dpm_prepared_list);
  59 static LIST_HEAD(dpm_suspended_list);
  60 static LIST_HEAD(dpm_late_early_list);
  61 static LIST_HEAD(dpm_noirq_list);
  62
  63 struct suspend_stats suspend_stats;
  64 static DEFINE_MUTEX(dpm_list_mtx);
  65 static pm_message_t pm_transition;
  66
  67 static int async_error;
  68
  69 static const char *pm_verb(int event)
  70 {
  71         switch (event) {
  72         case PM_EVENT_SUSPEND:
  73                 return "suspend";
  74         case PM_EVENT_RESUME:
  75                 return "resume";
  76         case PM_EVENT_FREEZE:
  77                 return "freeze";
  78         case PM_EVENT_QUIESCE:
  79                 return "quiesce";
  80         case PM_EVENT_HIBERNATE:
  81                 return "hibernate";
  82         case PM_EVENT_THAW:
  83                 return "thaw";
  84         case PM_EVENT_RESTORE:
  85                 return "restore";
  86         case PM_EVENT_RECOVER:
  87                 return "recover";
  88         default:
  89                 return "(unknown PM event)";
  90         }
  91 }
  92
  93 /**
  94  * device_pm_sleep_init - Initialize system suspend-related device fields.
  95  * @dev: Device object being initialized.
  96  */
  97 void device_pm_sleep_init(struct device *dev)
  98 {
  99         dev->power.is_prepared = false;
 100         dev->power.is_suspended = false;
 101         dev->power.is_noirq_suspended = false;
 102         dev->power.is_late_suspended = false;
 103         init_completion(&dev->power.completion);
 104         complete_all(&dev->power.completion);
 105         dev->power.wakeup = NULL;
 106         INIT_LIST_HEAD(&dev->power.entry);
 107 }
 108
 109 /**
 110  * device_pm_lock - Lock the list of active devices used by the PM core.
 111  */
 112 void device_pm_lock(void)
 113 {
 114         mutex_lock(&dpm_list_mtx);
 115 }
 116
 117 /**
 118  * device_pm_unlock - Unlock the list of active devices used by the PM core.
 119  */
 120 void device_pm_unlock(void)
 121 {
 122         mutex_unlock(&dpm_list_mtx);
 123 }
 124
 125 /**
 126  * device_pm_add - Add a device to the PM core's list of active devices.
 127  * @dev: Device to add to the list.
 128  */
 129 void device_pm_add(struct device *dev)
 130 {
 131         /* Skip PM setup/initialization. */
 132         if (device_pm_not_required(dev))
 133                 return;
 134
 135         pr_debug("Adding info for %s:%s\n",
 136                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 137         device_pm_check_callbacks(dev);
 138         mutex_lock(&dpm_list_mtx);
 139         if (dev->parent && dev->parent->power.is_prepared)
 140                 dev_warn(dev, "parent %s should not be sleeping\n",
 141                         dev_name(dev->parent));
 142         list_add_tail(&dev->power.entry, &dpm_list);
 143         dev->power.in_dpm_list = true;
 144         mutex_unlock(&dpm_list_mtx);
 145 }
 146
 147 /**
 148  * device_pm_remove - Remove a device from the PM core's list of active devices.
 149  * @dev: Device to be removed from the list.
 150  */
 151 void device_pm_remove(struct device *dev)
 152 {
 153         if (device_pm_not_required(dev))
 154                 return;
 155
 156         pr_debug("Removing info for %s:%s\n",
 157                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 158         complete_all(&dev->power.completion);
 159         mutex_lock(&dpm_list_mtx);
 160         list_del_init(&dev->power.entry);
 161         dev->power.in_dpm_list = false;
 162         mutex_unlock(&dpm_list_mtx);
 163         device_wakeup_disable(dev);
 164         pm_runtime_remove(dev);
 165         device_pm_check_callbacks(dev);
 166 }
 167
 168 /**
 169  * device_pm_move_before - Move device in the PM core's list of active devices.
 170  * @deva: Device to move in dpm_list.
 171  * @devb: Device @deva should come before.
 172  */
 173 void device_pm_move_before(struct device *deva, struct device *devb)
 174 {
 175         pr_debug("Moving %s:%s before %s:%s\n",
 176                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 177                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 178         /* Delete deva from dpm_list and reinsert before devb. */
 179         list_move_tail(&deva->power.entry, &devb->power.entry);
 180 }
 181
 182 /**
 183  * device_pm_move_after - Move device in the PM core's list of active devices.
 184  * @deva: Device to move in dpm_list.
 185  * @devb: Device @deva should come after.
 186  */
 187 void device_pm_move_after(struct device *deva, struct device *devb)
 188 {
 189         pr_debug("Moving %s:%s after %s:%s\n",
 190                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 191                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 192         /* Delete deva from dpm_list and reinsert after devb. */
 193         list_move(&deva->power.entry, &devb->power.entry);
 194 }
 195
 196 /**
 197  * device_pm_move_last - Move device to end of the PM core's list of devices.
 198  * @dev: Device to move in dpm_list.
 199  */
 200 void device_pm_move_last(struct device *dev)
 201 {
 202         pr_debug("Moving %s:%s to end of list\n",
 203                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 204         list_move_tail(&dev->power.entry, &dpm_list);
 205 }
 206
 207 static ktime_t initcall_debug_start(struct device *dev, void *cb)
 208 {
 209         if (!pm_print_times_enabled)
 210                 return 0;
 211
 212         dev_info(dev, "calling %pS @ %i, parent: %s\n", cb,
 213                  task_pid_nr(current),
 214                  dev->parent ? dev_name(dev->parent) : "none");
 215         return ktime_get();
 216 }
 217
 218 static void initcall_debug_report(struct device *dev, ktime_t calltime,
 219                                   void *cb, int error)
 220 {
 221         ktime_t rettime;
 222
 223         if (!pm_print_times_enabled)
 224                 return;
 225
 226         rettime = ktime_get();
 227         dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error,
 228                  (unsigned long long)ktime_us_delta(rettime, calltime));
 229 }
 230
 231 /**
 232  * dpm_wait - Wait for a PM operation to complete.
 233  * @dev: Device to wait for.
 234  * @async: If unset, wait only if the device's power.async_suspend flag is set.
 235  */
 236 static void dpm_wait(struct device *dev, bool async)
 237 {
 238         if (!dev)
 239                 return;
 240
 241         if (async || (pm_async_enabled && dev->power.async_suspend))
 242                 wait_for_completion(&dev->power.completion);
 243 }
 244
 245 static int dpm_wait_fn(struct device *dev, void *async_ptr)
 246 {
 247         dpm_wait(dev, *((bool *)async_ptr));
 248         return 0;
 249 }
 250
 251 static void dpm_wait_for_children(struct device *dev, bool async)
 252 {
 253        device_for_each_child(dev, &async, dpm_wait_fn);
 254 }
 255
 256 static void dpm_wait_for_suppliers(struct device *dev, bool async)
 257 {
 258         struct device_link *link;
 259         int idx;
 260
 261         idx = device_links_read_lock();
 262
 263         /*
 264          * If the supplier goes away right after we've checked the link to it,
 265          * we'll wait for its completion to change the state, but that's fine,
 266          * because the only things that will block as a result are the SRCU
 267          * callbacks freeing the link objects for the links in the list we're
 268          * walking.
 269          */
 270         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
 271                 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
 272                         dpm_wait(link->supplier, async);
 273
 274         device_links_read_unlock(idx);
 275 }
 276
 277 static bool dpm_wait_for_superior(struct device *dev, bool async)
 278 {
 279         struct device *parent;
 280
 281         /*
 282          * If the device is resumed asynchronously and the parent's callback
 283          * deletes both the device and the parent itself, the parent object may
 284          * be freed while this function is running, so avoid that by reference
 285          * counting the parent once more unless the device has been deleted
 286          * already (in which case return right away).
 287          */
 288         mutex_lock(&dpm_list_mtx);
 289
 290         if (!device_pm_initialized(dev)) {
 291                 mutex_unlock(&dpm_list_mtx);
 292                 return false;
 293         }
 294
 295         parent = get_device(dev->parent);
 296
 297         mutex_unlock(&dpm_list_mtx);
 298
 299         dpm_wait(parent, async);
 300         put_device(parent);
 301
 302         dpm_wait_for_suppliers(dev, async);
 303
 304         /*
 305          * If the parent's callback has deleted the device, attempting to resume
 306          * it would be invalid, so avoid doing that then.
 307          */
 308         return device_pm_initialized(dev);
 309 }
 310
 311 static void dpm_wait_for_consumers(struct device *dev, bool async)
 312 {
 313         struct device_link *link;
 314         int idx;
 315
 316         idx = device_links_read_lock();
 317
 318         /*
 319          * The status of a device link can only be changed from "dormant" by a
 320          * probe, but that cannot happen during system suspend/resume.  In
 321          * theory it can change to "dormant" at that time, but then it is
 322          * reasonable to wait for the target device anyway (eg. if it goes
 323          * away, it's better to wait for it to go away completely and then
 324          * continue instead of trying to continue in parallel with its
 325          * unregistration).
 326          */
 327         list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node)
 328                 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
 329                         dpm_wait(link->consumer, async);
 330
 331         device_links_read_unlock(idx);
 332 }
 333
 334 static void dpm_wait_for_subordinate(struct device *dev, bool async)
 335 {
 336         dpm_wait_for_children(dev, async);
 337         dpm_wait_for_consumers(dev, async);
 338 }
 339
 340 /**
 341  * pm_op - Return the PM operation appropriate for given PM event.
 342  * @ops: PM operations to choose from.
 343  * @state: PM transition of the system being carried out.
 344  */
 345 static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
 346 {
 347         switch (state.event) {
 348 #ifdef CONFIG_SUSPEND
 349         case PM_EVENT_SUSPEND:
 350                 return ops->suspend;
 351         case PM_EVENT_RESUME:
 352                 return ops->resume;
 353 #endif /* CONFIG_SUSPEND */
 354 #ifdef CONFIG_HIBERNATE_CALLBACKS
 355         case PM_EVENT_FREEZE:
 356         case PM_EVENT_QUIESCE:
 357                 return ops->freeze;
 358         case PM_EVENT_HIBERNATE:
 359                 return ops->poweroff;
 360         case PM_EVENT_THAW:
 361         case PM_EVENT_RECOVER:
 362                 return ops->thaw;
 363         case PM_EVENT_RESTORE:
 364                 return ops->restore;
 365 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 366         }
 367
 368         return NULL;
 369 }
 370
 371 /**
 372  * pm_late_early_op - Return the PM operation appropriate for given PM event.
 373  * @ops: PM operations to choose from.
 374  * @state: PM transition of the system being carried out.
 375  *
 376  * Runtime PM is disabled for @dev while this function is being executed.
 377  */
 378 static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
 379                                       pm_message_t state)
 380 {
 381         switch (state.event) {
 382 #ifdef CONFIG_SUSPEND
 383         case PM_EVENT_SUSPEND:
 384                 return ops->suspend_late;
 385         case PM_EVENT_RESUME:
 386                 return ops->resume_early;
 387 #endif /* CONFIG_SUSPEND */
 388 #ifdef CONFIG_HIBERNATE_CALLBACKS
 389         case PM_EVENT_FREEZE:
 390         case PM_EVENT_QUIESCE:
 391                 return ops->freeze_late;
 392         case PM_EVENT_HIBERNATE:
 393                 return ops->poweroff_late;
 394         case PM_EVENT_THAW:
 395         case PM_EVENT_RECOVER:
 396                 return ops->thaw_early;
 397         case PM_EVENT_RESTORE:
 398                 return ops->restore_early;
 399 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 400         }
 401
 402         return NULL;
 403 }
 404
 405 /**
 406  * pm_noirq_op - Return the PM operation appropriate for given PM event.
 407  * @ops: PM operations to choose from.
 408  * @state: PM transition of the system being carried out.
 409  *
 410  * The driver of @dev will not receive interrupts while this function is being
 411  * executed.
 412  */
 413 static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
 414 {
 415         switch (state.event) {
 416 #ifdef CONFIG_SUSPEND
 417         case PM_EVENT_SUSPEND:
 418                 return ops->suspend_noirq;
 419         case PM_EVENT_RESUME:
 420                 return ops->resume_noirq;
 421 #endif /* CONFIG_SUSPEND */
 422 #ifdef CONFIG_HIBERNATE_CALLBACKS
 423         case PM_EVENT_FREEZE:
 424         case PM_EVENT_QUIESCE:
 425                 return ops->freeze_noirq;
 426         case PM_EVENT_HIBERNATE:
 427                 return ops->poweroff_noirq;
 428         case PM_EVENT_THAW:
 429         case PM_EVENT_RECOVER:
 430                 return ops->thaw_noirq;
 431         case PM_EVENT_RESTORE:
 432                 return ops->restore_noirq;
 433 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 434         }
 435
 436         return NULL;
 437 }
 438
 439 static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
 440 {
 441         dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event),
 442                 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
 443                 ", may wakeup" : "", dev->power.driver_flags);
 444 }
 445
 446 static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
 447                         int error)
 448 {
 449         dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info,
 450                 error);
 451 }
 452
 453 static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
 454                           const char *info)
 455 {
 456         ktime_t calltime;
 457         u64 usecs64;
 458         int usecs;
 459
 460         calltime = ktime_get();
 461         usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
 462         do_div(usecs64, NSEC_PER_USEC);
 463         usecs = usecs64;
 464         if (usecs == 0)
 465                 usecs = 1;
 466
 467         pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
 468                   info ?: "", info ? " " : "", pm_verb(state.event),
 469                   error ? "aborted" : "complete",
 470                   usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
 471 }
 472
 473 static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 474                             pm_message_t state, const char *info)
 475 {
 476         ktime_t calltime;
 477         int error;
 478
 479         if (!cb)
 480                 return 0;
 481
 482         calltime = initcall_debug_start(dev, cb);
 483
 484         pm_dev_dbg(dev, state, info);
 485         trace_device_pm_callback_start(dev, info, state.event);
 486         error = cb(dev);
 487         trace_device_pm_callback_end(dev, error);
 488         suspend_report_result(cb, error);
 489
 490         initcall_debug_report(dev, calltime, cb, error);
 491
 492         return error;
 493 }
 494
 495 #ifdef CONFIG_DPM_WATCHDOG
 496 struct dpm_watchdog {
 497         struct device           *dev;
 498         struct task_struct      *tsk;
 499         struct timer_list       timer;
 500 };
 501
 502 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
 503         struct dpm_watchdog wd
 504
 505 /**
 506  * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 507  * @t: The timer that PM watchdog depends on.
 508  *
 509  * Called when a driver has timed out suspending or resuming.
 510  * There's not much we can do here to recover so panic() to
 511  * capture a crash-dump in pstore.
 512  */
 513 static void dpm_watchdog_handler(struct timer_list *t)
 514 {
 515         struct dpm_watchdog *wd = from_timer(wd, t, timer);
 516
 517         dev_emerg(wd->dev, "**** DPM device timeout ****\n");
 518         show_stack(wd->tsk, NULL, KERN_EMERG);
 519         panic("%s %s: unrecoverable failure\n",
 520                 dev_driver_string(wd->dev), dev_name(wd->dev));
 521 }
 522
 523 /**
 524  * dpm_watchdog_set - Enable pm watchdog for given device.
 525  * @wd: Watchdog. Must be allocated on the stack.
 526  * @dev: Device to handle.
 527  */
 528 static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
 529 {
 530         struct timer_list *timer = &wd->timer;
 531
 532         wd->dev = dev;
 533         wd->tsk = current;
 534
 535         timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
 536         /* use same timeout value for both suspend and resume */
 537         timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
 538         add_timer(timer);
 539 }
 540
 541 /**
 542  * dpm_watchdog_clear - Disable suspend/resume watchdog.
 543  * @wd: Watchdog to disable.
 544  */
 545 static void dpm_watchdog_clear(struct dpm_watchdog *wd)
 546 {
 547         struct timer_list *timer = &wd->timer;
 548
 549         del_timer_sync(timer);
 550         destroy_timer_on_stack(timer);
 551 }
 552 #else
 553 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
 554 #define dpm_watchdog_set(x, y)
 555 #define dpm_watchdog_clear(x)
 556 #endif
 557
 558 /*------------------------- Resume routines -------------------------*/
 559
 560 /**
 561  * dev_pm_skip_resume - System-wide device resume optimization check.
 562  * @dev: Target device.
 563  *
 564  * Return:
 565  * - %false if the transition under way is RESTORE.
 566  * - Return value of dev_pm_skip_suspend() if the transition under way is THAW.
 567  * - The logical negation of %power.must_resume otherwise (that is, when the
 568  *   transition under way is RESUME).
 569  */
 570 bool dev_pm_skip_resume(struct device *dev)
 571 {
 572         if (pm_transition.event == PM_EVENT_RESTORE)
 573                 return false;
 574
 575         if (pm_transition.event == PM_EVENT_THAW)
 576                 return dev_pm_skip_suspend(dev);
 577
 578         return !dev->power.must_resume;
 579 }
 580
 581 /**
 582  * device_resume_noirq - Execute a "noirq resume" callback for given device.
 583  * @dev: Device to handle.
 584  * @state: PM transition of the system being carried out.
 585  * @async: If true, the device is being resumed asynchronously.
 586  *
 587  * The driver of @dev will not receive interrupts while this function is being
 588  * executed.
 589  */
 590 static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
 591 {
 592         pm_callback_t callback = NULL;
 593         const char *info = NULL;
 594         bool skip_resume;
 595         int error = 0;
 596
 597         TRACE_DEVICE(dev);
 598         TRACE_RESUME(0);
 599
 600         if (dev->power.syscore || dev->power.direct_complete)
 601                 goto Out;
 602
 603         if (!dev->power.is_noirq_suspended)
 604                 goto Out;
 605
 606         if (!dpm_wait_for_superior(dev, async))
 607                 goto Out;
 608
 609         skip_resume = dev_pm_skip_resume(dev);
 610         /*
 611          * If the driver callback is skipped below or by the middle layer
 612          * callback and device_resume_early() also skips the driver callback for
 613          * this device later, it needs to appear as "suspended" to PM-runtime,
 614          * so change its status accordingly.
 615          *
 616          * Otherwise, the device is going to be resumed, so set its PM-runtime
 617          * status to "active", but do that only if DPM_FLAG_SMART_SUSPEND is set
 618          * to avoid confusing drivers that don't use it.
 619          */
 620         if (skip_resume)
 621                 pm_runtime_set_suspended(dev);
 622         else if (dev_pm_skip_suspend(dev))
 623                 pm_runtime_set_active(dev);
 624
 625         if (dev->pm_domain) {
 626                 info = "noirq power domain ";
 627                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
 628         } else if (dev->type && dev->type->pm) {
 629                 info = "noirq type ";
 630                 callback = pm_noirq_op(dev->type->pm, state);
 631         } else if (dev->class && dev->class->pm) {
 632                 info = "noirq class ";
 633                 callback = pm_noirq_op(dev->class->pm, state);
 634         } else if (dev->bus && dev->bus->pm) {
 635                 info = "noirq bus ";
 636                 callback = pm_noirq_op(dev->bus->pm, state);
 637         }
 638         if (callback)
 639                 goto Run;
 640
 641         if (skip_resume)
 642                 goto Skip;
 643
 644         if (dev->driver && dev->driver->pm) {
 645                 info = "noirq driver ";
 646                 callback = pm_noirq_op(dev->driver->pm, state);
 647         }
 648
 649 Run:
 650         error = dpm_run_callback(callback, dev, state, info);
 651
 652 Skip:
 653         dev->power.is_noirq_suspended = false;
 654
 655 Out:
 656         complete_all(&dev->power.completion);
 657         TRACE_RESUME(error);
 658         return error;
 659 }
 660
 661 static bool is_async(struct device *dev)
 662 {
 663         return dev->power.async_suspend && pm_async_enabled
 664                 && !pm_trace_is_enabled();
 665 }
 666
 667 static bool dpm_async_fn(struct device *dev, async_func_t func)
 668 {
 669         reinit_completion(&dev->power.completion);
 670
 671         if (is_async(dev)) {
 672                 get_device(dev);
 673                 async_schedule_dev(func, dev);
 674                 return true;
 675         }
 676
 677         return false;
 678 }
 679
 680 static void async_resume_noirq(void *data, async_cookie_t cookie)
 681 {
 682         struct device *dev = (struct device *)data;
 683         int error;
 684
 685         error = device_resume_noirq(dev, pm_transition, true);
 686         if (error)
 687                 pm_dev_err(dev, pm_transition, " async", error);
 688
 689         put_device(dev);
 690 }
 691
 692 static void dpm_noirq_resume_devices(pm_message_t state)
 693 {
 694         struct device *dev;
 695         ktime_t starttime = ktime_get();
 696
 697         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
 698         mutex_lock(&dpm_list_mtx);
 699         pm_transition = state;
 700
 701         /*
 702          * Advanced the async threads upfront,
 703          * in case the starting of async threads is
 704          * delayed by non-async resuming devices.
 705          */
 706         list_for_each_entry(dev, &dpm_noirq_list, power.entry)
 707                 dpm_async_fn(dev, async_resume_noirq);
 708
 709         while (!list_empty(&dpm_noirq_list)) {
 710                 dev = to_device(dpm_noirq_list.next);
 711                 get_device(dev);
 712                 list_move_tail(&dev->power.entry, &dpm_late_early_list);
 713                 mutex_unlock(&dpm_list_mtx);
 714
 715                 if (!is_async(dev)) {
 716                         int error;
 717
 718                         error = device_resume_noirq(dev, state, false);
 719                         if (error) {
 720                                 suspend_stats.failed_resume_noirq++;
 721                                 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
 722                                 dpm_save_failed_dev(dev_name(dev));
 723                                 pm_dev_err(dev, state, " noirq", error);
 724                         }
 725                 }
 726
 727                 mutex_lock(&dpm_list_mtx);
 728                 put_device(dev);
 729         }
 730         mutex_unlock(&dpm_list_mtx);
 731         async_synchronize_full();
 732         dpm_show_time(starttime, state, 0, "noirq");
 733         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
 734 }
 735
 736 /**
 737  * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 738  * @state: PM transition of the system being carried out.
 739  *
 740  * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
 741  * allow device drivers' interrupt handlers to be called.
 742  */
 743 void dpm_resume_noirq(pm_message_t state)
 744 {
 745         dpm_noirq_resume_devices(state);
 746
 747         resume_device_irqs();
 748         device_wakeup_disarm_wake_irqs();
 749 }
 750
 751 /**
 752  * device_resume_early - Execute an "early resume" callback for given device.
 753  * @dev: Device to handle.
 754  * @state: PM transition of the system being carried out.
 755  * @async: If true, the device is being resumed asynchronously.
 756  *
 757  * Runtime PM is disabled for @dev while this function is being executed.
 758  */
 759 static int device_resume_early(struct device *dev, pm_message_t state, bool async)
 760 {
 761         pm_callback_t callback = NULL;
 762         const char *info = NULL;
 763         int error = 0;
 764
 765         TRACE_DEVICE(dev);
 766         TRACE_RESUME(0);
 767
 768         if (dev->power.syscore || dev->power.direct_complete)
 769                 goto Out;
 770
 771         if (!dev->power.is_late_suspended)
 772                 goto Out;
 773
 774         if (!dpm_wait_for_superior(dev, async))
 775                 goto Out;
 776
 777         if (dev->pm_domain) {
 778                 info = "early power domain ";
 779                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
 780         } else if (dev->type && dev->type->pm) {
 781                 info = "early type ";
 782                 callback = pm_late_early_op(dev->type->pm, state);
 783         } else if (dev->class && dev->class->pm) {
 784                 info = "early class ";
 785                 callback = pm_late_early_op(dev->class->pm, state);
 786         } else if (dev->bus && dev->bus->pm) {
 787                 info = "early bus ";
 788                 callback = pm_late_early_op(dev->bus->pm, state);
 789         }
 790         if (callback)
 791                 goto Run;
 792
 793         if (dev_pm_skip_resume(dev))
 794                 goto Skip;
 795
 796         if (dev->driver && dev->driver->pm) {
 797                 info = "early driver ";
 798                 callback = pm_late_early_op(dev->driver->pm, state);
 799         }
 800
 801 Run:
 802         error = dpm_run_callback(callback, dev, state, info);
 803
 804 Skip:
 805         dev->power.is_late_suspended = false;
 806
 807 Out:
 808         TRACE_RESUME(error);
 809
 810         pm_runtime_enable(dev);
 811         complete_all(&dev->power.completion);
 812         return error;
 813 }
 814
 815 static void async_resume_early(void *data, async_cookie_t cookie)
 816 {
 817         struct device *dev = (struct device *)data;
 818         int error;
 819
 820         error = device_resume_early(dev, pm_transition, true);
 821         if (error)
 822                 pm_dev_err(dev, pm_transition, " async", error);
 823
 824         put_device(dev);
 825 }
 826
 827 /**
 828  * dpm_resume_early - Execute "early resume" callbacks for all devices.
 829  * @state: PM transition of the system being carried out.
 830  */
 831 void dpm_resume_early(pm_message_t state)
 832 {
 833         struct device *dev;
 834         ktime_t starttime = ktime_get();
 835
 836         trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
 837         mutex_lock(&dpm_list_mtx);
 838         pm_transition = state;
 839
 840         /*
 841          * Advanced the async threads upfront,
 842          * in case the starting of async threads is
 843          * delayed by non-async resuming devices.
 844          */
 845         list_for_each_entry(dev, &dpm_late_early_list, power.entry)
 846                 dpm_async_fn(dev, async_resume_early);
 847
 848         while (!list_empty(&dpm_late_early_list)) {
 849                 dev = to_device(dpm_late_early_list.next);
 850                 get_device(dev);
 851                 list_move_tail(&dev->power.entry, &dpm_suspended_list);
 852                 mutex_unlock(&dpm_list_mtx);
 853
 854                 if (!is_async(dev)) {
 855                         int error;
 856
 857                         error = device_resume_early(dev, state, false);
 858                         if (error) {
 859                                 suspend_stats.failed_resume_early++;
 860                                 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
 861                                 dpm_save_failed_dev(dev_name(dev));
 862                                 pm_dev_err(dev, state, " early", error);
 863                         }
 864                 }
 865                 mutex_lock(&dpm_list_mtx);
 866                 put_device(dev);
 867         }
 868         mutex_unlock(&dpm_list_mtx);
 869         async_synchronize_full();
 870         dpm_show_time(starttime, state, 0, "early");
 871         trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
 872 }
 873
 874 /**
 875  * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 876  * @state: PM transition of the system being carried out.
 877  */
 878 void dpm_resume_start(pm_message_t state)
 879 {
 880         dpm_resume_noirq(state);
 881         dpm_resume_early(state);
 882 }
 883 EXPORT_SYMBOL_GPL(dpm_resume_start);
 884
 885 /**
 886  * device_resume - Execute "resume" callbacks for given device.
 887  * @dev: Device to handle.
 888  * @state: PM transition of the system being carried out.
 889  * @async: If true, the device is being resumed asynchronously.
 890  */
 891 static int device_resume(struct device *dev, pm_message_t state, bool async)
 892 {
 893         pm_callback_t callback = NULL;
 894         const char *info = NULL;
 895         int error = 0;
 896         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 897
 898         TRACE_DEVICE(dev);
 899         TRACE_RESUME(0);
 900
 901         if (dev->power.syscore)
 902                 goto Complete;
 903
 904         if (dev->power.direct_complete) {
 905                 /* Match the pm_runtime_disable() in __device_suspend(). */
 906                 pm_runtime_enable(dev);
 907                 goto Complete;
 908         }
 909
 910         if (!dpm_wait_for_superior(dev, async))
 911                 goto Complete;
 912
 913         dpm_watchdog_set(&wd, dev);
 914         device_lock(dev);
 915
 916         /*
 917          * This is a fib.  But we'll allow new children to be added below
 918          * a resumed device, even if the device hasn't been completed yet.
 919          */
 920         dev->power.is_prepared = false;
 921
 922         if (!dev->power.is_suspended)
 923                 goto Unlock;
 924
 925         if (dev->pm_domain) {
 926                 info = "power domain ";
 927                 callback = pm_op(&dev->pm_domain->ops, state);
 928                 goto Driver;
 929         }
 930
 931         if (dev->type && dev->type->pm) {
 932                 info = "type ";
 933                 callback = pm_op(dev->type->pm, state);
 934                 goto Driver;
 935         }
 936
 937         if (dev->class && dev->class->pm) {
 938                 info = "class ";
 939                 callback = pm_op(dev->class->pm, state);
 940                 goto Driver;
 941         }
 942
 943         if (dev->bus) {
 944                 if (dev->bus->pm) {
 945                         info = "bus ";
 946                         callback = pm_op(dev->bus->pm, state);
 947                 } else if (dev->bus->resume) {
 948                         info = "legacy bus ";
 949                         callback = dev->bus->resume;
 950                         goto End;
 951                 }
 952         }
 953
 954  Driver:
 955         if (!callback && dev->driver && dev->driver->pm) {
 956                 info = "driver ";
 957                 callback = pm_op(dev->driver->pm, state);
 958         }
 959
 960  End:
 961         error = dpm_run_callback(callback, dev, state, info);
 962         dev->power.is_suspended = false;
 963
 964  Unlock:
 965         device_unlock(dev);
 966         dpm_watchdog_clear(&wd);
 967
 968  Complete:
 969         complete_all(&dev->power.completion);
 970
 971         TRACE_RESUME(error);
 972
 973         return error;
 974 }
 975
 976 static void async_resume(void *data, async_cookie_t cookie)
 977 {
 978         struct device *dev = (struct device *)data;
 979         int error;
 980
 981         error = device_resume(dev, pm_transition, true);
 982         if (error)
 983                 pm_dev_err(dev, pm_transition, " async", error);
 984         put_device(dev);
 985 }
 986
 987 /**
 988  * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
 989  * @state: PM transition of the system being carried out.
 990  *
 991  * Execute the appropriate "resume" callback for all devices whose status
 992  * indicates that they are suspended.
 993  */
 994 void dpm_resume(pm_message_t state)
 995 {
 996         struct device *dev;
 997         ktime_t starttime = ktime_get();
 998
 999         trace_suspend_resume(TPS("dpm_resume"), state.event, true);
1000         might_sleep();
1001
1002         mutex_lock(&dpm_list_mtx);
1003         pm_transition = state;
1004         async_error = 0;
1005
1006         list_for_each_entry(dev, &dpm_suspended_list, power.entry)
1007                 dpm_async_fn(dev, async_resume);
1008
1009         while (!list_empty(&dpm_suspended_list)) {
1010                 dev = to_device(dpm_suspended_list.next);
1011                 get_device(dev);
1012                 if (!is_async(dev)) {
1013                         int error;
1014
1015                         mutex_unlock(&dpm_list_mtx);
1016
1017                         error = device_resume(dev, state, false);
1018                         if (error) {
1019                                 suspend_stats.failed_resume++;
1020                                 dpm_save_failed_step(SUSPEND_RESUME);
1021                                 dpm_save_failed_dev(dev_name(dev));
1022                                 pm_dev_err(dev, state, "", error);
1023                         }
1024
1025                         mutex_lock(&dpm_list_mtx);
1026                 }
1027                 if (!list_empty(&dev->power.entry))
1028                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1029                 put_device(dev);
1030         }
1031         mutex_unlock(&dpm_list_mtx);
1032         async_synchronize_full();
1033         dpm_show_time(starttime, state, 0, NULL);
1034
1035         cpufreq_resume();
1036         devfreq_resume();
1037         trace_suspend_resume(TPS("dpm_resume"), state.event, false);
1038 }
1039
1040 /**
1041  * device_complete - Complete a PM transition for given device.
1042  * @dev: Device to handle.
1043  * @state: PM transition of the system being carried out.
1044  */
1045 static void device_complete(struct device *dev, pm_message_t state)
1046 {
1047         void (*callback)(struct device *) = NULL;
1048         const char *info = NULL;
1049
1050         if (dev->power.syscore)
1051                 goto out;
1052
1053         device_lock(dev);
1054
1055         if (dev->pm_domain) {
1056                 info = "completing power domain ";
1057                 callback = dev->pm_domain->ops.complete;
1058         } else if (dev->type && dev->type->pm) {
1059                 info = "completing type ";
1060                 callback = dev->type->pm->complete;
1061         } else if (dev->class && dev->class->pm) {
1062                 info = "completing class ";
1063                 callback = dev->class->pm->complete;
1064         } else if (dev->bus && dev->bus->pm) {
1065                 info = "completing bus ";
1066                 callback = dev->bus->pm->complete;
1067         }
1068
1069         if (!callback && dev->driver && dev->driver->pm) {
1070                 info = "completing driver ";
1071                 callback = dev->driver->pm->complete;
1072         }
1073
1074         if (callback) {
1075                 pm_dev_dbg(dev, state, info);
1076                 callback(dev);
1077         }
1078
1079         device_unlock(dev);
1080
1081 out:
1082         pm_runtime_put(dev);
1083 }
1084
1085 /**
1086  * dpm_complete - Complete a PM transition for all non-sysdev devices.
1087  * @state: PM transition of the system being carried out.
1088  *
1089  * Execute the ->complete() callbacks for all devices whose PM status is not
1090  * DPM_ON (this allows new devices to be registered).
1091  */
1092 void dpm_complete(pm_message_t state)
1093 {
1094         struct list_head list;
1095
1096         trace_suspend_resume(TPS("dpm_complete"), state.event, true);
1097         might_sleep();
1098
1099         INIT_LIST_HEAD(&list);
1100         mutex_lock(&dpm_list_mtx);
1101         while (!list_empty(&dpm_prepared_list)) {
1102                 struct device *dev = to_device(dpm_prepared_list.prev);
1103
1104                 get_device(dev);
1105                 dev->power.is_prepared = false;
1106                 list_move(&dev->power.entry, &list);
1107                 mutex_unlock(&dpm_list_mtx);
1108
1109                 trace_device_pm_callback_start(dev, "", state.event);
1110                 device_complete(dev, state);
1111                 trace_device_pm_callback_end(dev, 0);
1112
1113                 mutex_lock(&dpm_list_mtx);
1114                 put_device(dev);
1115         }
1116         list_splice(&list, &dpm_list);
1117         mutex_unlock(&dpm_list_mtx);
1118
1119         /* Allow device probing and trigger re-probing of deferred devices */
1120         device_unblock_probing();
1121         trace_suspend_resume(TPS("dpm_complete"), state.event, false);
1122 }
1123
1124 /**
1125  * dpm_resume_end - Execute "resume" callbacks and complete system transition.
1126  * @state: PM transition of the system being carried out.
1127  *
1128  * Execute "resume" callbacks for all devices and complete the PM transition of
1129  * the system.
1130  */
1131 void dpm_resume_end(pm_message_t state)
1132 {
1133         dpm_resume(state);
1134         dpm_complete(state);
1135 }
1136 EXPORT_SYMBOL_GPL(dpm_resume_end);
1137
1138
1139 /*------------------------- Suspend routines -------------------------*/
1140
1141 /**
1142  * resume_event - Return a "resume" message for given "suspend" sleep state.
1143  * @sleep_state: PM message representing a sleep state.
1144  *
1145  * Return a PM message representing the resume event corresponding to given
1146  * sleep state.
1147  */
1148 static pm_message_t resume_event(pm_message_t sleep_state)
1149 {
1150         switch (sleep_state.event) {
1151         case PM_EVENT_SUSPEND:
1152                 return PMSG_RESUME;
1153         case PM_EVENT_FREEZE:
1154         case PM_EVENT_QUIESCE:
1155                 return PMSG_RECOVER;
1156         case PM_EVENT_HIBERNATE:
1157                 return PMSG_RESTORE;
1158         }
1159         return PMSG_ON;
1160 }
1161
1162 static void dpm_superior_set_must_resume(struct device *dev)
1163 {
1164         struct device_link *link;
1165         int idx;
1166
1167         if (dev->parent)
1168                 dev->parent->power.must_resume = true;
1169
1170         idx = device_links_read_lock();
1171
1172         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
1173                 link->supplier->power.must_resume = true;
1174
1175         device_links_read_unlock(idx);
1176 }
1177
1178 /**
1179  * __device_suspend_noirq - Execute a "noirq suspend" callback for given device.
1180  * @dev: Device to handle.
1181  * @state: PM transition of the system being carried out.
1182  * @async: If true, the device is being suspended asynchronously.
1183  *
1184  * The driver of @dev will not receive interrupts while this function is being
1185  * executed.
1186  */
1187 static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1188 {
1189         pm_callback_t callback = NULL;
1190         const char *info = NULL;
1191         int error = 0;
1192
1193         TRACE_DEVICE(dev);
1194         TRACE_SUSPEND(0);
1195
1196         dpm_wait_for_subordinate(dev, async);
1197
1198         if (async_error)
1199                 goto Complete;
1200
1201         if (dev->power.syscore || dev->power.direct_complete)
1202                 goto Complete;
1203
1204         if (dev->pm_domain) {
1205                 info = "noirq power domain ";
1206                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1207         } else if (dev->type && dev->type->pm) {
1208                 info = "noirq type ";
1209                 callback = pm_noirq_op(dev->type->pm, state);
1210         } else if (dev->class && dev->class->pm) {
1211                 info = "noirq class ";
1212                 callback = pm_noirq_op(dev->class->pm, state);
1213         } else if (dev->bus && dev->bus->pm) {
1214                 info = "noirq bus ";
1215                 callback = pm_noirq_op(dev->bus->pm, state);
1216         }
1217         if (callback)
1218                 goto Run;
1219
1220         if (dev_pm_skip_suspend(dev))
1221                 goto Skip;
1222
1223         if (dev->driver && dev->driver->pm) {
1224                 info = "noirq driver ";
1225                 callback = pm_noirq_op(dev->driver->pm, state);
1226         }
1227
1228 Run:
1229         error = dpm_run_callback(callback, dev, state, info);
1230         if (error) {
1231                 async_error = error;
1232                 goto Complete;
1233         }
1234
1235 Skip:
1236         dev->power.is_noirq_suspended = true;
1237
1238         /*
1239          * Skipping the resume of devices that were in use right before the
1240          * system suspend (as indicated by their PM-runtime usage counters)
1241          * would be suboptimal.  Also resume them if doing that is not allowed
1242          * to be skipped.
1243          */
1244         if (atomic_read(&dev->power.usage_count) > 1 ||
1245             !(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) &&
1246               dev->power.may_skip_resume))
1247                 dev->power.must_resume = true;
1248
1249         if (dev->power.must_resume)
1250                 dpm_superior_set_must_resume(dev);
1251
1252 Complete:
1253         complete_all(&dev->power.completion);
1254         TRACE_SUSPEND(error);
1255         return error;
1256 }
1257
1258 static void async_suspend_noirq(void *data, async_cookie_t cookie)
1259 {
1260         struct device *dev = (struct device *)data;
1261         int error;
1262
1263         error = __device_suspend_noirq(dev, pm_transition, true);
1264         if (error) {
1265                 dpm_save_failed_dev(dev_name(dev));
1266                 pm_dev_err(dev, pm_transition, " async", error);
1267         }
1268
1269         put_device(dev);
1270 }
1271
1272 static int device_suspend_noirq(struct device *dev)
1273 {
1274         if (dpm_async_fn(dev, async_suspend_noirq))
1275                 return 0;
1276
1277         return __device_suspend_noirq(dev, pm_transition, false);
1278 }
1279
1280 static int dpm_noirq_suspend_devices(pm_message_t state)
1281 {
1282         ktime_t starttime = ktime_get();
1283         int error = 0;
1284
1285         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1286         mutex_lock(&dpm_list_mtx);
1287         pm_transition = state;
1288         async_error = 0;
1289
1290         while (!list_empty(&dpm_late_early_list)) {
1291                 struct device *dev = to_device(dpm_late_early_list.prev);
1292
1293                 get_device(dev);
1294                 mutex_unlock(&dpm_list_mtx);
1295
1296                 error = device_suspend_noirq(dev);
1297
1298                 mutex_lock(&dpm_list_mtx);
1299                 if (error) {
1300                         pm_dev_err(dev, state, " noirq", error);
1301                         dpm_save_failed_dev(dev_name(dev));
1302                         put_device(dev);
1303                         break;
1304                 }
1305                 if (!list_empty(&dev->power.entry))
1306                         list_move(&dev->power.entry, &dpm_noirq_list);
1307                 put_device(dev);
1308
1309                 if (async_error)
1310                         break;
1311         }
1312         mutex_unlock(&dpm_list_mtx);
1313         async_synchronize_full();
1314         if (!error)
1315                 error = async_error;
1316
1317         if (error) {
1318                 suspend_stats.failed_suspend_noirq++;
1319                 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1320         }
1321         dpm_show_time(starttime, state, error, "noirq");
1322         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1323         return error;
1324 }
1325
1326 /**
1327  * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1328  * @state: PM transition of the system being carried out.
1329  *
1330  * Prevent device drivers' interrupt handlers from being called and invoke
1331  * "noirq" suspend callbacks for all non-sysdev devices.
1332  */
1333 int dpm_suspend_noirq(pm_message_t state)
1334 {
1335         int ret;
1336
1337         device_wakeup_arm_wake_irqs();
1338         suspend_device_irqs();
1339
1340         ret = dpm_noirq_suspend_devices(state);
1341         if (ret)
1342                 dpm_resume_noirq(resume_event(state));
1343
1344         return ret;
1345 }
1346
1347 static void dpm_propagate_wakeup_to_parent(struct device *dev)
1348 {
1349         struct device *parent = dev->parent;
1350
1351         if (!parent)
1352                 return;
1353
1354         spin_lock_irq(&parent->power.lock);
1355
1356         if (device_wakeup_path(dev) && !parent->power.ignore_children)
1357                 parent->power.wakeup_path = true;
1358
1359         spin_unlock_irq(&parent->power.lock);
1360 }
1361
1362 /**
1363  * __device_suspend_late - Execute a "late suspend" callback for given device.
1364  * @dev: Device to handle.
1365  * @state: PM transition of the system being carried out.
1366  * @async: If true, the device is being suspended asynchronously.
1367  *
1368  * Runtime PM is disabled for @dev while this function is being executed.
1369  */
1370 static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
1371 {
1372         pm_callback_t callback = NULL;
1373         const char *info = NULL;
1374         int error = 0;
1375
1376         TRACE_DEVICE(dev);
1377         TRACE_SUSPEND(0);
1378
1379         __pm_runtime_disable(dev, false);
1380
1381         dpm_wait_for_subordinate(dev, async);
1382
1383         if (async_error)
1384                 goto Complete;
1385
1386         if (pm_wakeup_pending()) {
1387                 async_error = -EBUSY;
1388                 goto Complete;
1389         }
1390
1391         if (dev->power.syscore || dev->power.direct_complete)
1392                 goto Complete;
1393
1394         if (dev->pm_domain) {
1395                 info = "late power domain ";
1396                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1397         } else if (dev->type && dev->type->pm) {
1398                 info = "late type ";
1399                 callback = pm_late_early_op(dev->type->pm, state);
1400         } else if (dev->class && dev->class->pm) {
1401                 info = "late class ";
1402                 callback = pm_late_early_op(dev->class->pm, state);
1403         } else if (dev->bus && dev->bus->pm) {
1404                 info = "late bus ";
1405                 callback = pm_late_early_op(dev->bus->pm, state);
1406         }
1407         if (callback)
1408                 goto Run;
1409
1410         if (dev_pm_skip_suspend(dev))
1411                 goto Skip;
1412
1413         if (dev->driver && dev->driver->pm) {
1414                 info = "late driver ";
1415                 callback = pm_late_early_op(dev->driver->pm, state);
1416         }
1417
1418 Run:
1419         error = dpm_run_callback(callback, dev, state, info);
1420         if (error) {
1421                 async_error = error;
1422                 goto Complete;
1423         }
1424         dpm_propagate_wakeup_to_parent(dev);
1425
1426 Skip:
1427         dev->power.is_late_suspended = true;
1428
1429 Complete:
1430         TRACE_SUSPEND(error);
1431         complete_all(&dev->power.completion);
1432         return error;
1433 }
1434
1435 static void async_suspend_late(void *data, async_cookie_t cookie)
1436 {
1437         struct device *dev = (struct device *)data;
1438         int error;
1439
1440         error = __device_suspend_late(dev, pm_transition, true);
1441         if (error) {
1442                 dpm_save_failed_dev(dev_name(dev));
1443                 pm_dev_err(dev, pm_transition, " async", error);
1444         }
1445         put_device(dev);
1446 }
1447
1448 static int device_suspend_late(struct device *dev)
1449 {
1450         if (dpm_async_fn(dev, async_suspend_late))
1451                 return 0;
1452
1453         return __device_suspend_late(dev, pm_transition, false);
1454 }
1455
1456 /**
1457  * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1458  * @state: PM transition of the system being carried out.
1459  */
1460 int dpm_suspend_late(pm_message_t state)
1461 {
1462         ktime_t starttime = ktime_get();
1463         int error = 0;
1464
1465         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1466         mutex_lock(&dpm_list_mtx);
1467         pm_transition = state;
1468         async_error = 0;
1469
1470         while (!list_empty(&dpm_suspended_list)) {
1471                 struct device *dev = to_device(dpm_suspended_list.prev);
1472
1473                 get_device(dev);
1474                 mutex_unlock(&dpm_list_mtx);
1475
1476                 error = device_suspend_late(dev);
1477
1478                 mutex_lock(&dpm_list_mtx);
1479                 if (!list_empty(&dev->power.entry))
1480                         list_move(&dev->power.entry, &dpm_late_early_list);
1481
1482                 if (error) {
1483                         pm_dev_err(dev, state, " late", error);
1484                         dpm_save_failed_dev(dev_name(dev));
1485                         put_device(dev);
1486                         break;
1487                 }
1488                 put_device(dev);
1489
1490                 if (async_error)
1491                         break;
1492         }
1493         mutex_unlock(&dpm_list_mtx);
1494         async_synchronize_full();
1495         if (!error)
1496                 error = async_error;
1497         if (error) {
1498                 suspend_stats.failed_suspend_late++;
1499                 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1500                 dpm_resume_early(resume_event(state));
1501         }
1502         dpm_show_time(starttime, state, error, "late");
1503         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1504         return error;
1505 }
1506
1507 /**
1508  * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1509  * @state: PM transition of the system being carried out.
1510  */
1511 int dpm_suspend_end(pm_message_t state)
1512 {
1513         ktime_t starttime = ktime_get();
1514         int error;
1515
1516         error = dpm_suspend_late(state);
1517         if (error)
1518                 goto out;
1519
1520         error = dpm_suspend_noirq(state);
1521         if (error)
1522                 dpm_resume_early(resume_event(state));
1523
1524 out:
1525         dpm_show_time(starttime, state, error, "end");
1526         return error;
1527 }
1528 EXPORT_SYMBOL_GPL(dpm_suspend_end);
1529
1530 /**
1531  * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1532  * @dev: Device to suspend.
1533  * @state: PM transition of the system being carried out.
1534  * @cb: Suspend callback to execute.
1535  * @info: string description of caller.
1536  */
1537 static int legacy_suspend(struct device *dev, pm_message_t state,
1538                           int (*cb)(struct device *dev, pm_message_t state),
1539                           const char *info)
1540 {
1541         int error;
1542         ktime_t calltime;
1543
1544         calltime = initcall_debug_start(dev, cb);
1545
1546         trace_device_pm_callback_start(dev, info, state.event);
1547         error = cb(dev, state);
1548         trace_device_pm_callback_end(dev, error);
1549         suspend_report_result(cb, error);
1550
1551         initcall_debug_report(dev, calltime, cb, error);
1552
1553         return error;
1554 }
1555
1556 static void dpm_clear_superiors_direct_complete(struct device *dev)
1557 {
1558         struct device_link *link;
1559         int idx;
1560
1561         if (dev->parent) {
1562                 spin_lock_irq(&dev->parent->power.lock);
1563                 dev->parent->power.direct_complete = false;
1564                 spin_unlock_irq(&dev->parent->power.lock);
1565         }
1566
1567         idx = device_links_read_lock();
1568
1569         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) {
1570                 spin_lock_irq(&link->supplier->power.lock);
1571                 link->supplier->power.direct_complete = false;
1572                 spin_unlock_irq(&link->supplier->power.lock);
1573         }
1574
1575         device_links_read_unlock(idx);
1576 }
1577
1578 /**
1579  * __device_suspend - Execute "suspend" callbacks for given device.
1580  * @dev: Device to handle.
1581  * @state: PM transition of the system being carried out.
1582  * @async: If true, the device is being suspended asynchronously.
1583  */
1584 static int __device_suspend(struct device *dev, pm_message_t state, bool async)
1585 {
1586         pm_callback_t callback = NULL;
1587         const char *info = NULL;
1588         int error = 0;
1589         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1590
1591         TRACE_DEVICE(dev);
1592         TRACE_SUSPEND(0);
1593
1594         dpm_wait_for_subordinate(dev, async);
1595
1596         if (async_error) {
1597                 dev->power.direct_complete = false;
1598                 goto Complete;
1599         }
1600
1601         /*
1602          * Wait for possible runtime PM transitions of the device in progress
1603          * to complete and if there's a runtime resume request pending for it,
1604          * resume it before proceeding with invoking the system-wide suspend
1605          * callbacks for it.
1606          *
1607          * If the system-wide suspend callbacks below change the configuration
1608          * of the device, they must disable runtime PM for it or otherwise
1609          * ensure that its runtime-resume callbacks will not be confused by that
1610          * change in case they are invoked going forward.
1611          */
1612         pm_runtime_barrier(dev);
1613
1614         if (pm_wakeup_pending()) {
1615                 dev->power.direct_complete = false;
1616                 async_error = -EBUSY;
1617                 goto Complete;
1618         }
1619
1620         if (dev->power.syscore)
1621                 goto Complete;
1622
1623         /* Avoid direct_complete to let wakeup_path propagate. */
1624         if (device_may_wakeup(dev) || device_wakeup_path(dev))
1625                 dev->power.direct_complete = false;
1626
1627         if (dev->power.direct_complete) {
1628                 if (pm_runtime_status_suspended(dev)) {
1629                         pm_runtime_disable(dev);
1630                         if (pm_runtime_status_suspended(dev)) {
1631                                 pm_dev_dbg(dev, state, "direct-complete ");
1632                                 goto Complete;
1633                         }
1634
1635                         pm_runtime_enable(dev);
1636                 }
1637                 dev->power.direct_complete = false;
1638         }
1639
1640         dev->power.may_skip_resume = true;
1641         dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME);
1642
1643         dpm_watchdog_set(&wd, dev);
1644         device_lock(dev);
1645
1646         if (dev->pm_domain) {
1647                 info = "power domain ";
1648                 callback = pm_op(&dev->pm_domain->ops, state);
1649                 goto Run;
1650         }
1651
1652         if (dev->type && dev->type->pm) {
1653                 info = "type ";
1654                 callback = pm_op(dev->type->pm, state);
1655                 goto Run;
1656         }
1657
1658         if (dev->class && dev->class->pm) {
1659                 info = "class ";
1660                 callback = pm_op(dev->class->pm, state);
1661                 goto Run;
1662         }
1663
1664         if (dev->bus) {
1665                 if (dev->bus->pm) {
1666                         info = "bus ";
1667                         callback = pm_op(dev->bus->pm, state);
1668                 } else if (dev->bus->suspend) {
1669                         pm_dev_dbg(dev, state, "legacy bus ");
1670                         error = legacy_suspend(dev, state, dev->bus->suspend,
1671                                                 "legacy bus ");
1672                         goto End;
1673                 }
1674         }
1675
1676  Run:
1677         if (!callback && dev->driver && dev->driver->pm) {
1678                 info = "driver ";
1679                 callback = pm_op(dev->driver->pm, state);
1680         }
1681
1682         error = dpm_run_callback(callback, dev, state, info);
1683
1684  End:
1685         if (!error) {
1686                 dev->power.is_suspended = true;
1687                 if (device_may_wakeup(dev))
1688                         dev->power.wakeup_path = true;
1689
1690                 dpm_propagate_wakeup_to_parent(dev);
1691                 dpm_clear_superiors_direct_complete(dev);
1692         }
1693
1694         device_unlock(dev);
1695         dpm_watchdog_clear(&wd);
1696
1697  Complete:
1698         if (error)
1699                 async_error = error;
1700
1701         complete_all(&dev->power.completion);
1702         TRACE_SUSPEND(error);
1703         return error;
1704 }
1705
1706 static void async_suspend(void *data, async_cookie_t cookie)
1707 {
1708         struct device *dev = (struct device *)data;
1709         int error;
1710
1711         error = __device_suspend(dev, pm_transition, true);
1712         if (error) {
1713                 dpm_save_failed_dev(dev_name(dev));
1714                 pm_dev_err(dev, pm_transition, " async", error);
1715         }
1716
1717         put_device(dev);
1718 }
1719
1720 static int device_suspend(struct device *dev)
1721 {
1722         if (dpm_async_fn(dev, async_suspend))
1723                 return 0;
1724
1725         return __device_suspend(dev, pm_transition, false);
1726 }
1727
1728 /**
1729  * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1730  * @state: PM transition of the system being carried out.
1731  */
1732 int dpm_suspend(pm_message_t state)
1733 {
1734         ktime_t starttime = ktime_get();
1735         int error = 0;
1736
1737         trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
1738         might_sleep();
1739
1740         devfreq_suspend();
1741         cpufreq_suspend();
1742
1743         mutex_lock(&dpm_list_mtx);
1744         pm_transition = state;
1745         async_error = 0;
1746         while (!list_empty(&dpm_prepared_list)) {
1747                 struct device *dev = to_device(dpm_prepared_list.prev);
1748
1749                 get_device(dev);
1750                 mutex_unlock(&dpm_list_mtx);
1751
1752                 error = device_suspend(dev);
1753
1754                 mutex_lock(&dpm_list_mtx);
1755                 if (error) {
1756                         pm_dev_err(dev, state, "", error);
1757                         dpm_save_failed_dev(dev_name(dev));
1758                         put_device(dev);
1759                         break;
1760                 }
1761                 if (!list_empty(&dev->power.entry))
1762                         list_move(&dev->power.entry, &dpm_suspended_list);
1763                 put_device(dev);
1764                 if (async_error)
1765                         break;
1766         }
1767         mutex_unlock(&dpm_list_mtx);
1768         async_synchronize_full();
1769         if (!error)
1770                 error = async_error;
1771         if (error) {
1772                 suspend_stats.failed_suspend++;
1773                 dpm_save_failed_step(SUSPEND_SUSPEND);
1774         }
1775         dpm_show_time(starttime, state, error, NULL);
1776         trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
1777         return error;
1778 }
1779
1780 /**
1781  * device_prepare - Prepare a device for system power transition.
1782  * @dev: Device to handle.
1783  * @state: PM transition of the system being carried out.
1784  *
1785  * Execute the ->prepare() callback(s) for given device.  No new children of the
1786  * device may be registered after this function has returned.
1787  */
1788 static int device_prepare(struct device *dev, pm_message_t state)
1789 {
1790         int (*callback)(struct device *) = NULL;
1791         int ret = 0;
1792
1793         /*
1794          * If a device's parent goes into runtime suspend at the wrong time,
1795          * it won't be possible to resume the device.  To prevent this we
1796          * block runtime suspend here, during the prepare phase, and allow
1797          * it again during the complete phase.
1798          */
1799         pm_runtime_get_noresume(dev);
1800
1801         if (dev->power.syscore)
1802                 return 0;
1803
1804         device_lock(dev);
1805
1806         dev->power.wakeup_path = false;
1807
1808         if (dev->power.no_pm_callbacks)
1809                 goto unlock;
1810
1811         if (dev->pm_domain)
1812                 callback = dev->pm_domain->ops.prepare;
1813         else if (dev->type && dev->type->pm)
1814                 callback = dev->type->pm->prepare;
1815         else if (dev->class && dev->class->pm)
1816                 callback = dev->class->pm->prepare;
1817         else if (dev->bus && dev->bus->pm)
1818                 callback = dev->bus->pm->prepare;
1819
1820         if (!callback && dev->driver && dev->driver->pm)
1821                 callback = dev->driver->pm->prepare;
1822
1823         if (callback)
1824                 ret = callback(dev);
1825
1826 unlock:
1827         device_unlock(dev);
1828
1829         if (ret < 0) {
1830                 suspend_report_result(callback, ret);
1831                 pm_runtime_put(dev);
1832                 return ret;
1833         }
1834         /*
1835          * A positive return value from ->prepare() means "this device appears
1836          * to be runtime-suspended and its state is fine, so if it really is
1837          * runtime-suspended, you can leave it in that state provided that you
1838          * will do the same thing with all of its descendants".  This only
1839          * applies to suspend transitions, however.
1840          */
1841         spin_lock_irq(&dev->power.lock);
1842         dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
1843                 (ret > 0 || dev->power.no_pm_callbacks) &&
1844                 !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);
1845         spin_unlock_irq(&dev->power.lock);
1846         return 0;
1847 }
1848
1849 /**
1850  * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1851  * @state: PM transition of the system being carried out.
1852  *
1853  * Execute the ->prepare() callback(s) for all devices.
1854  */
1855 int dpm_prepare(pm_message_t state)
1856 {
1857         int error = 0;
1858
1859         trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
1860         might_sleep();
1861
1862         /*
1863          * Give a chance for the known devices to complete their probes, before
1864          * disable probing of devices. This sync point is important at least
1865          * at boot time + hibernation restore.
1866          */
1867         wait_for_device_probe();
1868         /*
1869          * It is unsafe if probing of devices will happen during suspend or
1870          * hibernation and system behavior will be unpredictable in this case.
1871          * So, let's prohibit device's probing here and defer their probes
1872          * instead. The normal behavior will be restored in dpm_complete().
1873          */
1874         device_block_probing();
1875
1876         mutex_lock(&dpm_list_mtx);
1877         while (!list_empty(&dpm_list)) {
1878                 struct device *dev = to_device(dpm_list.next);
1879
1880                 get_device(dev);
1881                 mutex_unlock(&dpm_list_mtx);
1882
1883                 trace_device_pm_callback_start(dev, "", state.event);
1884                 error = device_prepare(dev, state);
1885                 trace_device_pm_callback_end(dev, error);
1886
1887                 mutex_lock(&dpm_list_mtx);
1888                 if (error) {
1889                         if (error == -EAGAIN) {
1890                                 put_device(dev);
1891                                 error = 0;
1892                                 continue;
1893                         }
1894                         dev_info(dev, "not prepared for power transition: code %d\n",
1895                                  error);
1896                         put_device(dev);
1897                         break;
1898                 }
1899                 dev->power.is_prepared = true;
1900                 if (!list_empty(&dev->power.entry))
1901                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1902                 put_device(dev);
1903         }
1904         mutex_unlock(&dpm_list_mtx);
1905         trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
1906         return error;
1907 }
1908
1909 /**
1910  * dpm_suspend_start - Prepare devices for PM transition and suspend them.
1911  * @state: PM transition of the system being carried out.
1912  *
1913  * Prepare all non-sysdev devices for system PM transition and execute "suspend"
1914  * callbacks for them.
1915  */
1916 int dpm_suspend_start(pm_message_t state)
1917 {
1918         ktime_t starttime = ktime_get();
1919         int error;
1920
1921         error = dpm_prepare(state);
1922         if (error) {
1923                 suspend_stats.failed_prepare++;
1924                 dpm_save_failed_step(SUSPEND_PREPARE);
1925         } else
1926                 error = dpm_suspend(state);
1927         dpm_show_time(starttime, state, error, "start");
1928         return error;
1929 }
1930 EXPORT_SYMBOL_GPL(dpm_suspend_start);
1931
1932 void __suspend_report_result(const char *function, void *fn, int ret)
1933 {
1934         if (ret)
1935                 pr_err("%s(): %pS returns %d\n", function, fn, ret);
1936 }
1937 EXPORT_SYMBOL_GPL(__suspend_report_result);
1938
1939 /**
1940  * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
1941  * @subordinate: Device that needs to wait for @dev.
1942  * @dev: Device to wait for.
1943  */
1944 int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
1945 {
1946         dpm_wait(dev, subordinate->power.async_suspend);
1947         return async_error;
1948 }
1949 EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
1950
1951 /**
1952  * dpm_for_each_dev - device iterator.
1953  * @data: data for the callback.
1954  * @fn: function to be called for each device.
1955  *
1956  * Iterate over devices in dpm_list, and call @fn for each device,
1957  * passing it @data.
1958  */
1959 void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
1960 {
1961         struct device *dev;
1962
1963         if (!fn)
1964                 return;
1965
1966         device_pm_lock();
1967         list_for_each_entry(dev, &dpm_list, power.entry)
1968                 fn(dev, data);
1969         device_pm_unlock();
1970 }
1971 EXPORT_SYMBOL_GPL(dpm_for_each_dev);
1972
1973 static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
1974 {
1975         if (!ops)
1976                 return true;
1977
1978         return !ops->prepare &&
1979                !ops->suspend &&
1980                !ops->suspend_late &&
1981                !ops->suspend_noirq &&
1982                !ops->resume_noirq &&
1983                !ops->resume_early &&
1984                !ops->resume &&
1985                !ops->complete;
1986 }
1987
1988 void device_pm_check_callbacks(struct device *dev)
1989 {
1990         spin_lock_irq(&dev->power.lock);
1991         dev->power.no_pm_callbacks =
1992                 (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
1993                  !dev->bus->suspend && !dev->bus->resume)) &&
1994                 (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
1995                 (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
1996                 (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
1997                 (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
1998                  !dev->driver->suspend && !dev->driver->resume));
1999         spin_unlock_irq(&dev->power.lock);
2000 }
2001
2002 bool dev_pm_skip_suspend(struct device *dev)
2003 {
2004         return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
2005                 pm_runtime_status_suspended(dev);
2006 }