drivers/base/power/main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * drivers/base/power/main.c - Where the driver meets power management.
   4  *
   5  * Copyright (c) 2003 Patrick Mochel
   6  * Copyright (c) 2003 Open Source Development Lab
   7  *
   8  * The driver model core calls device_pm_add() when a device is registered.
   9  * This will initialize the embedded device_pm_info object in the device
  10  * and add it to the list of power-controlled devices. sysfs entries for
  11  * controlling device power management will also be added.
  12  *
  13  * A separate list is used for keeping track of power info, because the power
  14  * domain dependencies may differ from the ancestral dependencies that the
  15  * subsystem list maintains.
  16  */
  17
  18 #define pr_fmt(fmt) "PM: " fmt
  19
  20 #include <linux/device.h>
  21 #include <linux/export.h>
  22 #include <linux/mutex.h>
  23 #include <linux/pm.h>
  24 #include <linux/pm_runtime.h>
  25 #include <linux/pm-trace.h>
  26 #include <linux/pm_wakeirq.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/sched.h>
  29 #include <linux/sched/debug.h>
  30 #include <linux/async.h>
  31 #include <linux/suspend.h>
  32 #include <trace/events/power.h>
  33 #include <linux/cpufreq.h>
  34 #include <linux/cpuidle.h>
  35 #include <linux/devfreq.h>
  36 #include <linux/timer.h>
  37
  38 #include "../base.h"
  39 #include "power.h"
  40
  41 typedef int (*pm_callback_t)(struct device *);
  42
  43 #define list_for_each_entry_rcu_locked(pos, head, member) \
  44         list_for_each_entry_rcu(pos, head, member, \
  45                         device_links_read_lock_held())
  46
  47 /*
  48  * The entries in the dpm_list list are in a depth first order, simply
  49  * because children are guaranteed to be discovered after parents, and
  50  * are inserted at the back of the list on discovery.
  51  *
  52  * Since device_pm_add() may be called with a device lock held,
  53  * we must never try to acquire a device lock while holding
  54  * dpm_list_mutex.
  55  */
  56
  57 LIST_HEAD(dpm_list);
  58 static LIST_HEAD(dpm_prepared_list);
  59 static LIST_HEAD(dpm_suspended_list);
  60 static LIST_HEAD(dpm_late_early_list);
  61 static LIST_HEAD(dpm_noirq_list);
  62
  63 struct suspend_stats suspend_stats;
  64 static DEFINE_MUTEX(dpm_list_mtx);
  65 static pm_message_t pm_transition;
  66
  67 static int async_error;
  68
  69 static const char *pm_verb(int event)
  70 {
  71         switch (event) {
  72         case PM_EVENT_SUSPEND:
  73                 return "suspend";
  74         case PM_EVENT_RESUME:
  75                 return "resume";
  76         case PM_EVENT_FREEZE:
  77                 return "freeze";
  78         case PM_EVENT_QUIESCE:
  79                 return "quiesce";
  80         case PM_EVENT_HIBERNATE:
  81                 return "hibernate";
  82         case PM_EVENT_THAW:
  83                 return "thaw";
  84         case PM_EVENT_RESTORE:
  85                 return "restore";
  86         case PM_EVENT_RECOVER:
  87                 return "recover";
  88         default:
  89                 return "(unknown PM event)";
  90         }
  91 }
  92
  93 /**
  94  * device_pm_sleep_init - Initialize system suspend-related device fields.
  95  * @dev: Device object being initialized.
  96  */
  97 void device_pm_sleep_init(struct device *dev)
  98 {
  99         dev->power.is_prepared = false;
 100         dev->power.is_suspended = false;
 101         dev->power.is_noirq_suspended = false;
 102         dev->power.is_late_suspended = false;
 103         init_completion(&dev->power.completion);
 104         complete_all(&dev->power.completion);
 105         dev->power.wakeup = NULL;
 106         INIT_LIST_HEAD(&dev->power.entry);
 107 }
 108
 109 /**
 110  * device_pm_lock - Lock the list of active devices used by the PM core.
 111  */
 112 void device_pm_lock(void)
 113 {
 114         mutex_lock(&dpm_list_mtx);
 115 }
 116
 117 /**
 118  * device_pm_unlock - Unlock the list of active devices used by the PM core.
 119  */
 120 void device_pm_unlock(void)
 121 {
 122         mutex_unlock(&dpm_list_mtx);
 123 }
 124
 125 /**
 126  * device_pm_add - Add a device to the PM core's list of active devices.
 127  * @dev: Device to add to the list.
 128  */
 129 void device_pm_add(struct device *dev)
 130 {
 131         /* Skip PM setup/initialization. */
 132         if (device_pm_not_required(dev))
 133                 return;
 134
 135         pr_debug("Adding info for %s:%s\n",
 136                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 137         device_pm_check_callbacks(dev);
 138         mutex_lock(&dpm_list_mtx);
 139         if (dev->parent && dev->parent->power.is_prepared)
 140                 dev_warn(dev, "parent %s should not be sleeping\n",
 141                         dev_name(dev->parent));
 142         list_add_tail(&dev->power.entry, &dpm_list);
 143         dev->power.in_dpm_list = true;
 144         mutex_unlock(&dpm_list_mtx);
 145 }
 146
 147 /**
 148  * device_pm_remove - Remove a device from the PM core's list of active devices.
 149  * @dev: Device to be removed from the list.
 150  */
 151 void device_pm_remove(struct device *dev)
 152 {
 153         if (device_pm_not_required(dev))
 154                 return;
 155
 156         pr_debug("Removing info for %s:%s\n",
 157                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 158         complete_all(&dev->power.completion);
 159         mutex_lock(&dpm_list_mtx);
 160         list_del_init(&dev->power.entry);
 161         dev->power.in_dpm_list = false;
 162         mutex_unlock(&dpm_list_mtx);
 163         device_wakeup_disable(dev);
 164         pm_runtime_remove(dev);
 165         device_pm_check_callbacks(dev);
 166 }
 167
 168 /**
 169  * device_pm_move_before - Move device in the PM core's list of active devices.
 170  * @deva: Device to move in dpm_list.
 171  * @devb: Device @deva should come before.
 172  */
 173 void device_pm_move_before(struct device *deva, struct device *devb)
 174 {
 175         pr_debug("Moving %s:%s before %s:%s\n",
 176                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 177                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 178         /* Delete deva from dpm_list and reinsert before devb. */
 179         list_move_tail(&deva->power.entry, &devb->power.entry);
 180 }
 181
 182 /**
 183  * device_pm_move_after - Move device in the PM core's list of active devices.
 184  * @deva: Device to move in dpm_list.
 185  * @devb: Device @deva should come after.
 186  */
 187 void device_pm_move_after(struct device *deva, struct device *devb)
 188 {
 189         pr_debug("Moving %s:%s after %s:%s\n",
 190                  deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
 191                  devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
 192         /* Delete deva from dpm_list and reinsert after devb. */
 193         list_move(&deva->power.entry, &devb->power.entry);
 194 }
 195
 196 /**
 197  * device_pm_move_last - Move device to end of the PM core's list of devices.
 198  * @dev: Device to move in dpm_list.
 199  */
 200 void device_pm_move_last(struct device *dev)
 201 {
 202         pr_debug("Moving %s:%s to end of list\n",
 203                  dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
 204         list_move_tail(&dev->power.entry, &dpm_list);
 205 }
 206
 207 static ktime_t initcall_debug_start(struct device *dev, void *cb)
 208 {
 209         if (!pm_print_times_enabled)
 210                 return 0;
 211
 212         dev_info(dev, "calling %pS @ %i, parent: %s\n", cb,
 213                  task_pid_nr(current),
 214                  dev->parent ? dev_name(dev->parent) : "none");
 215         return ktime_get();
 216 }
 217
 218 static void initcall_debug_report(struct device *dev, ktime_t calltime,
 219                                   void *cb, int error)
 220 {
 221         ktime_t rettime;
 222         s64 nsecs;
 223
 224         if (!pm_print_times_enabled)
 225                 return;
 226
 227         rettime = ktime_get();
 228         nsecs = (s64) ktime_to_ns(ktime_sub(rettime, calltime));
 229
 230         dev_info(dev, "%pS returned %d after %Ld usecs\n", cb, error,
 231                  (unsigned long long)nsecs >> 10);
 232 }
 233
 234 /**
 235  * dpm_wait - Wait for a PM operation to complete.
 236  * @dev: Device to wait for.
 237  * @async: If unset, wait only if the device's power.async_suspend flag is set.
 238  */
 239 static void dpm_wait(struct device *dev, bool async)
 240 {
 241         if (!dev)
 242                 return;
 243
 244         if (async || (pm_async_enabled && dev->power.async_suspend))
 245                 wait_for_completion(&dev->power.completion);
 246 }
 247
 248 static int dpm_wait_fn(struct device *dev, void *async_ptr)
 249 {
 250         dpm_wait(dev, *((bool *)async_ptr));
 251         return 0;
 252 }
 253
 254 static void dpm_wait_for_children(struct device *dev, bool async)
 255 {
 256        device_for_each_child(dev, &async, dpm_wait_fn);
 257 }
 258
 259 static void dpm_wait_for_suppliers(struct device *dev, bool async)
 260 {
 261         struct device_link *link;
 262         int idx;
 263
 264         idx = device_links_read_lock();
 265
 266         /*
 267          * If the supplier goes away right after we've checked the link to it,
 268          * we'll wait for its completion to change the state, but that's fine,
 269          * because the only things that will block as a result are the SRCU
 270          * callbacks freeing the link objects for the links in the list we're
 271          * walking.
 272          */
 273         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
 274                 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
 275                         dpm_wait(link->supplier, async);
 276
 277         device_links_read_unlock(idx);
 278 }
 279
 280 static bool dpm_wait_for_superior(struct device *dev, bool async)
 281 {
 282         struct device *parent;
 283
 284         /*
 285          * If the device is resumed asynchronously and the parent's callback
 286          * deletes both the device and the parent itself, the parent object may
 287          * be freed while this function is running, so avoid that by reference
 288          * counting the parent once more unless the device has been deleted
 289          * already (in which case return right away).
 290          */
 291         mutex_lock(&dpm_list_mtx);
 292
 293         if (!device_pm_initialized(dev)) {
 294                 mutex_unlock(&dpm_list_mtx);
 295                 return false;
 296         }
 297
 298         parent = get_device(dev->parent);
 299
 300         mutex_unlock(&dpm_list_mtx);
 301
 302         dpm_wait(parent, async);
 303         put_device(parent);
 304
 305         dpm_wait_for_suppliers(dev, async);
 306
 307         /*
 308          * If the parent's callback has deleted the device, attempting to resume
 309          * it would be invalid, so avoid doing that then.
 310          */
 311         return device_pm_initialized(dev);
 312 }
 313
 314 static void dpm_wait_for_consumers(struct device *dev, bool async)
 315 {
 316         struct device_link *link;
 317         int idx;
 318
 319         idx = device_links_read_lock();
 320
 321         /*
 322          * The status of a device link can only be changed from "dormant" by a
 323          * probe, but that cannot happen during system suspend/resume.  In
 324          * theory it can change to "dormant" at that time, but then it is
 325          * reasonable to wait for the target device anyway (eg. if it goes
 326          * away, it's better to wait for it to go away completely and then
 327          * continue instead of trying to continue in parallel with its
 328          * unregistration).
 329          */
 330         list_for_each_entry_rcu_locked(link, &dev->links.consumers, s_node)
 331                 if (READ_ONCE(link->status) != DL_STATE_DORMANT)
 332                         dpm_wait(link->consumer, async);
 333
 334         device_links_read_unlock(idx);
 335 }
 336
 337 static void dpm_wait_for_subordinate(struct device *dev, bool async)
 338 {
 339         dpm_wait_for_children(dev, async);
 340         dpm_wait_for_consumers(dev, async);
 341 }
 342
 343 /**
 344  * pm_op - Return the PM operation appropriate for given PM event.
 345  * @ops: PM operations to choose from.
 346  * @state: PM transition of the system being carried out.
 347  */
 348 static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
 349 {
 350         switch (state.event) {
 351 #ifdef CONFIG_SUSPEND
 352         case PM_EVENT_SUSPEND:
 353                 return ops->suspend;
 354         case PM_EVENT_RESUME:
 355                 return ops->resume;
 356 #endif /* CONFIG_SUSPEND */
 357 #ifdef CONFIG_HIBERNATE_CALLBACKS
 358         case PM_EVENT_FREEZE:
 359         case PM_EVENT_QUIESCE:
 360                 return ops->freeze;
 361         case PM_EVENT_HIBERNATE:
 362                 return ops->poweroff;
 363         case PM_EVENT_THAW:
 364         case PM_EVENT_RECOVER:
 365                 return ops->thaw;
 366                 break;
 367         case PM_EVENT_RESTORE:
 368                 return ops->restore;
 369 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 370         }
 371
 372         return NULL;
 373 }
 374
 375 /**
 376  * pm_late_early_op - Return the PM operation appropriate for given PM event.
 377  * @ops: PM operations to choose from.
 378  * @state: PM transition of the system being carried out.
 379  *
 380  * Runtime PM is disabled for @dev while this function is being executed.
 381  */
 382 static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
 383                                       pm_message_t state)
 384 {
 385         switch (state.event) {
 386 #ifdef CONFIG_SUSPEND
 387         case PM_EVENT_SUSPEND:
 388                 return ops->suspend_late;
 389         case PM_EVENT_RESUME:
 390                 return ops->resume_early;
 391 #endif /* CONFIG_SUSPEND */
 392 #ifdef CONFIG_HIBERNATE_CALLBACKS
 393         case PM_EVENT_FREEZE:
 394         case PM_EVENT_QUIESCE:
 395                 return ops->freeze_late;
 396         case PM_EVENT_HIBERNATE:
 397                 return ops->poweroff_late;
 398         case PM_EVENT_THAW:
 399         case PM_EVENT_RECOVER:
 400                 return ops->thaw_early;
 401         case PM_EVENT_RESTORE:
 402                 return ops->restore_early;
 403 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 404         }
 405
 406         return NULL;
 407 }
 408
 409 /**
 410  * pm_noirq_op - Return the PM operation appropriate for given PM event.
 411  * @ops: PM operations to choose from.
 412  * @state: PM transition of the system being carried out.
 413  *
 414  * The driver of @dev will not receive interrupts while this function is being
 415  * executed.
 416  */
 417 static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
 418 {
 419         switch (state.event) {
 420 #ifdef CONFIG_SUSPEND
 421         case PM_EVENT_SUSPEND:
 422                 return ops->suspend_noirq;
 423         case PM_EVENT_RESUME:
 424                 return ops->resume_noirq;
 425 #endif /* CONFIG_SUSPEND */
 426 #ifdef CONFIG_HIBERNATE_CALLBACKS
 427         case PM_EVENT_FREEZE:
 428         case PM_EVENT_QUIESCE:
 429                 return ops->freeze_noirq;
 430         case PM_EVENT_HIBERNATE:
 431                 return ops->poweroff_noirq;
 432         case PM_EVENT_THAW:
 433         case PM_EVENT_RECOVER:
 434                 return ops->thaw_noirq;
 435         case PM_EVENT_RESTORE:
 436                 return ops->restore_noirq;
 437 #endif /* CONFIG_HIBERNATE_CALLBACKS */
 438         }
 439
 440         return NULL;
 441 }
 442
 443 static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
 444 {
 445         dev_dbg(dev, "%s%s%s\n", info, pm_verb(state.event),
 446                 ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
 447                 ", may wakeup" : "");
 448 }
 449
 450 static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
 451                         int error)
 452 {
 453         pr_err("Device %s failed to %s%s: error %d\n",
 454                dev_name(dev), pm_verb(state.event), info, error);
 455 }
 456
 457 static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
 458                           const char *info)
 459 {
 460         ktime_t calltime;
 461         u64 usecs64;
 462         int usecs;
 463
 464         calltime = ktime_get();
 465         usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
 466         do_div(usecs64, NSEC_PER_USEC);
 467         usecs = usecs64;
 468         if (usecs == 0)
 469                 usecs = 1;
 470
 471         pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
 472                   info ?: "", info ? " " : "", pm_verb(state.event),
 473                   error ? "aborted" : "complete",
 474                   usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
 475 }
 476
 477 static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 478                             pm_message_t state, const char *info)
 479 {
 480         ktime_t calltime;
 481         int error;
 482
 483         if (!cb)
 484                 return 0;
 485
 486         calltime = initcall_debug_start(dev, cb);
 487
 488         pm_dev_dbg(dev, state, info);
 489         trace_device_pm_callback_start(dev, info, state.event);
 490         error = cb(dev);
 491         trace_device_pm_callback_end(dev, error);
 492         suspend_report_result(cb, error);
 493
 494         initcall_debug_report(dev, calltime, cb, error);
 495
 496         return error;
 497 }
 498
 499 #ifdef CONFIG_DPM_WATCHDOG
 500 struct dpm_watchdog {
 501         struct device           *dev;
 502         struct task_struct      *tsk;
 503         struct timer_list       timer;
 504 };
 505
 506 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
 507         struct dpm_watchdog wd
 508
 509 /**
 510  * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 511  * @t: The timer that PM watchdog depends on.
 512  *
 513  * Called when a driver has timed out suspending or resuming.
 514  * There's not much we can do here to recover so panic() to
 515  * capture a crash-dump in pstore.
 516  */
 517 static void dpm_watchdog_handler(struct timer_list *t)
 518 {
 519         struct dpm_watchdog *wd = from_timer(wd, t, timer);
 520
 521         dev_emerg(wd->dev, "**** DPM device timeout ****\n");
 522         show_stack(wd->tsk, NULL);
 523         panic("%s %s: unrecoverable failure\n",
 524                 dev_driver_string(wd->dev), dev_name(wd->dev));
 525 }
 526
 527 /**
 528  * dpm_watchdog_set - Enable pm watchdog for given device.
 529  * @wd: Watchdog. Must be allocated on the stack.
 530  * @dev: Device to handle.
 531  */
 532 static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
 533 {
 534         struct timer_list *timer = &wd->timer;
 535
 536         wd->dev = dev;
 537         wd->tsk = current;
 538
 539         timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
 540         /* use same timeout value for both suspend and resume */
 541         timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_TIMEOUT;
 542         add_timer(timer);
 543 }
 544
 545 /**
 546  * dpm_watchdog_clear - Disable suspend/resume watchdog.
 547  * @wd: Watchdog to disable.
 548  */
 549 static void dpm_watchdog_clear(struct dpm_watchdog *wd)
 550 {
 551         struct timer_list *timer = &wd->timer;
 552
 553         del_timer_sync(timer);
 554         destroy_timer_on_stack(timer);
 555 }
 556 #else
 557 #define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
 558 #define dpm_watchdog_set(x, y)
 559 #define dpm_watchdog_clear(x)
 560 #endif
 561
 562 /*------------------------- Resume routines -------------------------*/
 563
 564 /**
 565  * suspend_event - Return a "suspend" message for given "resume" one.
 566  * @resume_msg: PM message representing a system-wide resume transition.
 567  */
 568 static pm_message_t suspend_event(pm_message_t resume_msg)
 569 {
 570         switch (resume_msg.event) {
 571         case PM_EVENT_RESUME:
 572                 return PMSG_SUSPEND;
 573         case PM_EVENT_THAW:
 574         case PM_EVENT_RESTORE:
 575                 return PMSG_FREEZE;
 576         case PM_EVENT_RECOVER:
 577                 return PMSG_HIBERNATE;
 578         }
 579         return PMSG_ON;
 580 }
 581
 582 /**
 583  * dev_pm_may_skip_resume - System-wide device resume optimization check.
 584  * @dev: Target device.
 585  *
 586  * Checks whether or not the device may be left in suspend after a system-wide
 587  * transition to the working state.
 588  */
 589 bool dev_pm_may_skip_resume(struct device *dev)
 590 {
 591         return !dev->power.must_resume && pm_transition.event != PM_EVENT_RESTORE;
 592 }
 593
 594 static pm_callback_t dpm_subsys_resume_noirq_cb(struct device *dev,
 595                                                 pm_message_t state,
 596                                                 const char **info_p)
 597 {
 598         pm_callback_t callback;
 599         const char *info;
 600
 601         if (dev->pm_domain) {
 602                 info = "noirq power domain ";
 603                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
 604         } else if (dev->type && dev->type->pm) {
 605                 info = "noirq type ";
 606                 callback = pm_noirq_op(dev->type->pm, state);
 607         } else if (dev->class && dev->class->pm) {
 608                 info = "noirq class ";
 609                 callback = pm_noirq_op(dev->class->pm, state);
 610         } else if (dev->bus && dev->bus->pm) {
 611                 info = "noirq bus ";
 612                 callback = pm_noirq_op(dev->bus->pm, state);
 613         } else {
 614                 return NULL;
 615         }
 616
 617         if (info_p)
 618                 *info_p = info;
 619
 620         return callback;
 621 }
 622
 623 static pm_callback_t dpm_subsys_suspend_noirq_cb(struct device *dev,
 624                                                  pm_message_t state,
 625                                                  const char **info_p);
 626
 627 static pm_callback_t dpm_subsys_suspend_late_cb(struct device *dev,
 628                                                 pm_message_t state,
 629                                                 const char **info_p);
 630
 631 /**
 632  * device_resume_noirq - Execute a "noirq resume" callback for given device.
 633  * @dev: Device to handle.
 634  * @state: PM transition of the system being carried out.
 635  * @async: If true, the device is being resumed asynchronously.
 636  *
 637  * The driver of @dev will not receive interrupts while this function is being
 638  * executed.
 639  */
 640 static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
 641 {
 642         pm_callback_t callback;
 643         const char *info;
 644         bool skip_resume;
 645         int error = 0;
 646
 647         TRACE_DEVICE(dev);
 648         TRACE_RESUME(0);
 649
 650         if (dev->power.syscore || dev->power.direct_complete)
 651                 goto Out;
 652
 653         if (!dev->power.is_noirq_suspended)
 654                 goto Out;
 655
 656         if (!dpm_wait_for_superior(dev, async))
 657                 goto Out;
 658
 659         skip_resume = dev_pm_may_skip_resume(dev);
 660
 661         callback = dpm_subsys_resume_noirq_cb(dev, state, &info);
 662         if (callback)
 663                 goto Run;
 664
 665         if (skip_resume)
 666                 goto Skip;
 667
 668         if (dev_pm_smart_suspend_and_suspended(dev)) {
 669                 pm_message_t suspend_msg = suspend_event(state);
 670
 671                 /*
 672                  * If "freeze" callbacks have been skipped during a transition
 673                  * related to hibernation, the subsequent "thaw" callbacks must
 674                  * be skipped too or bad things may happen.  Otherwise, resume
 675                  * callbacks are going to be run for the device, so its runtime
 676                  * PM status must be changed to reflect the new state after the
 677                  * transition under way.
 678                  */
 679                 if (!dpm_subsys_suspend_late_cb(dev, suspend_msg, NULL) &&
 680                     !dpm_subsys_suspend_noirq_cb(dev, suspend_msg, NULL)) {
 681                         if (state.event == PM_EVENT_THAW) {
 682                                 skip_resume = true;
 683                                 goto Skip;
 684                         } else {
 685                                 pm_runtime_set_active(dev);
 686                         }
 687                 }
 688         }
 689
 690         if (dev->driver && dev->driver->pm) {
 691                 info = "noirq driver ";
 692                 callback = pm_noirq_op(dev->driver->pm, state);
 693         }
 694
 695 Run:
 696         error = dpm_run_callback(callback, dev, state, info);
 697
 698 Skip:
 699         dev->power.is_noirq_suspended = false;
 700
 701         if (skip_resume) {
 702                 /* Make the next phases of resume skip the device. */
 703                 dev->power.is_late_suspended = false;
 704                 dev->power.is_suspended = false;
 705                 /*
 706                  * The device is going to be left in suspend, but it might not
 707                  * have been in runtime suspend before the system suspended, so
 708                  * its runtime PM status needs to be updated to avoid confusing
 709                  * the runtime PM framework when runtime PM is enabled for the
 710                  * device again.
 711                  */
 712                 pm_runtime_set_suspended(dev);
 713         }
 714
 715 Out:
 716         complete_all(&dev->power.completion);
 717         TRACE_RESUME(error);
 718         return error;
 719 }
 720
 721 static bool is_async(struct device *dev)
 722 {
 723         return dev->power.async_suspend && pm_async_enabled
 724                 && !pm_trace_is_enabled();
 725 }
 726
 727 static bool dpm_async_fn(struct device *dev, async_func_t func)
 728 {
 729         reinit_completion(&dev->power.completion);
 730
 731         if (is_async(dev)) {
 732                 get_device(dev);
 733                 async_schedule_dev(func, dev);
 734                 return true;
 735         }
 736
 737         return false;
 738 }
 739
 740 static void async_resume_noirq(void *data, async_cookie_t cookie)
 741 {
 742         struct device *dev = (struct device *)data;
 743         int error;
 744
 745         error = device_resume_noirq(dev, pm_transition, true);
 746         if (error)
 747                 pm_dev_err(dev, pm_transition, " async", error);
 748
 749         put_device(dev);
 750 }
 751
 752 static void dpm_noirq_resume_devices(pm_message_t state)
 753 {
 754         struct device *dev;
 755         ktime_t starttime = ktime_get();
 756
 757         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);
 758         mutex_lock(&dpm_list_mtx);
 759         pm_transition = state;
 760
 761         /*
 762          * Advanced the async threads upfront,
 763          * in case the starting of async threads is
 764          * delayed by non-async resuming devices.
 765          */
 766         list_for_each_entry(dev, &dpm_noirq_list, power.entry)
 767                 dpm_async_fn(dev, async_resume_noirq);
 768
 769         while (!list_empty(&dpm_noirq_list)) {
 770                 dev = to_device(dpm_noirq_list.next);
 771                 get_device(dev);
 772                 list_move_tail(&dev->power.entry, &dpm_late_early_list);
 773                 mutex_unlock(&dpm_list_mtx);
 774
 775                 if (!is_async(dev)) {
 776                         int error;
 777
 778                         error = device_resume_noirq(dev, state, false);
 779                         if (error) {
 780                                 suspend_stats.failed_resume_noirq++;
 781                                 dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);
 782                                 dpm_save_failed_dev(dev_name(dev));
 783                                 pm_dev_err(dev, state, " noirq", error);
 784                         }
 785                 }
 786
 787                 mutex_lock(&dpm_list_mtx);
 788                 put_device(dev);
 789         }
 790         mutex_unlock(&dpm_list_mtx);
 791         async_synchronize_full();
 792         dpm_show_time(starttime, state, 0, "noirq");
 793         trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
 794 }
 795
 796 /**
 797  * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 798  * @state: PM transition of the system being carried out.
 799  *
 800  * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
 801  * allow device drivers' interrupt handlers to be called.
 802  */
 803 void dpm_resume_noirq(pm_message_t state)
 804 {
 805         dpm_noirq_resume_devices(state);
 806
 807         resume_device_irqs();
 808         device_wakeup_disarm_wake_irqs();
 809
 810         cpuidle_resume();
 811 }
 812
 813 static pm_callback_t dpm_subsys_resume_early_cb(struct device *dev,
 814                                                 pm_message_t state,
 815                                                 const char **info_p)
 816 {
 817         pm_callback_t callback;
 818         const char *info;
 819
 820         if (dev->pm_domain) {
 821                 info = "early power domain ";
 822                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
 823         } else if (dev->type && dev->type->pm) {
 824                 info = "early type ";
 825                 callback = pm_late_early_op(dev->type->pm, state);
 826         } else if (dev->class && dev->class->pm) {
 827                 info = "early class ";
 828                 callback = pm_late_early_op(dev->class->pm, state);
 829         } else if (dev->bus && dev->bus->pm) {
 830                 info = "early bus ";
 831                 callback = pm_late_early_op(dev->bus->pm, state);
 832         } else {
 833                 return NULL;
 834         }
 835
 836         if (info_p)
 837                 *info_p = info;
 838
 839         return callback;
 840 }
 841
 842 /**
 843  * device_resume_early - Execute an "early resume" callback for given device.
 844  * @dev: Device to handle.
 845  * @state: PM transition of the system being carried out.
 846  * @async: If true, the device is being resumed asynchronously.
 847  *
 848  * Runtime PM is disabled for @dev while this function is being executed.
 849  */
 850 static int device_resume_early(struct device *dev, pm_message_t state, bool async)
 851 {
 852         pm_callback_t callback;
 853         const char *info;
 854         int error = 0;
 855
 856         TRACE_DEVICE(dev);
 857         TRACE_RESUME(0);
 858
 859         if (dev->power.syscore || dev->power.direct_complete)
 860                 goto Out;
 861
 862         if (!dev->power.is_late_suspended)
 863                 goto Out;
 864
 865         if (!dpm_wait_for_superior(dev, async))
 866                 goto Out;
 867
 868         callback = dpm_subsys_resume_early_cb(dev, state, &info);
 869
 870         if (!callback && dev->driver && dev->driver->pm) {
 871                 info = "early driver ";
 872                 callback = pm_late_early_op(dev->driver->pm, state);
 873         }
 874
 875         error = dpm_run_callback(callback, dev, state, info);
 876         dev->power.is_late_suspended = false;
 877
 878  Out:
 879         TRACE_RESUME(error);
 880
 881         pm_runtime_enable(dev);
 882         complete_all(&dev->power.completion);
 883         return error;
 884 }
 885
 886 static void async_resume_early(void *data, async_cookie_t cookie)
 887 {
 888         struct device *dev = (struct device *)data;
 889         int error;
 890
 891         error = device_resume_early(dev, pm_transition, true);
 892         if (error)
 893                 pm_dev_err(dev, pm_transition, " async", error);
 894
 895         put_device(dev);
 896 }
 897
 898 /**
 899  * dpm_resume_early - Execute "early resume" callbacks for all devices.
 900  * @state: PM transition of the system being carried out.
 901  */
 902 void dpm_resume_early(pm_message_t state)
 903 {
 904         struct device *dev;
 905         ktime_t starttime = ktime_get();
 906
 907         trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);
 908         mutex_lock(&dpm_list_mtx);
 909         pm_transition = state;
 910
 911         /*
 912          * Advanced the async threads upfront,
 913          * in case the starting of async threads is
 914          * delayed by non-async resuming devices.
 915          */
 916         list_for_each_entry(dev, &dpm_late_early_list, power.entry)
 917                 dpm_async_fn(dev, async_resume_early);
 918
 919         while (!list_empty(&dpm_late_early_list)) {
 920                 dev = to_device(dpm_late_early_list.next);
 921                 get_device(dev);
 922                 list_move_tail(&dev->power.entry, &dpm_suspended_list);
 923                 mutex_unlock(&dpm_list_mtx);
 924
 925                 if (!is_async(dev)) {
 926                         int error;
 927
 928                         error = device_resume_early(dev, state, false);
 929                         if (error) {
 930                                 suspend_stats.failed_resume_early++;
 931                                 dpm_save_failed_step(SUSPEND_RESUME_EARLY);
 932                                 dpm_save_failed_dev(dev_name(dev));
 933                                 pm_dev_err(dev, state, " early", error);
 934                         }
 935                 }
 936                 mutex_lock(&dpm_list_mtx);
 937                 put_device(dev);
 938         }
 939         mutex_unlock(&dpm_list_mtx);
 940         async_synchronize_full();
 941         dpm_show_time(starttime, state, 0, "early");
 942         trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
 943 }
 944
 945 /**
 946  * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 947  * @state: PM transition of the system being carried out.
 948  */
 949 void dpm_resume_start(pm_message_t state)
 950 {
 951         dpm_resume_noirq(state);
 952         dpm_resume_early(state);
 953 }
 954 EXPORT_SYMBOL_GPL(dpm_resume_start);
 955
 956 /**
 957  * device_resume - Execute "resume" callbacks for given device.
 958  * @dev: Device to handle.
 959  * @state: PM transition of the system being carried out.
 960  * @async: If true, the device is being resumed asynchronously.
 961  */
 962 static int device_resume(struct device *dev, pm_message_t state, bool async)
 963 {
 964         pm_callback_t callback = NULL;
 965         const char *info = NULL;
 966         int error = 0;
 967         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 968
 969         TRACE_DEVICE(dev);
 970         TRACE_RESUME(0);
 971
 972         if (dev->power.syscore)
 973                 goto Complete;
 974
 975         if (dev->power.direct_complete) {
 976                 /* Match the pm_runtime_disable() in __device_suspend(). */
 977                 pm_runtime_enable(dev);
 978                 goto Complete;
 979         }
 980
 981         if (!dpm_wait_for_superior(dev, async))
 982                 goto Complete;
 983
 984         dpm_watchdog_set(&wd, dev);
 985         device_lock(dev);
 986
 987         /*
 988          * This is a fib.  But we'll allow new children to be added below
 989          * a resumed device, even if the device hasn't been completed yet.
 990          */
 991         dev->power.is_prepared = false;
 992
 993         if (!dev->power.is_suspended)
 994                 goto Unlock;
 995
 996         if (dev->pm_domain) {
 997                 info = "power domain ";
 998                 callback = pm_op(&dev->pm_domain->ops, state);
 999                 goto Driver;
1000         }
1001
1002         if (dev->type && dev->type->pm) {
1003                 info = "type ";
1004                 callback = pm_op(dev->type->pm, state);
1005                 goto Driver;
1006         }
1007
1008         if (dev->class && dev->class->pm) {
1009                 info = "class ";
1010                 callback = pm_op(dev->class->pm, state);
1011                 goto Driver;
1012         }
1013
1014         if (dev->bus) {
1015                 if (dev->bus->pm) {
1016                         info = "bus ";
1017                         callback = pm_op(dev->bus->pm, state);
1018                 } else if (dev->bus->resume) {
1019                         info = "legacy bus ";
1020                         callback = dev->bus->resume;
1021                         goto End;
1022                 }
1023         }
1024
1025  Driver:
1026         if (!callback && dev->driver && dev->driver->pm) {
1027                 info = "driver ";
1028                 callback = pm_op(dev->driver->pm, state);
1029         }
1030
1031  End:
1032         error = dpm_run_callback(callback, dev, state, info);
1033         dev->power.is_suspended = false;
1034
1035  Unlock:
1036         device_unlock(dev);
1037         dpm_watchdog_clear(&wd);
1038
1039  Complete:
1040         complete_all(&dev->power.completion);
1041
1042         TRACE_RESUME(error);
1043
1044         return error;
1045 }
1046
1047 static void async_resume(void *data, async_cookie_t cookie)
1048 {
1049         struct device *dev = (struct device *)data;
1050         int error;
1051
1052         error = device_resume(dev, pm_transition, true);
1053         if (error)
1054                 pm_dev_err(dev, pm_transition, " async", error);
1055         put_device(dev);
1056 }
1057
1058 /**
1059  * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
1060  * @state: PM transition of the system being carried out.
1061  *
1062  * Execute the appropriate "resume" callback for all devices whose status
1063  * indicates that they are suspended.
1064  */
1065 void dpm_resume(pm_message_t state)
1066 {
1067         struct device *dev;
1068         ktime_t starttime = ktime_get();
1069
1070         trace_suspend_resume(TPS("dpm_resume"), state.event, true);
1071         might_sleep();
1072
1073         mutex_lock(&dpm_list_mtx);
1074         pm_transition = state;
1075         async_error = 0;
1076
1077         list_for_each_entry(dev, &dpm_suspended_list, power.entry)
1078                 dpm_async_fn(dev, async_resume);
1079
1080         while (!list_empty(&dpm_suspended_list)) {
1081                 dev = to_device(dpm_suspended_list.next);
1082                 get_device(dev);
1083                 if (!is_async(dev)) {
1084                         int error;
1085
1086                         mutex_unlock(&dpm_list_mtx);
1087
1088                         error = device_resume(dev, state, false);
1089                         if (error) {
1090                                 suspend_stats.failed_resume++;
1091                                 dpm_save_failed_step(SUSPEND_RESUME);
1092                                 dpm_save_failed_dev(dev_name(dev));
1093                                 pm_dev_err(dev, state, "", error);
1094                         }
1095
1096                         mutex_lock(&dpm_list_mtx);
1097                 }
1098                 if (!list_empty(&dev->power.entry))
1099                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
1100                 put_device(dev);
1101         }
1102         mutex_unlock(&dpm_list_mtx);
1103         async_synchronize_full();
1104         dpm_show_time(starttime, state, 0, NULL);
1105
1106         cpufreq_resume();
1107         devfreq_resume();
1108         trace_suspend_resume(TPS("dpm_resume"), state.event, false);
1109 }
1110
1111 /**
1112  * device_complete - Complete a PM transition for given device.
1113  * @dev: Device to handle.
1114  * @state: PM transition of the system being carried out.
1115  */
1116 static void device_complete(struct device *dev, pm_message_t state)
1117 {
1118         void (*callback)(struct device *) = NULL;
1119         const char *info = NULL;
1120
1121         if (dev->power.syscore)
1122                 return;
1123
1124         device_lock(dev);
1125
1126         if (dev->pm_domain) {
1127                 info = "completing power domain ";
1128                 callback = dev->pm_domain->ops.complete;
1129         } else if (dev->type && dev->type->pm) {
1130                 info = "completing type ";
1131                 callback = dev->type->pm->complete;
1132         } else if (dev->class && dev->class->pm) {
1133                 info = "completing class ";
1134                 callback = dev->class->pm->complete;
1135         } else if (dev->bus && dev->bus->pm) {
1136                 info = "completing bus ";
1137                 callback = dev->bus->pm->complete;
1138         }
1139
1140         if (!callback && dev->driver && dev->driver->pm) {
1141                 info = "completing driver ";
1142                 callback = dev->driver->pm->complete;
1143         }
1144
1145         if (callback) {
1146                 pm_dev_dbg(dev, state, info);
1147                 callback(dev);
1148         }
1149
1150         device_unlock(dev);
1151
1152         pm_runtime_put(dev);
1153 }
1154
1155 /**
1156  * dpm_complete - Complete a PM transition for all non-sysdev devices.
1157  * @state: PM transition of the system being carried out.
1158  *
1159  * Execute the ->complete() callbacks for all devices whose PM status is not
1160  * DPM_ON (this allows new devices to be registered).
1161  */
1162 void dpm_complete(pm_message_t state)
1163 {
1164         struct list_head list;
1165
1166         trace_suspend_resume(TPS("dpm_complete"), state.event, true);
1167         might_sleep();
1168
1169         INIT_LIST_HEAD(&list);
1170         mutex_lock(&dpm_list_mtx);
1171         while (!list_empty(&dpm_prepared_list)) {
1172                 struct device *dev = to_device(dpm_prepared_list.prev);
1173
1174                 get_device(dev);
1175                 dev->power.is_prepared = false;
1176                 list_move(&dev->power.entry, &list);
1177                 mutex_unlock(&dpm_list_mtx);
1178
1179                 trace_device_pm_callback_start(dev, "", state.event);
1180                 device_complete(dev, state);
1181                 trace_device_pm_callback_end(dev, 0);
1182
1183                 mutex_lock(&dpm_list_mtx);
1184                 put_device(dev);
1185         }
1186         list_splice(&list, &dpm_list);
1187         mutex_unlock(&dpm_list_mtx);
1188
1189         /* Allow device probing and trigger re-probing of deferred devices */
1190         device_unblock_probing();
1191         trace_suspend_resume(TPS("dpm_complete"), state.event, false);
1192 }
1193
1194 /**
1195  * dpm_resume_end - Execute "resume" callbacks and complete system transition.
1196  * @state: PM transition of the system being carried out.
1197  *
1198  * Execute "resume" callbacks for all devices and complete the PM transition of
1199  * the system.
1200  */
1201 void dpm_resume_end(pm_message_t state)
1202 {
1203         dpm_resume(state);
1204         dpm_complete(state);
1205 }
1206 EXPORT_SYMBOL_GPL(dpm_resume_end);
1207
1208
1209 /*------------------------- Suspend routines -------------------------*/
1210
1211 /**
1212  * resume_event - Return a "resume" message for given "suspend" sleep state.
1213  * @sleep_state: PM message representing a sleep state.
1214  *
1215  * Return a PM message representing the resume event corresponding to given
1216  * sleep state.
1217  */
1218 static pm_message_t resume_event(pm_message_t sleep_state)
1219 {
1220         switch (sleep_state.event) {
1221         case PM_EVENT_SUSPEND:
1222                 return PMSG_RESUME;
1223         case PM_EVENT_FREEZE:
1224         case PM_EVENT_QUIESCE:
1225                 return PMSG_RECOVER;
1226         case PM_EVENT_HIBERNATE:
1227                 return PMSG_RESTORE;
1228         }
1229         return PMSG_ON;
1230 }
1231
1232 static void dpm_superior_set_must_resume(struct device *dev)
1233 {
1234         struct device_link *link;
1235         int idx;
1236
1237         if (dev->parent)
1238                 dev->parent->power.must_resume = true;
1239
1240         idx = device_links_read_lock();
1241
1242         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node)
1243                 link->supplier->power.must_resume = true;
1244
1245         device_links_read_unlock(idx);
1246 }
1247
1248 static pm_callback_t dpm_subsys_suspend_noirq_cb(struct device *dev,
1249                                                  pm_message_t state,
1250                                                  const char **info_p)
1251 {
1252         pm_callback_t callback;
1253         const char *info;
1254
1255         if (dev->pm_domain) {
1256                 info = "noirq power domain ";
1257                 callback = pm_noirq_op(&dev->pm_domain->ops, state);
1258         } else if (dev->type && dev->type->pm) {
1259                 info = "noirq type ";
1260                 callback = pm_noirq_op(dev->type->pm, state);
1261         } else if (dev->class && dev->class->pm) {
1262                 info = "noirq class ";
1263                 callback = pm_noirq_op(dev->class->pm, state);
1264         } else if (dev->bus && dev->bus->pm) {
1265                 info = "noirq bus ";
1266                 callback = pm_noirq_op(dev->bus->pm, state);
1267         } else {
1268                 return NULL;
1269         }
1270
1271         if (info_p)
1272                 *info_p = info;
1273
1274         return callback;
1275 }
1276
1277 static bool device_must_resume(struct device *dev, pm_message_t state,
1278                                bool no_subsys_suspend_noirq)
1279 {
1280         pm_message_t resume_msg = resume_event(state);
1281
1282         /*
1283          * If all of the device driver's "noirq", "late" and "early" callbacks
1284          * are invoked directly by the core, the decision to allow the device to
1285          * stay in suspend can be based on its current runtime PM status and its
1286          * wakeup settings.
1287          */
1288         if (no_subsys_suspend_noirq &&
1289             !dpm_subsys_suspend_late_cb(dev, state, NULL) &&
1290             !dpm_subsys_resume_early_cb(dev, resume_msg, NULL) &&
1291             !dpm_subsys_resume_noirq_cb(dev, resume_msg, NULL))
1292                 return !pm_runtime_status_suspended(dev) &&
1293                         (resume_msg.event != PM_EVENT_RESUME ||
1294                          (device_can_wakeup(dev) && !device_may_wakeup(dev)));
1295
1296         /*
1297          * The only safe strategy here is to require that if the device may not
1298          * be left in suspend, resume callbacks must be invoked for it.
1299          */
1300         return !dev->power.may_skip_resume;
1301 }
1302
1303 /**
1304  * __device_suspend_noirq - Execute a "noirq suspend" callback for given device.
1305  * @dev: Device to handle.
1306  * @state: PM transition of the system being carried out.
1307  * @async: If true, the device is being suspended asynchronously.
1308  *
1309  * The driver of @dev will not receive interrupts while this function is being
1310  * executed.
1311  */
1312 static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
1313 {
1314         pm_callback_t callback;
1315         const char *info;
1316         bool no_subsys_cb = false;
1317         int error = 0;
1318
1319         TRACE_DEVICE(dev);
1320         TRACE_SUSPEND(0);
1321
1322         dpm_wait_for_subordinate(dev, async);
1323
1324         if (async_error)
1325                 goto Complete;
1326
1327         if (dev->power.syscore || dev->power.direct_complete)
1328                 goto Complete;
1329
1330         callback = dpm_subsys_suspend_noirq_cb(dev, state, &info);
1331         if (callback)
1332                 goto Run;
1333
1334         no_subsys_cb = !dpm_subsys_suspend_late_cb(dev, state, NULL);
1335
1336         if (dev_pm_smart_suspend_and_suspended(dev) && no_subsys_cb)
1337                 goto Skip;
1338
1339         if (dev->driver && dev->driver->pm) {
1340                 info = "noirq driver ";
1341                 callback = pm_noirq_op(dev->driver->pm, state);
1342         }
1343
1344 Run:
1345         error = dpm_run_callback(callback, dev, state, info);
1346         if (error) {
1347                 async_error = error;
1348                 goto Complete;
1349         }
1350
1351 Skip:
1352         dev->power.is_noirq_suspended = true;
1353
1354         if (dev_pm_test_driver_flags(dev, DPM_FLAG_LEAVE_SUSPENDED)) {
1355                 dev->power.must_resume = dev->power.must_resume ||
1356                                 atomic_read(&dev->power.usage_count) > 1 ||
1357                                 device_must_resume(dev, state, no_subsys_cb);
1358         } else {
1359                 dev->power.must_resume = true;
1360         }
1361
1362         if (dev->power.must_resume)
1363                 dpm_superior_set_must_resume(dev);
1364
1365 Complete:
1366         complete_all(&dev->power.completion);
1367         TRACE_SUSPEND(error);
1368         return error;
1369 }
1370
1371 static void async_suspend_noirq(void *data, async_cookie_t cookie)
1372 {
1373         struct device *dev = (struct device *)data;
1374         int error;
1375
1376         error = __device_suspend_noirq(dev, pm_transition, true);
1377         if (error) {
1378                 dpm_save_failed_dev(dev_name(dev));
1379                 pm_dev_err(dev, pm_transition, " async", error);
1380         }
1381
1382         put_device(dev);
1383 }
1384
1385 static int device_suspend_noirq(struct device *dev)
1386 {
1387         if (dpm_async_fn(dev, async_suspend_noirq))
1388                 return 0;
1389
1390         return __device_suspend_noirq(dev, pm_transition, false);
1391 }
1392
1393 static int dpm_noirq_suspend_devices(pm_message_t state)
1394 {
1395         ktime_t starttime = ktime_get();
1396         int error = 0;
1397
1398         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
1399         mutex_lock(&dpm_list_mtx);
1400         pm_transition = state;
1401         async_error = 0;
1402
1403         while (!list_empty(&dpm_late_early_list)) {
1404                 struct device *dev = to_device(dpm_late_early_list.prev);
1405
1406                 get_device(dev);
1407                 mutex_unlock(&dpm_list_mtx);
1408
1409                 error = device_suspend_noirq(dev);
1410
1411                 mutex_lock(&dpm_list_mtx);
1412                 if (error) {
1413                         pm_dev_err(dev, state, " noirq", error);
1414                         dpm_save_failed_dev(dev_name(dev));
1415                         put_device(dev);
1416                         break;
1417                 }
1418                 if (!list_empty(&dev->power.entry))
1419                         list_move(&dev->power.entry, &dpm_noirq_list);
1420                 put_device(dev);
1421
1422                 if (async_error)
1423                         break;
1424         }
1425         mutex_unlock(&dpm_list_mtx);
1426         async_synchronize_full();
1427         if (!error)
1428                 error = async_error;
1429
1430         if (error) {
1431                 suspend_stats.failed_suspend_noirq++;
1432                 dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
1433         }
1434         dpm_show_time(starttime, state, error, "noirq");
1435         trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
1436         return error;
1437 }
1438
1439 /**
1440  * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
1441  * @state: PM transition of the system being carried out.
1442  *
1443  * Prevent device drivers' interrupt handlers from being called and invoke
1444  * "noirq" suspend callbacks for all non-sysdev devices.
1445  */
1446 int dpm_suspend_noirq(pm_message_t state)
1447 {
1448         int ret;
1449
1450         cpuidle_pause();
1451
1452         device_wakeup_arm_wake_irqs();
1453         suspend_device_irqs();
1454
1455         ret = dpm_noirq_suspend_devices(state);
1456         if (ret)
1457                 dpm_resume_noirq(resume_event(state));
1458
1459         return ret;
1460 }
1461
1462 static void dpm_propagate_wakeup_to_parent(struct device *dev)
1463 {
1464         struct device *parent = dev->parent;
1465
1466         if (!parent)
1467                 return;
1468
1469         spin_lock_irq(&parent->power.lock);
1470
1471         if (dev->power.wakeup_path && !parent->power.ignore_children)
1472                 parent->power.wakeup_path = true;
1473
1474         spin_unlock_irq(&parent->power.lock);
1475 }
1476
1477 static pm_callback_t dpm_subsys_suspend_late_cb(struct device *dev,
1478                                                 pm_message_t state,
1479                                                 const char **info_p)
1480 {
1481         pm_callback_t callback;
1482         const char *info;
1483
1484         if (dev->pm_domain) {
1485                 info = "late power domain ";
1486                 callback = pm_late_early_op(&dev->pm_domain->ops, state);
1487         } else if (dev->type && dev->type->pm) {
1488                 info = "late type ";
1489                 callback = pm_late_early_op(dev->type->pm, state);
1490         } else if (dev->class && dev->class->pm) {
1491                 info = "late class ";
1492                 callback = pm_late_early_op(dev->class->pm, state);
1493         } else if (dev->bus && dev->bus->pm) {
1494                 info = "late bus ";
1495                 callback = pm_late_early_op(dev->bus->pm, state);
1496         } else {
1497                 return NULL;
1498         }
1499
1500         if (info_p)
1501                 *info_p = info;
1502
1503         return callback;
1504 }
1505
1506 /**
1507  * __device_suspend_late - Execute a "late suspend" callback for given device.
1508  * @dev: Device to handle.
1509  * @state: PM transition of the system being carried out.
1510  * @async: If true, the device is being suspended asynchronously.
1511  *
1512  * Runtime PM is disabled for @dev while this function is being executed.
1513  */
1514 static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
1515 {
1516         pm_callback_t callback;
1517         const char *info;
1518         int error = 0;
1519
1520         TRACE_DEVICE(dev);
1521         TRACE_SUSPEND(0);
1522
1523         __pm_runtime_disable(dev, false);
1524
1525         dpm_wait_for_subordinate(dev, async);
1526
1527         if (async_error)
1528                 goto Complete;
1529
1530         if (pm_wakeup_pending()) {
1531                 async_error = -EBUSY;
1532                 goto Complete;
1533         }
1534
1535         if (dev->power.syscore || dev->power.direct_complete)
1536                 goto Complete;
1537
1538         callback = dpm_subsys_suspend_late_cb(dev, state, &info);
1539         if (callback)
1540                 goto Run;
1541
1542         if (dev_pm_smart_suspend_and_suspended(dev) &&
1543             !dpm_subsys_suspend_noirq_cb(dev, state, NULL))
1544                 goto Skip;
1545
1546         if (dev->driver && dev->driver->pm) {
1547                 info = "late driver ";
1548                 callback = pm_late_early_op(dev->driver->pm, state);
1549         }
1550
1551 Run:
1552         error = dpm_run_callback(callback, dev, state, info);
1553         if (error) {
1554                 async_error = error;
1555                 goto Complete;
1556         }
1557         dpm_propagate_wakeup_to_parent(dev);
1558
1559 Skip:
1560         dev->power.is_late_suspended = true;
1561
1562 Complete:
1563         TRACE_SUSPEND(error);
1564         complete_all(&dev->power.completion);
1565         return error;
1566 }
1567
1568 static void async_suspend_late(void *data, async_cookie_t cookie)
1569 {
1570         struct device *dev = (struct device *)data;
1571         int error;
1572
1573         error = __device_suspend_late(dev, pm_transition, true);
1574         if (error) {
1575                 dpm_save_failed_dev(dev_name(dev));
1576                 pm_dev_err(dev, pm_transition, " async", error);
1577         }
1578         put_device(dev);
1579 }
1580
1581 static int device_suspend_late(struct device *dev)
1582 {
1583         if (dpm_async_fn(dev, async_suspend_late))
1584                 return 0;
1585
1586         return __device_suspend_late(dev, pm_transition, false);
1587 }
1588
1589 /**
1590  * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
1591  * @state: PM transition of the system being carried out.
1592  */
1593 int dpm_suspend_late(pm_message_t state)
1594 {
1595         ktime_t starttime = ktime_get();
1596         int error = 0;
1597
1598         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);
1599         mutex_lock(&dpm_list_mtx);
1600         pm_transition = state;
1601         async_error = 0;
1602
1603         while (!list_empty(&dpm_suspended_list)) {
1604                 struct device *dev = to_device(dpm_suspended_list.prev);
1605
1606                 get_device(dev);
1607                 mutex_unlock(&dpm_list_mtx);
1608
1609                 error = device_suspend_late(dev);
1610
1611                 mutex_lock(&dpm_list_mtx);
1612                 if (!list_empty(&dev->power.entry))
1613                         list_move(&dev->power.entry, &dpm_late_early_list);
1614
1615                 if (error) {
1616                         pm_dev_err(dev, state, " late", error);
1617                         dpm_save_failed_dev(dev_name(dev));
1618                         put_device(dev);
1619                         break;
1620                 }
1621                 put_device(dev);
1622
1623                 if (async_error)
1624                         break;
1625         }
1626         mutex_unlock(&dpm_list_mtx);
1627         async_synchronize_full();
1628         if (!error)
1629                 error = async_error;
1630         if (error) {
1631                 suspend_stats.failed_suspend_late++;
1632                 dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
1633                 dpm_resume_early(resume_event(state));
1634         }
1635         dpm_show_time(starttime, state, error, "late");
1636         trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
1637         return error;
1638 }
1639
1640 /**
1641  * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
1642  * @state: PM transition of the system being carried out.
1643  */
1644 int dpm_suspend_end(pm_message_t state)
1645 {
1646         ktime_t starttime = ktime_get();
1647         int error;
1648
1649         error = dpm_suspend_late(state);
1650         if (error)
1651                 goto out;
1652
1653         error = dpm_suspend_noirq(state);
1654         if (error)
1655                 dpm_resume_early(resume_event(state));
1656
1657 out:
1658         dpm_show_time(starttime, state, error, "end");
1659         return error;
1660 }
1661 EXPORT_SYMBOL_GPL(dpm_suspend_end);
1662
1663 /**
1664  * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
1665  * @dev: Device to suspend.
1666  * @state: PM transition of the system being carried out.
1667  * @cb: Suspend callback to execute.
1668  * @info: string description of caller.
1669  */
1670 static int legacy_suspend(struct device *dev, pm_message_t state,
1671                           int (*cb)(struct device *dev, pm_message_t state),
1672                           const char *info)
1673 {
1674         int error;
1675         ktime_t calltime;
1676
1677         calltime = initcall_debug_start(dev, cb);
1678
1679         trace_device_pm_callback_start(dev, info, state.event);
1680         error = cb(dev, state);
1681         trace_device_pm_callback_end(dev, error);
1682         suspend_report_result(cb, error);
1683
1684         initcall_debug_report(dev, calltime, cb, error);
1685
1686         return error;
1687 }
1688
1689 static void dpm_clear_superiors_direct_complete(struct device *dev)
1690 {
1691         struct device_link *link;
1692         int idx;
1693
1694         if (dev->parent) {
1695                 spin_lock_irq(&dev->parent->power.lock);
1696                 dev->parent->power.direct_complete = false;
1697                 spin_unlock_irq(&dev->parent->power.lock);
1698         }
1699
1700         idx = device_links_read_lock();
1701
1702         list_for_each_entry_rcu_locked(link, &dev->links.suppliers, c_node) {
1703                 spin_lock_irq(&link->supplier->power.lock);
1704                 link->supplier->power.direct_complete = false;
1705                 spin_unlock_irq(&link->supplier->power.lock);
1706         }
1707
1708         device_links_read_unlock(idx);
1709 }
1710
1711 /**
1712  * __device_suspend - Execute "suspend" callbacks for given device.
1713  * @dev: Device to handle.
1714  * @state: PM transition of the system being carried out.
1715  * @async: If true, the device is being suspended asynchronously.
1716  */
1717 static int __device_suspend(struct device *dev, pm_message_t state, bool async)
1718 {
1719         pm_callback_t callback = NULL;
1720         const char *info = NULL;
1721         int error = 0;
1722         DECLARE_DPM_WATCHDOG_ON_STACK(wd);
1723
1724         TRACE_DEVICE(dev);
1725         TRACE_SUSPEND(0);
1726
1727         dpm_wait_for_subordinate(dev, async);
1728
1729         if (async_error) {
1730                 dev->power.direct_complete = false;
1731                 goto Complete;
1732         }
1733
1734         /*
1735          * If a device configured to wake up the system from sleep states
1736          * has been suspended at run time and there's a resume request pending
1737          * for it, this is equivalent to the device signaling wakeup, so the
1738          * system suspend operation should be aborted.
1739          */
1740         if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
1741                 pm_wakeup_event(dev, 0);
1742
1743         if (pm_wakeup_pending()) {
1744                 dev->power.direct_complete = false;
1745                 async_error = -EBUSY;
1746                 goto Complete;
1747         }
1748
1749         if (dev->power.syscore)
1750                 goto Complete;
1751
1752         /* Avoid direct_complete to let wakeup_path propagate. */
1753         if (device_may_wakeup(dev) || dev->power.wakeup_path)
1754                 dev->power.direct_complete = false;
1755
1756         if (dev->power.direct_complete) {
1757                 if (pm_runtime_status_suspended(dev)) {
1758                         pm_runtime_disable(dev);
1759                         if (pm_runtime_status_suspended(dev)) {
1760                                 pm_dev_dbg(dev, state, "direct-complete ");
1761                                 goto Complete;
1762                         }
1763
1764                         pm_runtime_enable(dev);
1765                 }
1766                 dev->power.direct_complete = false;
1767         }
1768
1769         dev->power.may_skip_resume = false;
1770         dev->power.must_resume = false;
1771
1772         dpm_watchdog_set(&wd, dev);
1773         device_lock(dev);
1774
1775         if (dev->pm_domain) {
1776                 info = "power domain ";
1777                 callback = pm_op(&dev->pm_domain->ops, state);
1778                 goto Run;
1779         }
1780
1781         if (dev->type && dev->type->pm) {
1782                 info = "type ";
1783                 callback = pm_op(dev->type->pm, state);
1784                 goto Run;
1785         }
1786
1787         if (dev->class && dev->class->pm) {
1788                 info = "class ";
1789                 callback = pm_op(dev->class->pm, state);
1790                 goto Run;
1791         }
1792
1793         if (dev->bus) {
1794                 if (dev->bus->pm) {
1795                         info = "bus ";
1796                         callback = pm_op(dev->bus->pm, state);
1797                 } else if (dev->bus->suspend) {
1798                         pm_dev_dbg(dev, state, "legacy bus ");
1799                         error = legacy_suspend(dev, state, dev->bus->suspend,
1800                                                 "legacy bus ");
1801                         goto End;
1802                 }
1803         }
1804
1805  Run:
1806         if (!callback && dev->driver && dev->driver->pm) {
1807                 info = "driver ";
1808                 callback = pm_op(dev->driver->pm, state);
1809         }
1810
1811         error = dpm_run_callback(callback, dev, state, info);
1812
1813  End:
1814         if (!error) {
1815                 dev->power.is_suspended = true;
1816                 if (device_may_wakeup(dev))
1817                         dev->power.wakeup_path = true;
1818
1819                 dpm_propagate_wakeup_to_parent(dev);
1820                 dpm_clear_superiors_direct_complete(dev);
1821         }
1822
1823         device_unlock(dev);
1824         dpm_watchdog_clear(&wd);
1825
1826  Complete:
1827         if (error)
1828                 async_error = error;
1829
1830         complete_all(&dev->power.completion);
1831         TRACE_SUSPEND(error);
1832         return error;
1833 }
1834
1835 static void async_suspend(void *data, async_cookie_t cookie)
1836 {
1837         struct device *dev = (struct device *)data;
1838         int error;
1839
1840         error = __device_suspend(dev, pm_transition, true);
1841         if (error) {
1842                 dpm_save_failed_dev(dev_name(dev));
1843                 pm_dev_err(dev, pm_transition, " async", error);
1844         }
1845
1846         put_device(dev);
1847 }
1848
1849 static int device_suspend(struct device *dev)
1850 {
1851         if (dpm_async_fn(dev, async_suspend))
1852                 return 0;
1853
1854         return __device_suspend(dev, pm_transition, false);
1855 }
1856
1857 /**
1858  * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
1859  * @state: PM transition of the system being carried out.
1860  */
1861 int dpm_suspend(pm_message_t state)
1862 {
1863         ktime_t starttime = ktime_get();
1864         int error = 0;
1865
1866         trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
1867         might_sleep();
1868
1869         devfreq_suspend();
1870         cpufreq_suspend();
1871
1872         mutex_lock(&dpm_list_mtx);
1873         pm_transition = state;
1874         async_error = 0;
1875         while (!list_empty(&dpm_prepared_list)) {
1876                 struct device *dev = to_device(dpm_prepared_list.prev);
1877
1878                 get_device(dev);
1879                 mutex_unlock(&dpm_list_mtx);
1880
1881                 error = device_suspend(dev);
1882
1883                 mutex_lock(&dpm_list_mtx);
1884                 if (error) {
1885                         pm_dev_err(dev, state, "", error);
1886                         dpm_save_failed_dev(dev_name(dev));
1887                         put_device(dev);
1888                         break;
1889                 }
1890                 if (!list_empty(&dev->power.entry))
1891                         list_move(&dev->power.entry, &dpm_suspended_list);
1892                 put_device(dev);
1893                 if (async_error)
1894                         break;
1895         }
1896         mutex_unlock(&dpm_list_mtx);
1897         async_synchronize_full();
1898         if (!error)
1899                 error = async_error;
1900         if (error) {
1901                 suspend_stats.failed_suspend++;
1902                 dpm_save_failed_step(SUSPEND_SUSPEND);
1903         }
1904         dpm_show_time(starttime, state, error, NULL);
1905         trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
1906         return error;
1907 }
1908
1909 /**
1910  * device_prepare - Prepare a device for system power transition.
1911  * @dev: Device to handle.
1912  * @state: PM transition of the system being carried out.
1913  *
1914  * Execute the ->prepare() callback(s) for given device.  No new children of the
1915  * device may be registered after this function has returned.
1916  */
1917 static int device_prepare(struct device *dev, pm_message_t state)
1918 {
1919         int (*callback)(struct device *) = NULL;
1920         int ret = 0;
1921
1922         if (dev->power.syscore)
1923                 return 0;
1924
1925         /*
1926          * If a device's parent goes into runtime suspend at the wrong time,
1927          * it won't be possible to resume the device.  To prevent this we
1928          * block runtime suspend here, during the prepare phase, and allow
1929          * it again during the complete phase.
1930          */
1931         pm_runtime_get_noresume(dev);
1932
1933         device_lock(dev);
1934
1935         dev->power.wakeup_path = false;
1936
1937         if (dev->power.no_pm_callbacks)
1938                 goto unlock;
1939
1940         if (dev->pm_domain)
1941                 callback = dev->pm_domain->ops.prepare;
1942         else if (dev->type && dev->type->pm)
1943                 callback = dev->type->pm->prepare;
1944         else if (dev->class && dev->class->pm)
1945                 callback = dev->class->pm->prepare;
1946         else if (dev->bus && dev->bus->pm)
1947                 callback = dev->bus->pm->prepare;
1948
1949         if (!callback && dev->driver && dev->driver->pm)
1950                 callback = dev->driver->pm->prepare;
1951
1952         if (callback)
1953                 ret = callback(dev);
1954
1955 unlock:
1956         device_unlock(dev);
1957
1958         if (ret < 0) {
1959                 suspend_report_result(callback, ret);
1960                 pm_runtime_put(dev);
1961                 return ret;
1962         }
1963         /*
1964          * A positive return value from ->prepare() means "this device appears
1965          * to be runtime-suspended and its state is fine, so if it really is
1966          * runtime-suspended, you can leave it in that state provided that you
1967          * will do the same thing with all of its descendants".  This only
1968          * applies to suspend transitions, however.
1969          */
1970         spin_lock_irq(&dev->power.lock);
1971         dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
1972                 (ret > 0 || dev->power.no_pm_callbacks) &&
1973                 !dev_pm_test_driver_flags(dev, DPM_FLAG_NEVER_SKIP);
1974         spin_unlock_irq(&dev->power.lock);
1975         return 0;
1976 }
1977
1978 /**
1979  * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
1980  * @state: PM transition of the system being carried out.
1981  *
1982  * Execute the ->prepare() callback(s) for all devices.
1983  */
1984 int dpm_prepare(pm_message_t state)
1985 {
1986         int error = 0;
1987
1988         trace_suspend_resume(TPS("dpm_prepare"), state.event, true);
1989         might_sleep();
1990
1991         /*
1992          * Give a chance for the known devices to complete their probes, before
1993          * disable probing of devices. This sync point is important at least
1994          * at boot time + hibernation restore.
1995          */
1996         wait_for_device_probe();
1997         /*
1998          * It is unsafe if probing of devices will happen during suspend or
1999          * hibernation and system behavior will be unpredictable in this case.
2000          * So, let's prohibit device's probing here and defer their probes
2001          * instead. The normal behavior will be restored in dpm_complete().
2002          */
2003         device_block_probing();
2004
2005         mutex_lock(&dpm_list_mtx);
2006         while (!list_empty(&dpm_list)) {
2007                 struct device *dev = to_device(dpm_list.next);
2008
2009                 get_device(dev);
2010                 mutex_unlock(&dpm_list_mtx);
2011
2012                 trace_device_pm_callback_start(dev, "", state.event);
2013                 error = device_prepare(dev, state);
2014                 trace_device_pm_callback_end(dev, error);
2015
2016                 mutex_lock(&dpm_list_mtx);
2017                 if (error) {
2018                         if (error == -EAGAIN) {
2019                                 put_device(dev);
2020                                 error = 0;
2021                                 continue;
2022                         }
2023                         pr_info("Device %s not prepared for power transition: code %d\n",
2024                                 dev_name(dev), error);
2025                         put_device(dev);
2026                         break;
2027                 }
2028                 dev->power.is_prepared = true;
2029                 if (!list_empty(&dev->power.entry))
2030                         list_move_tail(&dev->power.entry, &dpm_prepared_list);
2031                 put_device(dev);
2032         }
2033         mutex_unlock(&dpm_list_mtx);
2034         trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
2035         return error;
2036 }
2037
2038 /**
2039  * dpm_suspend_start - Prepare devices for PM transition and suspend them.
2040  * @state: PM transition of the system being carried out.
2041  *
2042  * Prepare all non-sysdev devices for system PM transition and execute "suspend"
2043  * callbacks for them.
2044  */
2045 int dpm_suspend_start(pm_message_t state)
2046 {
2047         ktime_t starttime = ktime_get();
2048         int error;
2049
2050         error = dpm_prepare(state);
2051         if (error) {
2052                 suspend_stats.failed_prepare++;
2053                 dpm_save_failed_step(SUSPEND_PREPARE);
2054         } else
2055                 error = dpm_suspend(state);
2056         dpm_show_time(starttime, state, error, "start");
2057         return error;
2058 }
2059 EXPORT_SYMBOL_GPL(dpm_suspend_start);
2060
2061 void __suspend_report_result(const char *function, void *fn, int ret)
2062 {
2063         if (ret)
2064                 pr_err("%s(): %pS returns %d\n", function, fn, ret);
2065 }
2066 EXPORT_SYMBOL_GPL(__suspend_report_result);
2067
2068 /**
2069  * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
2070  * @subordinate: Device that needs to wait for @dev.
2071  * @dev: Device to wait for.
2072  */
2073 int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
2074 {
2075         dpm_wait(dev, subordinate->power.async_suspend);
2076         return async_error;
2077 }
2078 EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);
2079
2080 /**
2081  * dpm_for_each_dev - device iterator.
2082  * @data: data for the callback.
2083  * @fn: function to be called for each device.
2084  *
2085  * Iterate over devices in dpm_list, and call @fn for each device,
2086  * passing it @data.
2087  */
2088 void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
2089 {
2090         struct device *dev;
2091
2092         if (!fn)
2093                 return;
2094
2095         device_pm_lock();
2096         list_for_each_entry(dev, &dpm_list, power.entry)
2097                 fn(dev, data);
2098         device_pm_unlock();
2099 }
2100 EXPORT_SYMBOL_GPL(dpm_for_each_dev);
2101
2102 static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
2103 {
2104         if (!ops)
2105                 return true;
2106
2107         return !ops->prepare &&
2108                !ops->suspend &&
2109                !ops->suspend_late &&
2110                !ops->suspend_noirq &&
2111                !ops->resume_noirq &&
2112                !ops->resume_early &&
2113                !ops->resume &&
2114                !ops->complete;
2115 }
2116
2117 void device_pm_check_callbacks(struct device *dev)
2118 {
2119         spin_lock_irq(&dev->power.lock);
2120         dev->power.no_pm_callbacks =
2121                 (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
2122                  !dev->bus->suspend && !dev->bus->resume)) &&
2123                 (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
2124                 (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
2125                 (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
2126                 (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
2127                  !dev->driver->suspend && !dev->driver->resume));
2128         spin_unlock_irq(&dev->power.lock);
2129 }
2130
2131 bool dev_pm_smart_suspend_and_suspended(struct device *dev)
2132 {
2133         return dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
2134                 pm_runtime_status_suspended(dev);
2135 }