drivers/base/power/runtime.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * drivers/base/power/runtime.c - Helper functions for device runtime PM
   4  *
   5  * Copyright (c) 2009 Rafael J. Wysocki <[email protected]>, Novell Inc.
   6  * Copyright (C) 2010 Alan Stern <[email protected]>
   7  */
   8 #include <linux/sched/mm.h>
   9 #include <linux/ktime.h>
  10 #include <linux/hrtimer.h>
  11 #include <linux/export.h>
  12 #include <linux/pm_runtime.h>
  13 #include <linux/pm_wakeirq.h>
  14 #include <trace/events/rpm.h>
  15
  16 #include "../base.h"
  17 #include "power.h"
  18
  19 typedef int (*pm_callback_t)(struct device *);
  20
  21 static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
  22 {
  23         pm_callback_t cb;
  24         const struct dev_pm_ops *ops;
  25
  26         if (dev->pm_domain)
  27                 ops = &dev->pm_domain->ops;
  28         else if (dev->type && dev->type->pm)
  29                 ops = dev->type->pm;
  30         else if (dev->class && dev->class->pm)
  31                 ops = dev->class->pm;
  32         else if (dev->bus && dev->bus->pm)
  33                 ops = dev->bus->pm;
  34         else
  35                 ops = NULL;
  36
  37         if (ops)
  38                 cb = *(pm_callback_t *)((void *)ops + cb_offset);
  39         else
  40                 cb = NULL;
  41
  42         if (!cb && dev->driver && dev->driver->pm)
  43                 cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
  44
  45         return cb;
  46 }
  47
  48 #define RPM_GET_CALLBACK(dev, callback) \
  49                 __rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
  50
  51 static int rpm_resume(struct device *dev, int rpmflags);
  52 static int rpm_suspend(struct device *dev, int rpmflags);
  53
  54 /**
  55  * update_pm_runtime_accounting - Update the time accounting of power states
  56  * @dev: Device to update the accounting for
  57  *
  58  * In order to be able to have time accounting of the various power states
  59  * (as used by programs such as PowerTOP to show the effectiveness of runtime
  60  * PM), we need to track the time spent in each state.
  61  * update_pm_runtime_accounting must be called each time before the
  62  * runtime_status field is updated, to account the time in the old state
  63  * correctly.
  64  */
  65 static void update_pm_runtime_accounting(struct device *dev)
  66 {
  67         u64 now, last, delta;
  68
  69         if (dev->power.disable_depth > 0)
  70                 return;
  71
  72         last = dev->power.accounting_timestamp;
  73
  74         now = ktime_get_mono_fast_ns();
  75         dev->power.accounting_timestamp = now;
  76
  77         /*
  78          * Because ktime_get_mono_fast_ns() is not monotonic during
  79          * timekeeping updates, ensure that 'now' is after the last saved
  80          * timesptamp.
  81          */
  82         if (now < last)
  83                 return;
  84
  85         delta = now - last;
  86
  87         if (dev->power.runtime_status == RPM_SUSPENDED)
  88                 dev->power.suspended_time += delta;
  89         else
  90                 dev->power.active_time += delta;
  91 }
  92
  93 static void __update_runtime_status(struct device *dev, enum rpm_status status)
  94 {
  95         update_pm_runtime_accounting(dev);
  96         dev->power.runtime_status = status;
  97 }
  98
  99 static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
 100 {
 101         u64 time;
 102         unsigned long flags;
 103
 104         spin_lock_irqsave(&dev->power.lock, flags);
 105
 106         update_pm_runtime_accounting(dev);
 107         time = suspended ? dev->power.suspended_time : dev->power.active_time;
 108
 109         spin_unlock_irqrestore(&dev->power.lock, flags);
 110
 111         return time;
 112 }
 113
 114 u64 pm_runtime_active_time(struct device *dev)
 115 {
 116         return rpm_get_accounted_time(dev, false);
 117 }
 118
 119 u64 pm_runtime_suspended_time(struct device *dev)
 120 {
 121         return rpm_get_accounted_time(dev, true);
 122 }
 123 EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
 124
 125 /**
 126  * pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
 127  * @dev: Device to handle.
 128  */
 129 static void pm_runtime_deactivate_timer(struct device *dev)
 130 {
 131         if (dev->power.timer_expires > 0) {
 132                 hrtimer_try_to_cancel(&dev->power.suspend_timer);
 133                 dev->power.timer_expires = 0;
 134         }
 135 }
 136
 137 /**
 138  * pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
 139  * @dev: Device to handle.
 140  */
 141 static void pm_runtime_cancel_pending(struct device *dev)
 142 {
 143         pm_runtime_deactivate_timer(dev);
 144         /*
 145          * In case there's a request pending, make sure its work function will
 146          * return without doing anything.
 147          */
 148         dev->power.request = RPM_REQ_NONE;
 149 }
 150
 151 /*
 152  * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
 153  * @dev: Device to handle.
 154  *
 155  * Compute the autosuspend-delay expiration time based on the device's
 156  * power.last_busy time.  If the delay has already expired or is disabled
 157  * (negative) or the power.use_autosuspend flag isn't set, return 0.
 158  * Otherwise return the expiration time in nanoseconds (adjusted to be nonzero).
 159  *
 160  * This function may be called either with or without dev->power.lock held.
 161  * Either way it can be racy, since power.last_busy may be updated at any time.
 162  */
 163 u64 pm_runtime_autosuspend_expiration(struct device *dev)
 164 {
 165         int autosuspend_delay;
 166         u64 expires;
 167
 168         if (!dev->power.use_autosuspend)
 169                 return 0;
 170
 171         autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
 172         if (autosuspend_delay < 0)
 173                 return 0;
 174
 175         expires  = READ_ONCE(dev->power.last_busy);
 176         expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
 177         if (expires > ktime_get_mono_fast_ns())
 178                 return expires; /* Expires in the future */
 179
 180         return 0;
 181 }
 182 EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
 183
 184 static int dev_memalloc_noio(struct device *dev, void *data)
 185 {
 186         return dev->power.memalloc_noio;
 187 }
 188
 189 /*
 190  * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
 191  * @dev: Device to handle.
 192  * @enable: True for setting the flag and False for clearing the flag.
 193  *
 194  * Set the flag for all devices in the path from the device to the
 195  * root device in the device tree if @enable is true, otherwise clear
 196  * the flag for devices in the path whose siblings don't set the flag.
 197  *
 198  * The function should only be called by block device, or network
 199  * device driver for solving the deadlock problem during runtime
 200  * resume/suspend:
 201  *
 202  *     If memory allocation with GFP_KERNEL is called inside runtime
 203  *     resume/suspend callback of any one of its ancestors(or the
 204  *     block device itself), the deadlock may be triggered inside the
 205  *     memory allocation since it might not complete until the block
 206  *     device becomes active and the involed page I/O finishes. The
 207  *     situation is pointed out first by Alan Stern. Network device
 208  *     are involved in iSCSI kind of situation.
 209  *
 210  * The lock of dev_hotplug_mutex is held in the function for handling
 211  * hotplug race because pm_runtime_set_memalloc_noio() may be called
 212  * in async probe().
 213  *
 214  * The function should be called between device_add() and device_del()
 215  * on the affected device(block/network device).
 216  */
 217 void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
 218 {
 219         static DEFINE_MUTEX(dev_hotplug_mutex);
 220
 221         mutex_lock(&dev_hotplug_mutex);
 222         for (;;) {
 223                 bool enabled;
 224
 225                 /* hold power lock since bitfield is not SMP-safe. */
 226                 spin_lock_irq(&dev->power.lock);
 227                 enabled = dev->power.memalloc_noio;
 228                 dev->power.memalloc_noio = enable;
 229                 spin_unlock_irq(&dev->power.lock);
 230
 231                 /*
 232                  * not need to enable ancestors any more if the device
 233                  * has been enabled.
 234                  */
 235                 if (enabled && enable)
 236                         break;
 237
 238                 dev = dev->parent;
 239
 240                 /*
 241                  * clear flag of the parent device only if all the
 242                  * children don't set the flag because ancestor's
 243                  * flag was set by any one of the descendants.
 244                  */
 245                 if (!dev || (!enable &&
 246                     device_for_each_child(dev, NULL, dev_memalloc_noio)))
 247                         break;
 248         }
 249         mutex_unlock(&dev_hotplug_mutex);
 250 }
 251 EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
 252
 253 /**
 254  * rpm_check_suspend_allowed - Test whether a device may be suspended.
 255  * @dev: Device to test.
 256  */
 257 static int rpm_check_suspend_allowed(struct device *dev)
 258 {
 259         int retval = 0;
 260
 261         if (dev->power.runtime_error)
 262                 retval = -EINVAL;
 263         else if (dev->power.disable_depth > 0)
 264                 retval = -EACCES;
 265         else if (atomic_read(&dev->power.usage_count))
 266                 retval = -EAGAIN;
 267         else if (!dev->power.ignore_children && atomic_read(&dev->power.child_count))
 268                 retval = -EBUSY;
 269
 270         /* Pending resume requests take precedence over suspends. */
 271         else if ((dev->power.deferred_resume &&
 272             dev->power.runtime_status == RPM_SUSPENDING) ||
 273             (dev->power.request_pending && dev->power.request == RPM_REQ_RESUME))
 274                 retval = -EAGAIN;
 275         else if (__dev_pm_qos_resume_latency(dev) == 0)
 276                 retval = -EPERM;
 277         else if (dev->power.runtime_status == RPM_SUSPENDED)
 278                 retval = 1;
 279
 280         return retval;
 281 }
 282
 283 static int rpm_get_suppliers(struct device *dev)
 284 {
 285         struct device_link *link;
 286
 287         list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 288                                 device_links_read_lock_held()) {
 289                 int retval;
 290
 291                 if (!(link->flags & DL_FLAG_PM_RUNTIME))
 292                         continue;
 293
 294                 retval = pm_runtime_get_sync(link->supplier);
 295                 /* Ignore suppliers with disabled runtime PM. */
 296                 if (retval < 0 && retval != -EACCES) {
 297                         pm_runtime_put_noidle(link->supplier);
 298                         return retval;
 299                 }
 300                 refcount_inc(&link->rpm_active);
 301         }
 302         return 0;
 303 }
 304
 305 /**
 306  * pm_runtime_release_supplier - Drop references to device link's supplier.
 307  * @link: Target device link.
 308  *
 309  * Drop all runtime PM references associated with @link to its supplier device.
 310  */
 311 void pm_runtime_release_supplier(struct device_link *link)
 312 {
 313         struct device *supplier = link->supplier;
 314
 315         /*
 316          * The additional power.usage_count check is a safety net in case
 317          * the rpm_active refcount becomes saturated, in which case
 318          * refcount_dec_not_one() would return true forever, but it is not
 319          * strictly necessary.
 320          */
 321         while (refcount_dec_not_one(&link->rpm_active) &&
 322                atomic_read(&supplier->power.usage_count) > 0)
 323                 pm_runtime_put_noidle(supplier);
 324 }
 325
 326 static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
 327 {
 328         struct device_link *link;
 329
 330         list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 331                                 device_links_read_lock_held()) {
 332                 pm_runtime_release_supplier(link);
 333                 if (try_to_suspend)
 334                         pm_request_idle(link->supplier);
 335         }
 336 }
 337
 338 static void rpm_put_suppliers(struct device *dev)
 339 {
 340         __rpm_put_suppliers(dev, true);
 341 }
 342
 343 static void rpm_suspend_suppliers(struct device *dev)
 344 {
 345         struct device_link *link;
 346         int idx = device_links_read_lock();
 347
 348         list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 349                                 device_links_read_lock_held())
 350                 pm_request_idle(link->supplier);
 351
 352         device_links_read_unlock(idx);
 353 }
 354
 355 /**
 356  * __rpm_callback - Run a given runtime PM callback for a given device.
 357  * @cb: Runtime PM callback to run.
 358  * @dev: Device to run the callback for.
 359  */
 360 static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
 361         __releases(&dev->power.lock) __acquires(&dev->power.lock)
 362 {
 363         int retval = 0, idx;
 364         bool use_links = dev->power.links_count > 0;
 365
 366         if (dev->power.irq_safe) {
 367                 spin_unlock(&dev->power.lock);
 368         } else {
 369                 spin_unlock_irq(&dev->power.lock);
 370
 371                 /*
 372                  * Resume suppliers if necessary.
 373                  *
 374                  * The device's runtime PM status cannot change until this
 375                  * routine returns, so it is safe to read the status outside of
 376                  * the lock.
 377                  */
 378                 if (use_links && dev->power.runtime_status == RPM_RESUMING) {
 379                         idx = device_links_read_lock();
 380
 381                         retval = rpm_get_suppliers(dev);
 382                         if (retval) {
 383                                 rpm_put_suppliers(dev);
 384                                 goto fail;
 385                         }
 386
 387                         device_links_read_unlock(idx);
 388                 }
 389         }
 390
 391         if (cb)
 392                 retval = cb(dev);
 393
 394         if (dev->power.irq_safe) {
 395                 spin_lock(&dev->power.lock);
 396         } else {
 397                 /*
 398                  * If the device is suspending and the callback has returned
 399                  * success, drop the usage counters of the suppliers that have
 400                  * been reference counted on its resume.
 401                  *
 402                  * Do that if resume fails too.
 403                  */
 404                 if (use_links &&
 405                     ((dev->power.runtime_status == RPM_SUSPENDING && !retval) ||
 406                     (dev->power.runtime_status == RPM_RESUMING && retval))) {
 407                         idx = device_links_read_lock();
 408
 409                         __rpm_put_suppliers(dev, false);
 410
 411 fail:
 412                         device_links_read_unlock(idx);
 413                 }
 414
 415                 spin_lock_irq(&dev->power.lock);
 416         }
 417
 418         return retval;
 419 }
 420
 421 /**
 422  * rpm_callback - Run a given runtime PM callback for a given device.
 423  * @cb: Runtime PM callback to run.
 424  * @dev: Device to run the callback for.
 425  */
 426 static int rpm_callback(int (*cb)(struct device *), struct device *dev)
 427 {
 428         int retval;
 429
 430         if (dev->power.memalloc_noio) {
 431                 unsigned int noio_flag;
 432
 433                 /*
 434                  * Deadlock might be caused if memory allocation with
 435                  * GFP_KERNEL happens inside runtime_suspend and
 436                  * runtime_resume callbacks of one block device's
 437                  * ancestor or the block device itself. Network
 438                  * device might be thought as part of iSCSI block
 439                  * device, so network device and its ancestor should
 440                  * be marked as memalloc_noio too.
 441                  */
 442                 noio_flag = memalloc_noio_save();
 443                 retval = __rpm_callback(cb, dev);
 444                 memalloc_noio_restore(noio_flag);
 445         } else {
 446                 retval = __rpm_callback(cb, dev);
 447         }
 448
 449         dev->power.runtime_error = retval;
 450         return retval != -EACCES ? retval : -EIO;
 451 }
 452
 453 /**
 454  * rpm_idle - Notify device bus type if the device can be suspended.
 455  * @dev: Device to notify the bus type about.
 456  * @rpmflags: Flag bits.
 457  *
 458  * Check if the device's runtime PM status allows it to be suspended.  If
 459  * another idle notification has been started earlier, return immediately.  If
 460  * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
 461  * run the ->runtime_idle() callback directly. If the ->runtime_idle callback
 462  * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
 463  *
 464  * This function must be called under dev->power.lock with interrupts disabled.
 465  */
 466 static int rpm_idle(struct device *dev, int rpmflags)
 467 {
 468         int (*callback)(struct device *);
 469         int retval;
 470
 471         trace_rpm_idle(dev, rpmflags);
 472         retval = rpm_check_suspend_allowed(dev);
 473         if (retval < 0)
 474                 ;       /* Conditions are wrong. */
 475
 476         /* Idle notifications are allowed only in the RPM_ACTIVE state. */
 477         else if (dev->power.runtime_status != RPM_ACTIVE)
 478                 retval = -EAGAIN;
 479
 480         /*
 481          * Any pending request other than an idle notification takes
 482          * precedence over us, except that the timer may be running.
 483          */
 484         else if (dev->power.request_pending &&
 485             dev->power.request > RPM_REQ_IDLE)
 486                 retval = -EAGAIN;
 487
 488         /* Act as though RPM_NOWAIT is always set. */
 489         else if (dev->power.idle_notification)
 490                 retval = -EINPROGRESS;
 491
 492         if (retval)
 493                 goto out;
 494
 495         /* Pending requests need to be canceled. */
 496         dev->power.request = RPM_REQ_NONE;
 497
 498         callback = RPM_GET_CALLBACK(dev, runtime_idle);
 499
 500         /* If no callback assume success. */
 501         if (!callback || dev->power.no_callbacks)
 502                 goto out;
 503
 504         /* Carry out an asynchronous or a synchronous idle notification. */
 505         if (rpmflags & RPM_ASYNC) {
 506                 dev->power.request = RPM_REQ_IDLE;
 507                 if (!dev->power.request_pending) {
 508                         dev->power.request_pending = true;
 509                         queue_work(pm_wq, &dev->power.work);
 510                 }
 511                 trace_rpm_return_int(dev, _THIS_IP_, 0);
 512                 return 0;
 513         }
 514
 515         dev->power.idle_notification = true;
 516
 517         if (dev->power.irq_safe)
 518                 spin_unlock(&dev->power.lock);
 519         else
 520                 spin_unlock_irq(&dev->power.lock);
 521
 522         retval = callback(dev);
 523
 524         if (dev->power.irq_safe)
 525                 spin_lock(&dev->power.lock);
 526         else
 527                 spin_lock_irq(&dev->power.lock);
 528
 529         dev->power.idle_notification = false;
 530         wake_up_all(&dev->power.wait_queue);
 531
 532  out:
 533         trace_rpm_return_int(dev, _THIS_IP_, retval);
 534         return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
 535 }
 536
 537 /**
 538  * rpm_suspend - Carry out runtime suspend of given device.
 539  * @dev: Device to suspend.
 540  * @rpmflags: Flag bits.
 541  *
 542  * Check if the device's runtime PM status allows it to be suspended.
 543  * Cancel a pending idle notification, autosuspend or suspend. If
 544  * another suspend has been started earlier, either return immediately
 545  * or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
 546  * flags. If the RPM_ASYNC flag is set then queue a suspend request;
 547  * otherwise run the ->runtime_suspend() callback directly. When
 548  * ->runtime_suspend succeeded, if a deferred resume was requested while
 549  * the callback was running then carry it out, otherwise send an idle
 550  * notification for its parent (if the suspend succeeded and both
 551  * ignore_children of parent->power and irq_safe of dev->power are not set).
 552  * If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
 553  * flag is set and the next autosuspend-delay expiration time is in the
 554  * future, schedule another autosuspend attempt.
 555  *
 556  * This function must be called under dev->power.lock with interrupts disabled.
 557  */
 558 static int rpm_suspend(struct device *dev, int rpmflags)
 559         __releases(&dev->power.lock) __acquires(&dev->power.lock)
 560 {
 561         int (*callback)(struct device *);
 562         struct device *parent = NULL;
 563         int retval;
 564
 565         trace_rpm_suspend(dev, rpmflags);
 566
 567  repeat:
 568         retval = rpm_check_suspend_allowed(dev);
 569         if (retval < 0)
 570                 goto out;       /* Conditions are wrong. */
 571
 572         /* Synchronous suspends are not allowed in the RPM_RESUMING state. */
 573         if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
 574                 retval = -EAGAIN;
 575
 576         if (retval)
 577                 goto out;
 578
 579         /* If the autosuspend_delay time hasn't expired yet, reschedule. */
 580         if ((rpmflags & RPM_AUTO) && dev->power.runtime_status != RPM_SUSPENDING) {
 581                 u64 expires = pm_runtime_autosuspend_expiration(dev);
 582
 583                 if (expires != 0) {
 584                         /* Pending requests need to be canceled. */
 585                         dev->power.request = RPM_REQ_NONE;
 586
 587                         /*
 588                          * Optimization: If the timer is already running and is
 589                          * set to expire at or before the autosuspend delay,
 590                          * avoid the overhead of resetting it.  Just let it
 591                          * expire; pm_suspend_timer_fn() will take care of the
 592                          * rest.
 593                          */
 594                         if (!(dev->power.timer_expires &&
 595                             dev->power.timer_expires <= expires)) {
 596                                 /*
 597                                  * We add a slack of 25% to gather wakeups
 598                                  * without sacrificing the granularity.
 599                                  */
 600                                 u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
 601                                                     (NSEC_PER_MSEC >> 2);
 602
 603                                 dev->power.timer_expires = expires;
 604                                 hrtimer_start_range_ns(&dev->power.suspend_timer,
 605                                                        ns_to_ktime(expires),
 606                                                        slack,
 607                                                        HRTIMER_MODE_ABS);
 608                         }
 609                         dev->power.timer_autosuspends = 1;
 610                         goto out;
 611                 }
 612         }
 613
 614         /* Other scheduled or pending requests need to be canceled. */
 615         pm_runtime_cancel_pending(dev);
 616
 617         if (dev->power.runtime_status == RPM_SUSPENDING) {
 618                 DEFINE_WAIT(wait);
 619
 620                 if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
 621                         retval = -EINPROGRESS;
 622                         goto out;
 623                 }
 624
 625                 if (dev->power.irq_safe) {
 626                         spin_unlock(&dev->power.lock);
 627
 628                         cpu_relax();
 629
 630                         spin_lock(&dev->power.lock);
 631                         goto repeat;
 632                 }
 633
 634                 /* Wait for the other suspend running in parallel with us. */
 635                 for (;;) {
 636                         prepare_to_wait(&dev->power.wait_queue, &wait,
 637                                         TASK_UNINTERRUPTIBLE);
 638                         if (dev->power.runtime_status != RPM_SUSPENDING)
 639                                 break;
 640
 641                         spin_unlock_irq(&dev->power.lock);
 642
 643                         schedule();
 644
 645                         spin_lock_irq(&dev->power.lock);
 646                 }
 647                 finish_wait(&dev->power.wait_queue, &wait);
 648                 goto repeat;
 649         }
 650
 651         if (dev->power.no_callbacks)
 652                 goto no_callback;       /* Assume success. */
 653
 654         /* Carry out an asynchronous or a synchronous suspend. */
 655         if (rpmflags & RPM_ASYNC) {
 656                 dev->power.request = (rpmflags & RPM_AUTO) ?
 657                     RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
 658                 if (!dev->power.request_pending) {
 659                         dev->power.request_pending = true;
 660                         queue_work(pm_wq, &dev->power.work);
 661                 }
 662                 goto out;
 663         }
 664
 665         __update_runtime_status(dev, RPM_SUSPENDING);
 666
 667         callback = RPM_GET_CALLBACK(dev, runtime_suspend);
 668
 669         dev_pm_enable_wake_irq_check(dev, true);
 670         retval = rpm_callback(callback, dev);
 671         if (retval)
 672                 goto fail;
 673
 674         dev_pm_enable_wake_irq_complete(dev);
 675
 676  no_callback:
 677         __update_runtime_status(dev, RPM_SUSPENDED);
 678         pm_runtime_deactivate_timer(dev);
 679
 680         if (dev->parent) {
 681                 parent = dev->parent;
 682                 atomic_add_unless(&parent->power.child_count, -1, 0);
 683         }
 684         wake_up_all(&dev->power.wait_queue);
 685
 686         if (dev->power.deferred_resume) {
 687                 dev->power.deferred_resume = false;
 688                 rpm_resume(dev, 0);
 689                 retval = -EAGAIN;
 690                 goto out;
 691         }
 692
 693         if (dev->power.irq_safe)
 694                 goto out;
 695
 696         /* Maybe the parent is now able to suspend. */
 697         if (parent && !parent->power.ignore_children) {
 698                 spin_unlock(&dev->power.lock);
 699
 700                 spin_lock(&parent->power.lock);
 701                 rpm_idle(parent, RPM_ASYNC);
 702                 spin_unlock(&parent->power.lock);
 703
 704                 spin_lock(&dev->power.lock);
 705         }
 706         /* Maybe the suppliers are now able to suspend. */
 707         if (dev->power.links_count > 0) {
 708                 spin_unlock_irq(&dev->power.lock);
 709
 710                 rpm_suspend_suppliers(dev);
 711
 712                 spin_lock_irq(&dev->power.lock);
 713         }
 714
 715  out:
 716         trace_rpm_return_int(dev, _THIS_IP_, retval);
 717
 718         return retval;
 719
 720  fail:
 721         dev_pm_disable_wake_irq_check(dev, true);
 722         __update_runtime_status(dev, RPM_ACTIVE);
 723         dev->power.deferred_resume = false;
 724         wake_up_all(&dev->power.wait_queue);
 725
 726         if (retval == -EAGAIN || retval == -EBUSY) {
 727                 dev->power.runtime_error = 0;
 728
 729                 /*
 730                  * If the callback routine failed an autosuspend, and
 731                  * if the last_busy time has been updated so that there
 732                  * is a new autosuspend expiration time, automatically
 733                  * reschedule another autosuspend.
 734                  */
 735                 if ((rpmflags & RPM_AUTO) &&
 736                     pm_runtime_autosuspend_expiration(dev) != 0)
 737                         goto repeat;
 738         } else {
 739                 pm_runtime_cancel_pending(dev);
 740         }
 741         goto out;
 742 }
 743
 744 /**
 745  * rpm_resume - Carry out runtime resume of given device.
 746  * @dev: Device to resume.
 747  * @rpmflags: Flag bits.
 748  *
 749  * Check if the device's runtime PM status allows it to be resumed.  Cancel
 750  * any scheduled or pending requests.  If another resume has been started
 751  * earlier, either return immediately or wait for it to finish, depending on the
 752  * RPM_NOWAIT and RPM_ASYNC flags.  Similarly, if there's a suspend running in
 753  * parallel with this function, either tell the other process to resume after
 754  * suspending (deferred_resume) or wait for it to finish.  If the RPM_ASYNC
 755  * flag is set then queue a resume request; otherwise run the
 756  * ->runtime_resume() callback directly.  Queue an idle notification for the
 757  * device if the resume succeeded.
 758  *
 759  * This function must be called under dev->power.lock with interrupts disabled.
 760  */
 761 static int rpm_resume(struct device *dev, int rpmflags)
 762         __releases(&dev->power.lock) __acquires(&dev->power.lock)
 763 {
 764         int (*callback)(struct device *);
 765         struct device *parent = NULL;
 766         int retval = 0;
 767
 768         trace_rpm_resume(dev, rpmflags);
 769
 770  repeat:
 771         if (dev->power.runtime_error) {
 772                 retval = -EINVAL;
 773         } else if (dev->power.disable_depth > 0) {
 774                 if (dev->power.runtime_status == RPM_ACTIVE &&
 775                     dev->power.last_status == RPM_ACTIVE)
 776                         retval = 1;
 777                 else
 778                         retval = -EACCES;
 779         }
 780         if (retval)
 781                 goto out;
 782
 783         /*
 784          * Other scheduled or pending requests need to be canceled.  Small
 785          * optimization: If an autosuspend timer is running, leave it running
 786          * rather than cancelling it now only to restart it again in the near
 787          * future.
 788          */
 789         dev->power.request = RPM_REQ_NONE;
 790         if (!dev->power.timer_autosuspends)
 791                 pm_runtime_deactivate_timer(dev);
 792
 793         if (dev->power.runtime_status == RPM_ACTIVE) {
 794                 retval = 1;
 795                 goto out;
 796         }
 797
 798         if (dev->power.runtime_status == RPM_RESUMING ||
 799             dev->power.runtime_status == RPM_SUSPENDING) {
 800                 DEFINE_WAIT(wait);
 801
 802                 if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
 803                         if (dev->power.runtime_status == RPM_SUSPENDING) {
 804                                 dev->power.deferred_resume = true;
 805                                 if (rpmflags & RPM_NOWAIT)
 806                                         retval = -EINPROGRESS;
 807                         } else {
 808                                 retval = -EINPROGRESS;
 809                         }
 810                         goto out;
 811                 }
 812
 813                 if (dev->power.irq_safe) {
 814                         spin_unlock(&dev->power.lock);
 815
 816                         cpu_relax();
 817
 818                         spin_lock(&dev->power.lock);
 819                         goto repeat;
 820                 }
 821
 822                 /* Wait for the operation carried out in parallel with us. */
 823                 for (;;) {
 824                         prepare_to_wait(&dev->power.wait_queue, &wait,
 825                                         TASK_UNINTERRUPTIBLE);
 826                         if (dev->power.runtime_status != RPM_RESUMING &&
 827                             dev->power.runtime_status != RPM_SUSPENDING)
 828                                 break;
 829
 830                         spin_unlock_irq(&dev->power.lock);
 831
 832                         schedule();
 833
 834                         spin_lock_irq(&dev->power.lock);
 835                 }
 836                 finish_wait(&dev->power.wait_queue, &wait);
 837                 goto repeat;
 838         }
 839
 840         /*
 841          * See if we can skip waking up the parent.  This is safe only if
 842          * power.no_callbacks is set, because otherwise we don't know whether
 843          * the resume will actually succeed.
 844          */
 845         if (dev->power.no_callbacks && !parent && dev->parent) {
 846                 spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
 847                 if (dev->parent->power.disable_depth > 0 ||
 848                     dev->parent->power.ignore_children ||
 849                     dev->parent->power.runtime_status == RPM_ACTIVE) {
 850                         atomic_inc(&dev->parent->power.child_count);
 851                         spin_unlock(&dev->parent->power.lock);
 852                         retval = 1;
 853                         goto no_callback;       /* Assume success. */
 854                 }
 855                 spin_unlock(&dev->parent->power.lock);
 856         }
 857
 858         /* Carry out an asynchronous or a synchronous resume. */
 859         if (rpmflags & RPM_ASYNC) {
 860                 dev->power.request = RPM_REQ_RESUME;
 861                 if (!dev->power.request_pending) {
 862                         dev->power.request_pending = true;
 863                         queue_work(pm_wq, &dev->power.work);
 864                 }
 865                 retval = 0;
 866                 goto out;
 867         }
 868
 869         if (!parent && dev->parent) {
 870                 /*
 871                  * Increment the parent's usage counter and resume it if
 872                  * necessary.  Not needed if dev is irq-safe; then the
 873                  * parent is permanently resumed.
 874                  */
 875                 parent = dev->parent;
 876                 if (dev->power.irq_safe)
 877                         goto skip_parent;
 878
 879                 spin_unlock(&dev->power.lock);
 880
 881                 pm_runtime_get_noresume(parent);
 882
 883                 spin_lock(&parent->power.lock);
 884                 /*
 885                  * Resume the parent if it has runtime PM enabled and not been
 886                  * set to ignore its children.
 887                  */
 888                 if (!parent->power.disable_depth &&
 889                     !parent->power.ignore_children) {
 890                         rpm_resume(parent, 0);
 891                         if (parent->power.runtime_status != RPM_ACTIVE)
 892                                 retval = -EBUSY;
 893                 }
 894                 spin_unlock(&parent->power.lock);
 895
 896                 spin_lock(&dev->power.lock);
 897                 if (retval)
 898                         goto out;
 899
 900                 goto repeat;
 901         }
 902  skip_parent:
 903
 904         if (dev->power.no_callbacks)
 905                 goto no_callback;       /* Assume success. */
 906
 907         __update_runtime_status(dev, RPM_RESUMING);
 908
 909         callback = RPM_GET_CALLBACK(dev, runtime_resume);
 910
 911         dev_pm_disable_wake_irq_check(dev, false);
 912         retval = rpm_callback(callback, dev);
 913         if (retval) {
 914                 __update_runtime_status(dev, RPM_SUSPENDED);
 915                 pm_runtime_cancel_pending(dev);
 916                 dev_pm_enable_wake_irq_check(dev, false);
 917         } else {
 918  no_callback:
 919                 __update_runtime_status(dev, RPM_ACTIVE);
 920                 pm_runtime_mark_last_busy(dev);
 921                 if (parent)
 922                         atomic_inc(&parent->power.child_count);
 923         }
 924         wake_up_all(&dev->power.wait_queue);
 925
 926         if (retval >= 0)
 927                 rpm_idle(dev, RPM_ASYNC);
 928
 929  out:
 930         if (parent && !dev->power.irq_safe) {
 931                 spin_unlock_irq(&dev->power.lock);
 932
 933                 pm_runtime_put(parent);
 934
 935                 spin_lock_irq(&dev->power.lock);
 936         }
 937
 938         trace_rpm_return_int(dev, _THIS_IP_, retval);
 939
 940         return retval;
 941 }
 942
 943 /**
 944  * pm_runtime_work - Universal runtime PM work function.
 945  * @work: Work structure used for scheduling the execution of this function.
 946  *
 947  * Use @work to get the device object the work is to be done for, determine what
 948  * is to be done and execute the appropriate runtime PM function.
 949  */
 950 static void pm_runtime_work(struct work_struct *work)
 951 {
 952         struct device *dev = container_of(work, struct device, power.work);
 953         enum rpm_request req;
 954
 955         spin_lock_irq(&dev->power.lock);
 956
 957         if (!dev->power.request_pending)
 958                 goto out;
 959
 960         req = dev->power.request;
 961         dev->power.request = RPM_REQ_NONE;
 962         dev->power.request_pending = false;
 963
 964         switch (req) {
 965         case RPM_REQ_NONE:
 966                 break;
 967         case RPM_REQ_IDLE:
 968                 rpm_idle(dev, RPM_NOWAIT);
 969                 break;
 970         case RPM_REQ_SUSPEND:
 971                 rpm_suspend(dev, RPM_NOWAIT);
 972                 break;
 973         case RPM_REQ_AUTOSUSPEND:
 974                 rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
 975                 break;
 976         case RPM_REQ_RESUME:
 977                 rpm_resume(dev, RPM_NOWAIT);
 978                 break;
 979         }
 980
 981  out:
 982         spin_unlock_irq(&dev->power.lock);
 983 }
 984
 985 /**
 986  * pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
 987  * @timer: hrtimer used by pm_schedule_suspend().
 988  *
 989  * Check if the time is right and queue a suspend request.
 990  */
 991 static enum hrtimer_restart  pm_suspend_timer_fn(struct hrtimer *timer)
 992 {
 993         struct device *dev = container_of(timer, struct device, power.suspend_timer);
 994         unsigned long flags;
 995         u64 expires;
 996
 997         spin_lock_irqsave(&dev->power.lock, flags);
 998
 999         expires = dev->power.timer_expires;
1000         /*
1001          * If 'expires' is after the current time, we've been called
1002          * too early.
1003          */
1004         if (expires > 0 && expires < ktime_get_mono_fast_ns()) {
1005                 dev->power.timer_expires = 0;
1006                 rpm_suspend(dev, dev->power.timer_autosuspends ?
1007                     (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
1008         }
1009
1010         spin_unlock_irqrestore(&dev->power.lock, flags);
1011
1012         return HRTIMER_NORESTART;
1013 }
1014
1015 /**
1016  * pm_schedule_suspend - Set up a timer to submit a suspend request in future.
1017  * @dev: Device to suspend.
1018  * @delay: Time to wait before submitting a suspend request, in milliseconds.
1019  */
1020 int pm_schedule_suspend(struct device *dev, unsigned int delay)
1021 {
1022         unsigned long flags;
1023         u64 expires;
1024         int retval;
1025
1026         spin_lock_irqsave(&dev->power.lock, flags);
1027
1028         if (!delay) {
1029                 retval = rpm_suspend(dev, RPM_ASYNC);
1030                 goto out;
1031         }
1032
1033         retval = rpm_check_suspend_allowed(dev);
1034         if (retval)
1035                 goto out;
1036
1037         /* Other scheduled or pending requests need to be canceled. */
1038         pm_runtime_cancel_pending(dev);
1039
1040         expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
1041         dev->power.timer_expires = expires;
1042         dev->power.timer_autosuspends = 0;
1043         hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);
1044
1045  out:
1046         spin_unlock_irqrestore(&dev->power.lock, flags);
1047
1048         return retval;
1049 }
1050 EXPORT_SYMBOL_GPL(pm_schedule_suspend);
1051
1052 static int rpm_drop_usage_count(struct device *dev)
1053 {
1054         int ret;
1055
1056         ret = atomic_sub_return(1, &dev->power.usage_count);
1057         if (ret >= 0)
1058                 return ret;
1059
1060         /*
1061          * Because rpm_resume() does not check the usage counter, it will resume
1062          * the device even if the usage counter is 0 or negative, so it is
1063          * sufficient to increment the usage counter here to reverse the change
1064          * made above.
1065          */
1066         atomic_inc(&dev->power.usage_count);
1067         dev_warn(dev, "Runtime PM usage count underflow!\n");
1068         return -EINVAL;
1069 }
1070
1071 /**
1072  * __pm_runtime_idle - Entry point for runtime idle operations.
1073  * @dev: Device to send idle notification for.
1074  * @rpmflags: Flag bits.
1075  *
1076  * If the RPM_GET_PUT flag is set, decrement the device's usage count and
1077  * return immediately if it is larger than zero (if it becomes negative, log a
1078  * warning, increment it, and return an error).  Then carry out an idle
1079  * notification, either synchronous or asynchronous.
1080  *
1081  * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1082  * or if pm_runtime_irq_safe() has been called.
1083  */
1084 int __pm_runtime_idle(struct device *dev, int rpmflags)
1085 {
1086         unsigned long flags;
1087         int retval;
1088
1089         if (rpmflags & RPM_GET_PUT) {
1090                 retval = rpm_drop_usage_count(dev);
1091                 if (retval < 0) {
1092                         return retval;
1093                 } else if (retval > 0) {
1094                         trace_rpm_usage(dev, rpmflags);
1095                         return 0;
1096                 }
1097         }
1098
1099         might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1100
1101         spin_lock_irqsave(&dev->power.lock, flags);
1102         retval = rpm_idle(dev, rpmflags);
1103         spin_unlock_irqrestore(&dev->power.lock, flags);
1104
1105         return retval;
1106 }
1107 EXPORT_SYMBOL_GPL(__pm_runtime_idle);
1108
1109 /**
1110  * __pm_runtime_suspend - Entry point for runtime put/suspend operations.
1111  * @dev: Device to suspend.
1112  * @rpmflags: Flag bits.
1113  *
1114  * If the RPM_GET_PUT flag is set, decrement the device's usage count and
1115  * return immediately if it is larger than zero (if it becomes negative, log a
1116  * warning, increment it, and return an error).  Then carry out a suspend,
1117  * either synchronous or asynchronous.
1118  *
1119  * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1120  * or if pm_runtime_irq_safe() has been called.
1121  */
1122 int __pm_runtime_suspend(struct device *dev, int rpmflags)
1123 {
1124         unsigned long flags;
1125         int retval;
1126
1127         if (rpmflags & RPM_GET_PUT) {
1128                 retval = rpm_drop_usage_count(dev);
1129                 if (retval < 0) {
1130                         return retval;
1131                 } else if (retval > 0) {
1132                         trace_rpm_usage(dev, rpmflags);
1133                         return 0;
1134                 }
1135         }
1136
1137         might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1138
1139         spin_lock_irqsave(&dev->power.lock, flags);
1140         retval = rpm_suspend(dev, rpmflags);
1141         spin_unlock_irqrestore(&dev->power.lock, flags);
1142
1143         return retval;
1144 }
1145 EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
1146
1147 /**
1148  * __pm_runtime_resume - Entry point for runtime resume operations.
1149  * @dev: Device to resume.
1150  * @rpmflags: Flag bits.
1151  *
1152  * If the RPM_GET_PUT flag is set, increment the device's usage count.  Then
1153  * carry out a resume, either synchronous or asynchronous.
1154  *
1155  * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1156  * or if pm_runtime_irq_safe() has been called.
1157  */
1158 int __pm_runtime_resume(struct device *dev, int rpmflags)
1159 {
1160         unsigned long flags;
1161         int retval;
1162
1163         might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
1164                         dev->power.runtime_status != RPM_ACTIVE);
1165
1166         if (rpmflags & RPM_GET_PUT)
1167                 atomic_inc(&dev->power.usage_count);
1168
1169         spin_lock_irqsave(&dev->power.lock, flags);
1170         retval = rpm_resume(dev, rpmflags);
1171         spin_unlock_irqrestore(&dev->power.lock, flags);
1172
1173         return retval;
1174 }
1175 EXPORT_SYMBOL_GPL(__pm_runtime_resume);
1176
1177 /**
1178  * pm_runtime_get_if_active - Conditionally bump up device usage counter.
1179  * @dev: Device to handle.
1180  * @ign_usage_count: Whether or not to look at the current usage counter value.
1181  *
1182  * Return -EINVAL if runtime PM is disabled for @dev.
1183  *
1184  * Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either
1185  * @ign_usage_count is %true or the runtime PM usage counter of @dev is not
1186  * zero, increment the usage counter of @dev and return 1. Otherwise, return 0
1187  * without changing the usage counter.
1188  *
1189  * If @ign_usage_count is %true, this function can be used to prevent suspending
1190  * the device when its runtime PM status is %RPM_ACTIVE.
1191  *
1192  * If @ign_usage_count is %false, this function can be used to prevent
1193  * suspending the device when both its runtime PM status is %RPM_ACTIVE and its
1194  * runtime PM usage counter is not zero.
1195  *
1196  * The caller is responsible for decrementing the runtime PM usage counter of
1197  * @dev after this function has returned a positive value for it.
1198  */
1199 int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count)
1200 {
1201         unsigned long flags;
1202         int retval;
1203
1204         spin_lock_irqsave(&dev->power.lock, flags);
1205         if (dev->power.disable_depth > 0) {
1206                 retval = -EINVAL;
1207         } else if (dev->power.runtime_status != RPM_ACTIVE) {
1208                 retval = 0;
1209         } else if (ign_usage_count) {
1210                 retval = 1;
1211                 atomic_inc(&dev->power.usage_count);
1212         } else {
1213                 retval = atomic_inc_not_zero(&dev->power.usage_count);
1214         }
1215         trace_rpm_usage(dev, 0);
1216         spin_unlock_irqrestore(&dev->power.lock, flags);
1217
1218         return retval;
1219 }
1220 EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
1221
1222 /**
1223  * __pm_runtime_set_status - Set runtime PM status of a device.
1224  * @dev: Device to handle.
1225  * @status: New runtime PM status of the device.
1226  *
1227  * If runtime PM of the device is disabled or its power.runtime_error field is
1228  * different from zero, the status may be changed either to RPM_ACTIVE, or to
1229  * RPM_SUSPENDED, as long as that reflects the actual state of the device.
1230  * However, if the device has a parent and the parent is not active, and the
1231  * parent's power.ignore_children flag is unset, the device's status cannot be
1232  * set to RPM_ACTIVE, so -EBUSY is returned in that case.
1233  *
1234  * If successful, __pm_runtime_set_status() clears the power.runtime_error field
1235  * and the device parent's counter of unsuspended children is modified to
1236  * reflect the new status.  If the new status is RPM_SUSPENDED, an idle
1237  * notification request for the parent is submitted.
1238  *
1239  * If @dev has any suppliers (as reflected by device links to them), and @status
1240  * is RPM_ACTIVE, they will be activated upfront and if the activation of one
1241  * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
1242  * of the @status value) and the suppliers will be deacticated on exit.  The
1243  * error returned by the failing supplier activation will be returned in that
1244  * case.
1245  */
1246 int __pm_runtime_set_status(struct device *dev, unsigned int status)
1247 {
1248         struct device *parent = dev->parent;
1249         bool notify_parent = false;
1250         unsigned long flags;
1251         int error = 0;
1252
1253         if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
1254                 return -EINVAL;
1255
1256         spin_lock_irqsave(&dev->power.lock, flags);
1257
1258         /*
1259          * Prevent PM-runtime from being enabled for the device or return an
1260          * error if it is enabled already and working.
1261          */
1262         if (dev->power.runtime_error || dev->power.disable_depth)
1263                 dev->power.disable_depth++;
1264         else
1265                 error = -EAGAIN;
1266
1267         spin_unlock_irqrestore(&dev->power.lock, flags);
1268
1269         if (error)
1270                 return error;
1271
1272         /*
1273          * If the new status is RPM_ACTIVE, the suppliers can be activated
1274          * upfront regardless of the current status, because next time
1275          * rpm_put_suppliers() runs, the rpm_active refcounts of the links
1276          * involved will be dropped down to one anyway.
1277          */
1278         if (status == RPM_ACTIVE) {
1279                 int idx = device_links_read_lock();
1280
1281                 error = rpm_get_suppliers(dev);
1282                 if (error)
1283                         status = RPM_SUSPENDED;
1284
1285                 device_links_read_unlock(idx);
1286         }
1287
1288         spin_lock_irqsave(&dev->power.lock, flags);
1289
1290         if (dev->power.runtime_status == status || !parent)
1291                 goto out_set;
1292
1293         if (status == RPM_SUSPENDED) {
1294                 atomic_add_unless(&parent->power.child_count, -1, 0);
1295                 notify_parent = !parent->power.ignore_children;
1296         } else {
1297                 spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
1298
1299                 /*
1300                  * It is invalid to put an active child under a parent that is
1301                  * not active, has runtime PM enabled and the
1302                  * 'power.ignore_children' flag unset.
1303                  */
1304                 if (!parent->power.disable_depth &&
1305                     !parent->power.ignore_children &&
1306                     parent->power.runtime_status != RPM_ACTIVE) {
1307                         dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
1308                                 dev_name(dev),
1309                                 dev_name(parent));
1310                         error = -EBUSY;
1311                 } else if (dev->power.runtime_status == RPM_SUSPENDED) {
1312                         atomic_inc(&parent->power.child_count);
1313                 }
1314
1315                 spin_unlock(&parent->power.lock);
1316
1317                 if (error) {
1318                         status = RPM_SUSPENDED;
1319                         goto out;
1320                 }
1321         }
1322
1323  out_set:
1324         __update_runtime_status(dev, status);
1325         if (!error)
1326                 dev->power.runtime_error = 0;
1327
1328  out:
1329         spin_unlock_irqrestore(&dev->power.lock, flags);
1330
1331         if (notify_parent)
1332                 pm_request_idle(parent);
1333
1334         if (status == RPM_SUSPENDED) {
1335                 int idx = device_links_read_lock();
1336
1337                 rpm_put_suppliers(dev);
1338
1339                 device_links_read_unlock(idx);
1340         }
1341
1342         pm_runtime_enable(dev);
1343
1344         return error;
1345 }
1346 EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
1347
1348 /**
1349  * __pm_runtime_barrier - Cancel pending requests and wait for completions.
1350  * @dev: Device to handle.
1351  *
1352  * Flush all pending requests for the device from pm_wq and wait for all
1353  * runtime PM operations involving the device in progress to complete.
1354  *
1355  * Should be called under dev->power.lock with interrupts disabled.
1356  */
1357 static void __pm_runtime_barrier(struct device *dev)
1358 {
1359         pm_runtime_deactivate_timer(dev);
1360
1361         if (dev->power.request_pending) {
1362                 dev->power.request = RPM_REQ_NONE;
1363                 spin_unlock_irq(&dev->power.lock);
1364
1365                 cancel_work_sync(&dev->power.work);
1366
1367                 spin_lock_irq(&dev->power.lock);
1368                 dev->power.request_pending = false;
1369         }
1370
1371         if (dev->power.runtime_status == RPM_SUSPENDING ||
1372             dev->power.runtime_status == RPM_RESUMING ||
1373             dev->power.idle_notification) {
1374                 DEFINE_WAIT(wait);
1375
1376                 /* Suspend, wake-up or idle notification in progress. */
1377                 for (;;) {
1378                         prepare_to_wait(&dev->power.wait_queue, &wait,
1379                                         TASK_UNINTERRUPTIBLE);
1380                         if (dev->power.runtime_status != RPM_SUSPENDING
1381                             && dev->power.runtime_status != RPM_RESUMING
1382                             && !dev->power.idle_notification)
1383                                 break;
1384                         spin_unlock_irq(&dev->power.lock);
1385
1386                         schedule();
1387
1388                         spin_lock_irq(&dev->power.lock);
1389                 }
1390                 finish_wait(&dev->power.wait_queue, &wait);
1391         }
1392 }
1393
1394 /**
1395  * pm_runtime_barrier - Flush pending requests and wait for completions.
1396  * @dev: Device to handle.
1397  *
1398  * Prevent the device from being suspended by incrementing its usage counter and
1399  * if there's a pending resume request for the device, wake the device up.
1400  * Next, make sure that all pending requests for the device have been flushed
1401  * from pm_wq and wait for all runtime PM operations involving the device in
1402  * progress to complete.
1403  *
1404  * Return value:
1405  * 1, if there was a resume request pending and the device had to be woken up,
1406  * 0, otherwise
1407  */
1408 int pm_runtime_barrier(struct device *dev)
1409 {
1410         int retval = 0;
1411
1412         pm_runtime_get_noresume(dev);
1413         spin_lock_irq(&dev->power.lock);
1414
1415         if (dev->power.request_pending
1416             && dev->power.request == RPM_REQ_RESUME) {
1417                 rpm_resume(dev, 0);
1418                 retval = 1;
1419         }
1420
1421         __pm_runtime_barrier(dev);
1422
1423         spin_unlock_irq(&dev->power.lock);
1424         pm_runtime_put_noidle(dev);
1425
1426         return retval;
1427 }
1428 EXPORT_SYMBOL_GPL(pm_runtime_barrier);
1429
1430 /**
1431  * __pm_runtime_disable - Disable runtime PM of a device.
1432  * @dev: Device to handle.
1433  * @check_resume: If set, check if there's a resume request for the device.
1434  *
1435  * Increment power.disable_depth for the device and if it was zero previously,
1436  * cancel all pending runtime PM requests for the device and wait for all
1437  * operations in progress to complete.  The device can be either active or
1438  * suspended after its runtime PM has been disabled.
1439  *
1440  * If @check_resume is set and there's a resume request pending when
1441  * __pm_runtime_disable() is called and power.disable_depth is zero, the
1442  * function will wake up the device before disabling its runtime PM.
1443  */
1444 void __pm_runtime_disable(struct device *dev, bool check_resume)
1445 {
1446         spin_lock_irq(&dev->power.lock);
1447
1448         if (dev->power.disable_depth > 0) {
1449                 dev->power.disable_depth++;
1450                 goto out;
1451         }
1452
1453         /*
1454          * Wake up the device if there's a resume request pending, because that
1455          * means there probably is some I/O to process and disabling runtime PM
1456          * shouldn't prevent the device from processing the I/O.
1457          */
1458         if (check_resume && dev->power.request_pending &&
1459             dev->power.request == RPM_REQ_RESUME) {
1460                 /*
1461                  * Prevent suspends and idle notifications from being carried
1462                  * out after we have woken up the device.
1463                  */
1464                 pm_runtime_get_noresume(dev);
1465
1466                 rpm_resume(dev, 0);
1467
1468                 pm_runtime_put_noidle(dev);
1469         }
1470
1471         /* Update time accounting before disabling PM-runtime. */
1472         update_pm_runtime_accounting(dev);
1473
1474         if (!dev->power.disable_depth++) {
1475                 __pm_runtime_barrier(dev);
1476                 dev->power.last_status = dev->power.runtime_status;
1477         }
1478
1479  out:
1480         spin_unlock_irq(&dev->power.lock);
1481 }
1482 EXPORT_SYMBOL_GPL(__pm_runtime_disable);
1483
1484 /**
1485  * pm_runtime_enable - Enable runtime PM of a device.
1486  * @dev: Device to handle.
1487  */
1488 void pm_runtime_enable(struct device *dev)
1489 {
1490         unsigned long flags;
1491
1492         spin_lock_irqsave(&dev->power.lock, flags);
1493
1494         if (!dev->power.disable_depth) {
1495                 dev_warn(dev, "Unbalanced %s!\n", __func__);
1496                 goto out;
1497         }
1498
1499         if (--dev->power.disable_depth > 0)
1500                 goto out;
1501
1502         dev->power.last_status = RPM_INVALID;
1503         dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
1504
1505         if (dev->power.runtime_status == RPM_SUSPENDED &&
1506             !dev->power.ignore_children &&
1507             atomic_read(&dev->power.child_count) > 0)
1508                 dev_warn(dev, "Enabling runtime PM for inactive device with active children\n");
1509
1510 out:
1511         spin_unlock_irqrestore(&dev->power.lock, flags);
1512 }
1513 EXPORT_SYMBOL_GPL(pm_runtime_enable);
1514
1515 static void pm_runtime_disable_action(void *data)
1516 {
1517         pm_runtime_dont_use_autosuspend(data);
1518         pm_runtime_disable(data);
1519 }
1520
1521 /**
1522  * devm_pm_runtime_enable - devres-enabled version of pm_runtime_enable.
1523  *
1524  * NOTE: this will also handle calling pm_runtime_dont_use_autosuspend() for
1525  * you at driver exit time if needed.
1526  *
1527  * @dev: Device to handle.
1528  */
1529 int devm_pm_runtime_enable(struct device *dev)
1530 {
1531         pm_runtime_enable(dev);
1532
1533         return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev);
1534 }
1535 EXPORT_SYMBOL_GPL(devm_pm_runtime_enable);
1536
1537 /**
1538  * pm_runtime_forbid - Block runtime PM of a device.
1539  * @dev: Device to handle.
1540  *
1541  * Increase the device's usage count and clear its power.runtime_auto flag,
1542  * so that it cannot be suspended at run time until pm_runtime_allow() is called
1543  * for it.
1544  */
1545 void pm_runtime_forbid(struct device *dev)
1546 {
1547         spin_lock_irq(&dev->power.lock);
1548         if (!dev->power.runtime_auto)
1549                 goto out;
1550
1551         dev->power.runtime_auto = false;
1552         atomic_inc(&dev->power.usage_count);
1553         rpm_resume(dev, 0);
1554
1555  out:
1556         spin_unlock_irq(&dev->power.lock);
1557 }
1558 EXPORT_SYMBOL_GPL(pm_runtime_forbid);
1559
1560 /**
1561  * pm_runtime_allow - Unblock runtime PM of a device.
1562  * @dev: Device to handle.
1563  *
1564  * Decrease the device's usage count and set its power.runtime_auto flag.
1565  */
1566 void pm_runtime_allow(struct device *dev)
1567 {
1568         int ret;
1569
1570         spin_lock_irq(&dev->power.lock);
1571         if (dev->power.runtime_auto)
1572                 goto out;
1573
1574         dev->power.runtime_auto = true;
1575         ret = rpm_drop_usage_count(dev);
1576         if (ret == 0)
1577                 rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
1578         else if (ret > 0)
1579                 trace_rpm_usage(dev, RPM_AUTO | RPM_ASYNC);
1580
1581  out:
1582         spin_unlock_irq(&dev->power.lock);
1583 }
1584 EXPORT_SYMBOL_GPL(pm_runtime_allow);
1585
1586 /**
1587  * pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
1588  * @dev: Device to handle.
1589  *
1590  * Set the power.no_callbacks flag, which tells the PM core that this
1591  * device is power-managed through its parent and has no runtime PM
1592  * callbacks of its own.  The runtime sysfs attributes will be removed.
1593  */
1594 void pm_runtime_no_callbacks(struct device *dev)
1595 {
1596         spin_lock_irq(&dev->power.lock);
1597         dev->power.no_callbacks = 1;
1598         spin_unlock_irq(&dev->power.lock);
1599         if (device_is_registered(dev))
1600                 rpm_sysfs_remove(dev);
1601 }
1602 EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
1603
1604 /**
1605  * pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
1606  * @dev: Device to handle
1607  *
1608  * Set the power.irq_safe flag, which tells the PM core that the
1609  * ->runtime_suspend() and ->runtime_resume() callbacks for this device should
1610  * always be invoked with the spinlock held and interrupts disabled.  It also
1611  * causes the parent's usage counter to be permanently incremented, preventing
1612  * the parent from runtime suspending -- otherwise an irq-safe child might have
1613  * to wait for a non-irq-safe parent.
1614  */
1615 void pm_runtime_irq_safe(struct device *dev)
1616 {
1617         if (dev->parent)
1618                 pm_runtime_get_sync(dev->parent);
1619
1620         spin_lock_irq(&dev->power.lock);
1621         dev->power.irq_safe = 1;
1622         spin_unlock_irq(&dev->power.lock);
1623 }
1624 EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
1625
1626 /**
1627  * update_autosuspend - Handle a change to a device's autosuspend settings.
1628  * @dev: Device to handle.
1629  * @old_delay: The former autosuspend_delay value.
1630  * @old_use: The former use_autosuspend value.
1631  *
1632  * Prevent runtime suspend if the new delay is negative and use_autosuspend is
1633  * set; otherwise allow it.  Send an idle notification if suspends are allowed.
1634  *
1635  * This function must be called under dev->power.lock with interrupts disabled.
1636  */
1637 static void update_autosuspend(struct device *dev, int old_delay, int old_use)
1638 {
1639         int delay = dev->power.autosuspend_delay;
1640
1641         /* Should runtime suspend be prevented now? */
1642         if (dev->power.use_autosuspend && delay < 0) {
1643
1644                 /* If it used to be allowed then prevent it. */
1645                 if (!old_use || old_delay >= 0) {
1646                         atomic_inc(&dev->power.usage_count);
1647                         rpm_resume(dev, 0);
1648                 } else {
1649                         trace_rpm_usage(dev, 0);
1650                 }
1651         }
1652
1653         /* Runtime suspend should be allowed now. */
1654         else {
1655
1656                 /* If it used to be prevented then allow it. */
1657                 if (old_use && old_delay < 0)
1658                         atomic_dec(&dev->power.usage_count);
1659
1660                 /* Maybe we can autosuspend now. */
1661                 rpm_idle(dev, RPM_AUTO);
1662         }
1663 }
1664
1665 /**
1666  * pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
1667  * @dev: Device to handle.
1668  * @delay: Value of the new delay in milliseconds.
1669  *
1670  * Set the device's power.autosuspend_delay value.  If it changes to negative
1671  * and the power.use_autosuspend flag is set, prevent runtime suspends.  If it
1672  * changes the other way, allow runtime suspends.
1673  */
1674 void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
1675 {
1676         int old_delay, old_use;
1677
1678         spin_lock_irq(&dev->power.lock);
1679         old_delay = dev->power.autosuspend_delay;
1680         old_use = dev->power.use_autosuspend;
1681         dev->power.autosuspend_delay = delay;
1682         update_autosuspend(dev, old_delay, old_use);
1683         spin_unlock_irq(&dev->power.lock);
1684 }
1685 EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
1686
1687 /**
1688  * __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
1689  * @dev: Device to handle.
1690  * @use: New value for use_autosuspend.
1691  *
1692  * Set the device's power.use_autosuspend flag, and allow or prevent runtime
1693  * suspends as needed.
1694  */
1695 void __pm_runtime_use_autosuspend(struct device *dev, bool use)
1696 {
1697         int old_delay, old_use;
1698
1699         spin_lock_irq(&dev->power.lock);
1700         old_delay = dev->power.autosuspend_delay;
1701         old_use = dev->power.use_autosuspend;
1702         dev->power.use_autosuspend = use;
1703         update_autosuspend(dev, old_delay, old_use);
1704         spin_unlock_irq(&dev->power.lock);
1705 }
1706 EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
1707
1708 /**
1709  * pm_runtime_init - Initialize runtime PM fields in given device object.
1710  * @dev: Device object to initialize.
1711  */
1712 void pm_runtime_init(struct device *dev)
1713 {
1714         dev->power.runtime_status = RPM_SUSPENDED;
1715         dev->power.last_status = RPM_INVALID;
1716         dev->power.idle_notification = false;
1717
1718         dev->power.disable_depth = 1;
1719         atomic_set(&dev->power.usage_count, 0);
1720
1721         dev->power.runtime_error = 0;
1722
1723         atomic_set(&dev->power.child_count, 0);
1724         pm_suspend_ignore_children(dev, false);
1725         dev->power.runtime_auto = true;
1726
1727         dev->power.request_pending = false;
1728         dev->power.request = RPM_REQ_NONE;
1729         dev->power.deferred_resume = false;
1730         dev->power.needs_force_resume = 0;
1731         INIT_WORK(&dev->power.work, pm_runtime_work);
1732
1733         dev->power.timer_expires = 0;
1734         hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1735         dev->power.suspend_timer.function = pm_suspend_timer_fn;
1736
1737         init_waitqueue_head(&dev->power.wait_queue);
1738 }
1739
1740 /**
1741  * pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
1742  * @dev: Device object to re-initialize.
1743  */
1744 void pm_runtime_reinit(struct device *dev)
1745 {
1746         if (!pm_runtime_enabled(dev)) {
1747                 if (dev->power.runtime_status == RPM_ACTIVE)
1748                         pm_runtime_set_suspended(dev);
1749                 if (dev->power.irq_safe) {
1750                         spin_lock_irq(&dev->power.lock);
1751                         dev->power.irq_safe = 0;
1752                         spin_unlock_irq(&dev->power.lock);
1753                         if (dev->parent)
1754                                 pm_runtime_put(dev->parent);
1755                 }
1756         }
1757 }
1758
1759 /**
1760  * pm_runtime_remove - Prepare for removing a device from device hierarchy.
1761  * @dev: Device object being removed from device hierarchy.
1762  */
1763 void pm_runtime_remove(struct device *dev)
1764 {
1765         __pm_runtime_disable(dev, false);
1766         pm_runtime_reinit(dev);
1767 }
1768
1769 /**
1770  * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
1771  * @dev: Consumer device.
1772  */
1773 void pm_runtime_get_suppliers(struct device *dev)
1774 {
1775         struct device_link *link;
1776         int idx;
1777
1778         idx = device_links_read_lock();
1779
1780         list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1781                                 device_links_read_lock_held())
1782                 if (link->flags & DL_FLAG_PM_RUNTIME) {
1783                         link->supplier_preactivated = true;
1784                         pm_runtime_get_sync(link->supplier);
1785                 }
1786
1787         device_links_read_unlock(idx);
1788 }
1789
1790 /**
1791  * pm_runtime_put_suppliers - Drop references to supplier devices.
1792  * @dev: Consumer device.
1793  */
1794 void pm_runtime_put_suppliers(struct device *dev)
1795 {
1796         struct device_link *link;
1797         int idx;
1798
1799         idx = device_links_read_lock();
1800
1801         list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1802                                 device_links_read_lock_held())
1803                 if (link->supplier_preactivated) {
1804                         link->supplier_preactivated = false;
1805                         pm_runtime_put(link->supplier);
1806                 }
1807
1808         device_links_read_unlock(idx);
1809 }
1810
1811 void pm_runtime_new_link(struct device *dev)
1812 {
1813         spin_lock_irq(&dev->power.lock);
1814         dev->power.links_count++;
1815         spin_unlock_irq(&dev->power.lock);
1816 }
1817
1818 static void pm_runtime_drop_link_count(struct device *dev)
1819 {
1820         spin_lock_irq(&dev->power.lock);
1821         WARN_ON(dev->power.links_count == 0);
1822         dev->power.links_count--;
1823         spin_unlock_irq(&dev->power.lock);
1824 }
1825
1826 /**
1827  * pm_runtime_drop_link - Prepare for device link removal.
1828  * @link: Device link going away.
1829  *
1830  * Drop the link count of the consumer end of @link and decrement the supplier
1831  * device's runtime PM usage counter as many times as needed to drop all of the
1832  * PM runtime reference to it from the consumer.
1833  */
1834 void pm_runtime_drop_link(struct device_link *link)
1835 {
1836         if (!(link->flags & DL_FLAG_PM_RUNTIME))
1837                 return;
1838
1839         pm_runtime_drop_link_count(link->consumer);
1840         pm_runtime_release_supplier(link);
1841         pm_request_idle(link->supplier);
1842 }
1843
1844 static bool pm_runtime_need_not_resume(struct device *dev)
1845 {
1846         return atomic_read(&dev->power.usage_count) <= 1 &&
1847                 (atomic_read(&dev->power.child_count) == 0 ||
1848                  dev->power.ignore_children);
1849 }
1850
1851 /**
1852  * pm_runtime_force_suspend - Force a device into suspend state if needed.
1853  * @dev: Device to suspend.
1854  *
1855  * Disable runtime PM so we safely can check the device's runtime PM status and
1856  * if it is active, invoke its ->runtime_suspend callback to suspend it and
1857  * change its runtime PM status field to RPM_SUSPENDED.  Also, if the device's
1858  * usage and children counters don't indicate that the device was in use before
1859  * the system-wide transition under way, decrement its parent's children counter
1860  * (if there is a parent).  Keep runtime PM disabled to preserve the state
1861  * unless we encounter errors.
1862  *
1863  * Typically this function may be invoked from a system suspend callback to make
1864  * sure the device is put into low power state and it should only be used during
1865  * system-wide PM transitions to sleep states.  It assumes that the analogous
1866  * pm_runtime_force_resume() will be used to resume the device.
1867  *
1868  * Do not use with DPM_FLAG_SMART_SUSPEND as this can lead to an inconsistent
1869  * state where this function has called the ->runtime_suspend callback but the
1870  * PM core marks the driver as runtime active.
1871  */
1872 int pm_runtime_force_suspend(struct device *dev)
1873 {
1874         int (*callback)(struct device *);
1875         int ret;
1876
1877         pm_runtime_disable(dev);
1878         if (pm_runtime_status_suspended(dev))
1879                 return 0;
1880
1881         callback = RPM_GET_CALLBACK(dev, runtime_suspend);
1882
1883         dev_pm_enable_wake_irq_check(dev, true);
1884         ret = callback ? callback(dev) : 0;
1885         if (ret)
1886                 goto err;
1887
1888         dev_pm_enable_wake_irq_complete(dev);
1889
1890         /*
1891          * If the device can stay in suspend after the system-wide transition
1892          * to the working state that will follow, drop the children counter of
1893          * its parent, but set its status to RPM_SUSPENDED anyway in case this
1894          * function will be called again for it in the meantime.
1895          */
1896         if (pm_runtime_need_not_resume(dev)) {
1897                 pm_runtime_set_suspended(dev);
1898         } else {
1899                 __update_runtime_status(dev, RPM_SUSPENDED);
1900                 dev->power.needs_force_resume = 1;
1901         }
1902
1903         return 0;
1904
1905 err:
1906         dev_pm_disable_wake_irq_check(dev, true);
1907         pm_runtime_enable(dev);
1908         return ret;
1909 }
1910 EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
1911
1912 /**
1913  * pm_runtime_force_resume - Force a device into resume state if needed.
1914  * @dev: Device to resume.
1915  *
1916  * Prior invoking this function we expect the user to have brought the device
1917  * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
1918  * those actions and bring the device into full power, if it is expected to be
1919  * used on system resume.  In the other case, we defer the resume to be managed
1920  * via runtime PM.
1921  *
1922  * Typically this function may be invoked from a system resume callback.
1923  */
1924 int pm_runtime_force_resume(struct device *dev)
1925 {
1926         int (*callback)(struct device *);
1927         int ret = 0;
1928
1929         if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume)
1930                 goto out;
1931
1932         /*
1933          * The value of the parent's children counter is correct already, so
1934          * just update the status of the device.
1935          */
1936         __update_runtime_status(dev, RPM_ACTIVE);
1937
1938         callback = RPM_GET_CALLBACK(dev, runtime_resume);
1939
1940         dev_pm_disable_wake_irq_check(dev, false);
1941         ret = callback ? callback(dev) : 0;
1942         if (ret) {
1943                 pm_runtime_set_suspended(dev);
1944                 dev_pm_enable_wake_irq_check(dev, false);
1945                 goto out;
1946         }
1947
1948         pm_runtime_mark_last_busy(dev);
1949 out:
1950         dev->power.needs_force_resume = 0;
1951         pm_runtime_enable(dev);
1952         return ret;
1953 }
1954 EXPORT_SYMBOL_GPL(pm_runtime_force_resume);