drivers/gpu/drm/i915/intel_wakeref.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/wait_bit.h>
   8
   9 #include "intel_runtime_pm.h"
  10 #include "intel_wakeref.h"
  11 #include "i915_drv.h"
  12
  13 int __intel_wakeref_get_first(struct intel_wakeref *wf)
  14 {
  15         intel_wakeref_t wakeref;
  16         int ret = 0;
  17
  18         wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm);
  19         /*
  20          * Treat get/put as different subclasses, as we may need to run
  21          * the put callback from under the shrinker and do not want to
  22          * cross-contanimate that callback with any extra work performed
  23          * upon acquiring the wakeref.
  24          */
  25         mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
  26
  27         if (!atomic_read(&wf->count)) {
  28                 INTEL_WAKEREF_BUG_ON(wf->wakeref);
  29                 wf->wakeref = wakeref;
  30                 wakeref = 0;
  31
  32                 ret = wf->ops->get(wf);
  33                 if (ret) {
  34                         wakeref = xchg(&wf->wakeref, 0);
  35                         wake_up_var(&wf->wakeref);
  36                         goto unlock;
  37                 }
  38
  39                 smp_mb__before_atomic(); /* release wf->count */
  40         }
  41
  42         atomic_inc(&wf->count);
  43         INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
  44
  45 unlock:
  46         mutex_unlock(&wf->mutex);
  47         if (unlikely(wakeref))
  48                 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
  49
  50         return ret;
  51 }
  52
  53 static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
  54 {
  55         intel_wakeref_t wakeref = 0;
  56
  57         INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
  58         if (unlikely(!atomic_dec_and_test(&wf->count)))
  59                 goto unlock;
  60
  61         /* ops->put() must reschedule its own release on error/deferral */
  62         if (likely(!wf->ops->put(wf))) {
  63                 INTEL_WAKEREF_BUG_ON(!wf->wakeref);
  64                 wakeref = xchg(&wf->wakeref, 0);
  65                 wake_up_var(&wf->wakeref);
  66         }
  67
  68 unlock:
  69         mutex_unlock(&wf->mutex);
  70         if (wakeref)
  71                 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
  72 }
  73
  74 void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
  75 {
  76         INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));
  77
  78         /* Assume we are not in process context and so cannot sleep. */
  79         if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
  80                 mod_delayed_work(wf->i915->unordered_wq, &wf->work,
  81                                  FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
  82                 return;
  83         }
  84
  85         ____intel_wakeref_put_last(wf);
  86 }
  87
  88 static void __intel_wakeref_put_work(struct work_struct *wrk)
  89 {
  90         struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);
  91
  92         if (atomic_add_unless(&wf->count, -1, 1))
  93                 return;
  94
  95         mutex_lock(&wf->mutex);
  96         ____intel_wakeref_put_last(wf);
  97 }
  98
  99 void __intel_wakeref_init(struct intel_wakeref *wf,
 100                           struct drm_i915_private *i915,
 101                           const struct intel_wakeref_ops *ops,
 102                           struct intel_wakeref_lockclass *key)
 103 {
 104         wf->i915 = i915;
 105         wf->ops = ops;
 106
 107         __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
 108         atomic_set(&wf->count, 0);
 109         wf->wakeref = 0;
 110
 111         INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);
 112         lockdep_init_map(&wf->work.work.lockdep_map,
 113                          "wakeref.work", &key->work, 0);
 114 }
 115
 116 int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
 117 {
 118         int err;
 119
 120         might_sleep();
 121
 122         err = wait_var_event_killable(&wf->wakeref,
 123                                       !intel_wakeref_is_active(wf));
 124         if (err)
 125                 return err;
 126
 127         intel_wakeref_unlock_wait(wf);
 128         return 0;
 129 }
 130
 131 static void wakeref_auto_timeout(struct timer_list *t)
 132 {
 133         struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
 134         intel_wakeref_t wakeref;
 135         unsigned long flags;
 136
 137         if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
 138                 return;
 139
 140         wakeref = fetch_and_zero(&wf->wakeref);
 141         spin_unlock_irqrestore(&wf->lock, flags);
 142
 143         intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
 144 }
 145
 146 void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
 147                              struct drm_i915_private *i915)
 148 {
 149         spin_lock_init(&wf->lock);
 150         timer_setup(&wf->timer, wakeref_auto_timeout, 0);
 151         refcount_set(&wf->count, 0);
 152         wf->wakeref = 0;
 153         wf->i915 = i915;
 154 }
 155
 156 void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
 157 {
 158         unsigned long flags;
 159
 160         if (!timeout) {
 161                 if (del_timer_sync(&wf->timer))
 162                         wakeref_auto_timeout(&wf->timer);
 163                 return;
 164         }
 165
 166         /* Our mission is that we only extend an already active wakeref */
 167         assert_rpm_wakelock_held(&wf->i915->runtime_pm);
 168
 169         if (!refcount_inc_not_zero(&wf->count)) {
 170                 spin_lock_irqsave(&wf->lock, flags);
 171                 if (!refcount_inc_not_zero(&wf->count)) {
 172                         INTEL_WAKEREF_BUG_ON(wf->wakeref);
 173                         wf->wakeref =
 174                                 intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm);
 175                         refcount_set(&wf->count, 1);
 176                 }
 177                 spin_unlock_irqrestore(&wf->lock, flags);
 178         }
 179
 180         /*
 181          * If we extend a pending timer, we will only get a single timer
 182          * callback and so need to cancel the local inc by running the
 183          * elided callback to keep the wf->count balanced.
 184          */
 185         if (mod_timer(&wf->timer, jiffies + timeout))
 186                 wakeref_auto_timeout(&wf->timer);
 187 }
 188
 189 void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
 190 {
 191         intel_wakeref_auto(wf, 0);
 192         INTEL_WAKEREF_BUG_ON(wf->wakeref);
 193 }