drivers/gpu/drm/i915/gt/intel_breadcrumbs.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/kthread.h>
  26 #include <trace/events/dma_fence.h>
  27 #include <uapi/linux/sched/types.h>
  28
  29 #include "i915_drv.h"
  30
  31 static void irq_enable(struct intel_engine_cs *engine)
  32 {
  33         if (!engine->irq_enable)
  34                 return;
  35
  36         /* Caller disables interrupts */
  37         spin_lock(&engine->i915->irq_lock);
  38         engine->irq_enable(engine);
  39         spin_unlock(&engine->i915->irq_lock);
  40 }
  41
  42 static void irq_disable(struct intel_engine_cs *engine)
  43 {
  44         if (!engine->irq_disable)
  45                 return;
  46
  47         /* Caller disables interrupts */
  48         spin_lock(&engine->i915->irq_lock);
  49         engine->irq_disable(engine);
  50         spin_unlock(&engine->i915->irq_lock);
  51 }
  52
  53 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
  54 {
  55         lockdep_assert_held(&b->irq_lock);
  56
  57         GEM_BUG_ON(!b->irq_enabled);
  58         if (!--b->irq_enabled)
  59                 irq_disable(container_of(b,
  60                                          struct intel_engine_cs,
  61                                          breadcrumbs));
  62
  63         b->irq_armed = false;
  64 }
  65
  66 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
  67 {
  68         struct intel_breadcrumbs *b = &engine->breadcrumbs;
  69
  70         if (!b->irq_armed)
  71                 return;
  72
  73         spin_lock_irq(&b->irq_lock);
  74         if (b->irq_armed)
  75                 __intel_breadcrumbs_disarm_irq(b);
  76         spin_unlock_irq(&b->irq_lock);
  77 }
  78
  79 static inline bool __request_completed(const struct i915_request *rq)
  80 {
  81         return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
  82 }
  83
  84 __maybe_unused static bool
  85 check_signal_order(struct intel_context *ce, struct i915_request *rq)
  86 {
  87         if (!list_is_last(&rq->signal_link, &ce->signals) &&
  88             i915_seqno_passed(rq->fence.seqno,
  89                               list_next_entry(rq, signal_link)->fence.seqno))
  90                 return false;
  91
  92         if (!list_is_first(&rq->signal_link, &ce->signals) &&
  93             i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
  94                               rq->fence.seqno))
  95                 return false;
  96
  97         return true;
  98 }
  99
 100 static bool
 101 __dma_fence_signal(struct dma_fence *fence)
 102 {
 103         return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
 104 }
 105
 106 static void
 107 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
 108 {
 109         fence->timestamp = timestamp;
 110         set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
 111         trace_dma_fence_signaled(fence);
 112 }
 113
 114 static void
 115 __dma_fence_signal__notify(struct dma_fence *fence)
 116 {
 117         struct dma_fence_cb *cur, *tmp;
 118
 119         lockdep_assert_held(fence->lock);
 120         lockdep_assert_irqs_disabled();
 121
 122         list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
 123                 INIT_LIST_HEAD(&cur->node);
 124                 cur->func(fence, cur);
 125         }
 126         INIT_LIST_HEAD(&fence->cb_list);
 127 }
 128
 129 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
 130 {
 131         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 132         const ktime_t timestamp = ktime_get();
 133         struct intel_context *ce, *cn;
 134         struct list_head *pos, *next;
 135         LIST_HEAD(signal);
 136
 137         spin_lock(&b->irq_lock);
 138
 139         if (b->irq_armed && list_empty(&b->signalers))
 140                 __intel_breadcrumbs_disarm_irq(b);
 141
 142         list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
 143                 GEM_BUG_ON(list_empty(&ce->signals));
 144
 145                 list_for_each_safe(pos, next, &ce->signals) {
 146                         struct i915_request *rq =
 147                                 list_entry(pos, typeof(*rq), signal_link);
 148
 149                         GEM_BUG_ON(!check_signal_order(ce, rq));
 150
 151                         if (!__request_completed(rq))
 152                                 break;
 153
 154                         GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
 155                                              &rq->fence.flags));
 156                         clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 157
 158                         if (!__dma_fence_signal(&rq->fence))
 159                                 continue;
 160
 161                         /*
 162                          * Queue for execution after dropping the signaling
 163                          * spinlock as the callback chain may end up adding
 164                          * more signalers to the same context or engine.
 165                          */
 166                         i915_request_get(rq);
 167                         list_add_tail(&rq->signal_link, &signal);
 168                 }
 169
 170                 /*
 171                  * We process the list deletion in bulk, only using a list_add
 172                  * (not list_move) above but keeping the status of
 173                  * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
 174                  */
 175                 if (!list_is_first(pos, &ce->signals)) {
 176                         /* Advance the list to the first incomplete request */
 177                         __list_del_many(&ce->signals, pos);
 178                         if (&ce->signals == pos) /* now empty */
 179                                 list_del_init(&ce->signal_link);
 180                 }
 181         }
 182
 183         spin_unlock(&b->irq_lock);
 184
 185         list_for_each_safe(pos, next, &signal) {
 186                 struct i915_request *rq =
 187                         list_entry(pos, typeof(*rq), signal_link);
 188
 189                 __dma_fence_signal__timestamp(&rq->fence, timestamp);
 190
 191                 spin_lock(&rq->lock);
 192                 __dma_fence_signal__notify(&rq->fence);
 193                 spin_unlock(&rq->lock);
 194
 195                 i915_request_put(rq);
 196         }
 197 }
 198
 199 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
 200 {
 201         local_irq_disable();
 202         intel_engine_breadcrumbs_irq(engine);
 203         local_irq_enable();
 204 }
 205
 206 static void signal_irq_work(struct irq_work *work)
 207 {
 208         struct intel_engine_cs *engine =
 209                 container_of(work, typeof(*engine), breadcrumbs.irq_work);
 210
 211         intel_engine_breadcrumbs_irq(engine);
 212 }
 213
 214 void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine)
 215 {
 216         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 217
 218         spin_lock_irq(&b->irq_lock);
 219         if (!b->irq_enabled++)
 220                 irq_enable(engine);
 221         GEM_BUG_ON(!b->irq_enabled); /* no overflow! */
 222         spin_unlock_irq(&b->irq_lock);
 223 }
 224
 225 void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine)
 226 {
 227         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 228
 229         spin_lock_irq(&b->irq_lock);
 230         GEM_BUG_ON(!b->irq_enabled); /* no underflow! */
 231         if (!--b->irq_enabled)
 232                 irq_disable(engine);
 233         spin_unlock_irq(&b->irq_lock);
 234 }
 235
 236 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
 237 {
 238         struct intel_engine_cs *engine =
 239                 container_of(b, struct intel_engine_cs, breadcrumbs);
 240
 241         lockdep_assert_held(&b->irq_lock);
 242         if (b->irq_armed)
 243                 return;
 244
 245         /*
 246          * The breadcrumb irq will be disarmed on the interrupt after the
 247          * waiters are signaled. This gives us a single interrupt window in
 248          * which we can add a new waiter and avoid the cost of re-enabling
 249          * the irq.
 250          */
 251         b->irq_armed = true;
 252
 253         /*
 254          * Since we are waiting on a request, the GPU should be busy
 255          * and should have its own rpm reference. This is tracked
 256          * by i915->gt.awake, we can forgo holding our own wakref
 257          * for the interrupt as before i915->gt.awake is released (when
 258          * the driver is idle) we disarm the breadcrumbs.
 259          */
 260
 261         if (!b->irq_enabled++)
 262                 irq_enable(engine);
 263 }
 264
 265 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
 266 {
 267         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 268
 269         spin_lock_init(&b->irq_lock);
 270         INIT_LIST_HEAD(&b->signalers);
 271
 272         init_irq_work(&b->irq_work, signal_irq_work);
 273 }
 274
 275 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 276 {
 277         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 278         unsigned long flags;
 279
 280         spin_lock_irqsave(&b->irq_lock, flags);
 281
 282         if (b->irq_enabled)
 283                 irq_enable(engine);
 284         else
 285                 irq_disable(engine);
 286
 287         spin_unlock_irqrestore(&b->irq_lock, flags);
 288 }
 289
 290 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 291 {
 292 }
 293
 294 bool i915_request_enable_breadcrumb(struct i915_request *rq)
 295 {
 296         lockdep_assert_held(&rq->lock);
 297         lockdep_assert_irqs_disabled();
 298
 299         if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
 300                 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
 301                 struct intel_context *ce = rq->hw_context;
 302                 struct list_head *pos;
 303
 304                 spin_lock(&b->irq_lock);
 305                 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
 306
 307                 __intel_breadcrumbs_arm_irq(b);
 308
 309                 /*
 310                  * We keep the seqno in retirement order, so we can break
 311                  * inside intel_engine_breadcrumbs_irq as soon as we've passed
 312                  * the last completed request (or seen a request that hasn't
 313                  * event started). We could iterate the timeline->requests list,
 314                  * but keeping a separate signalers_list has the advantage of
 315                  * hopefully being much smaller than the full list and so
 316                  * provides faster iteration and detection when there are no
 317                  * more interrupts required for this context.
 318                  *
 319                  * We typically expect to add new signalers in order, so we
 320                  * start looking for our insertion point from the tail of
 321                  * the list.
 322                  */
 323                 list_for_each_prev(pos, &ce->signals) {
 324                         struct i915_request *it =
 325                                 list_entry(pos, typeof(*it), signal_link);
 326
 327                         if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
 328                                 break;
 329                 }
 330                 list_add(&rq->signal_link, pos);
 331                 if (pos == &ce->signals) /* catch transitions from empty list */
 332                         list_move_tail(&ce->signal_link, &b->signalers);
 333                 GEM_BUG_ON(!check_signal_order(ce, rq));
 334
 335                 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 336                 spin_unlock(&b->irq_lock);
 337         }
 338
 339         return !__request_completed(rq);
 340 }
 341
 342 void i915_request_cancel_breadcrumb(struct i915_request *rq)
 343 {
 344         struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
 345
 346         lockdep_assert_held(&rq->lock);
 347         lockdep_assert_irqs_disabled();
 348
 349         /*
 350          * We must wait for b->irq_lock so that we know the interrupt handler
 351          * has released its reference to the intel_context and has completed
 352          * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
 353          * required).
 354          */
 355         spin_lock(&b->irq_lock);
 356         if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
 357                 struct intel_context *ce = rq->hw_context;
 358
 359                 list_del(&rq->signal_link);
 360                 if (list_empty(&ce->signals))
 361                         list_del_init(&ce->signal_link);
 362
 363                 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 364         }
 365         spin_unlock(&b->irq_lock);
 366 }
 367
 368 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
 369                                     struct drm_printer *p)
 370 {
 371         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 372         struct intel_context *ce;
 373         struct i915_request *rq;
 374
 375         if (list_empty(&b->signalers))
 376                 return;
 377
 378         drm_printf(p, "Signals:\n");
 379
 380         spin_lock_irq(&b->irq_lock);
 381         list_for_each_entry(ce, &b->signalers, signal_link) {
 382                 list_for_each_entry(rq, &ce->signals, signal_link) {
 383                         drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
 384                                    rq->fence.context, rq->fence.seqno,
 385                                    i915_request_completed(rq) ? "!" :
 386                                    i915_request_started(rq) ? "*" :
 387                                    "",
 388                                    jiffies_to_msecs(jiffies - rq->emitted_jiffies));
 389                 }
 390         }
 391         spin_unlock_irq(&b->irq_lock);
 392 }