drivers/gpu/drm/i915/gt/intel_breadcrumbs.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/kthread.h>
  26 #include <trace/events/dma_fence.h>
  27 #include <uapi/linux/sched/types.h>
  28
  29 #include "i915_drv.h"
  30 #include "i915_trace.h"
  31 #include "intel_gt_pm.h"
  32 #include "intel_gt_requests.h"
  33
  34 static void irq_enable(struct intel_engine_cs *engine)
  35 {
  36         if (!engine->irq_enable)
  37                 return;
  38
  39         /* Caller disables interrupts */
  40         spin_lock(&engine->gt->irq_lock);
  41         engine->irq_enable(engine);
  42         spin_unlock(&engine->gt->irq_lock);
  43 }
  44
  45 static void irq_disable(struct intel_engine_cs *engine)
  46 {
  47         if (!engine->irq_disable)
  48                 return;
  49
  50         /* Caller disables interrupts */
  51         spin_lock(&engine->gt->irq_lock);
  52         engine->irq_disable(engine);
  53         spin_unlock(&engine->gt->irq_lock);
  54 }
  55
  56 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
  57 {
  58         struct intel_engine_cs *engine =
  59                 container_of(b, struct intel_engine_cs, breadcrumbs);
  60
  61         lockdep_assert_held(&b->irq_lock);
  62
  63         GEM_BUG_ON(!b->irq_enabled);
  64         if (!--b->irq_enabled)
  65                 irq_disable(engine);
  66
  67         WRITE_ONCE(b->irq_armed, false);
  68         intel_gt_pm_put_async(engine->gt);
  69 }
  70
  71 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
  72 {
  73         struct intel_breadcrumbs *b = &engine->breadcrumbs;
  74         unsigned long flags;
  75
  76         if (!READ_ONCE(b->irq_armed))
  77                 return;
  78
  79         spin_lock_irqsave(&b->irq_lock, flags);
  80         if (b->irq_armed)
  81                 __intel_breadcrumbs_disarm_irq(b);
  82         spin_unlock_irqrestore(&b->irq_lock, flags);
  83 }
  84
  85 static inline bool __request_completed(const struct i915_request *rq)
  86 {
  87         return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
  88 }
  89
  90 __maybe_unused static bool
  91 check_signal_order(struct intel_context *ce, struct i915_request *rq)
  92 {
  93         if (!list_is_last(&rq->signal_link, &ce->signals) &&
  94             i915_seqno_passed(rq->fence.seqno,
  95                               list_next_entry(rq, signal_link)->fence.seqno))
  96                 return false;
  97
  98         if (!list_is_first(&rq->signal_link, &ce->signals) &&
  99             i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
 100                               rq->fence.seqno))
 101                 return false;
 102
 103         return true;
 104 }
 105
 106 static bool
 107 __dma_fence_signal(struct dma_fence *fence)
 108 {
 109         return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
 110 }
 111
 112 static void
 113 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
 114 {
 115         fence->timestamp = timestamp;
 116         set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
 117         trace_dma_fence_signaled(fence);
 118 }
 119
 120 static void
 121 __dma_fence_signal__notify(struct dma_fence *fence,
 122                            const struct list_head *list)
 123 {
 124         struct dma_fence_cb *cur, *tmp;
 125
 126         lockdep_assert_held(fence->lock);
 127
 128         list_for_each_entry_safe(cur, tmp, list, node) {
 129                 INIT_LIST_HEAD(&cur->node);
 130                 cur->func(fence, cur);
 131         }
 132 }
 133
 134 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
 135 {
 136         struct intel_engine_cs *engine =
 137                 container_of(b, struct intel_engine_cs, breadcrumbs);
 138
 139         if (unlikely(intel_engine_is_virtual(engine)))
 140                 engine = intel_virtual_engine_get_sibling(engine, 0);
 141
 142         intel_engine_add_retire(engine, tl);
 143 }
 144
 145 static void __signal_request(struct i915_request *rq, struct list_head *signals)
 146 {
 147         GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
 148         clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 149
 150         if (!__dma_fence_signal(&rq->fence))
 151                 return;
 152
 153         i915_request_get(rq);
 154         list_add_tail(&rq->signal_link, signals);
 155 }
 156
 157 static void signal_irq_work(struct irq_work *work)
 158 {
 159         struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
 160         const ktime_t timestamp = ktime_get();
 161         struct intel_context *ce, *cn;
 162         struct list_head *pos, *next;
 163         LIST_HEAD(signal);
 164
 165         spin_lock(&b->irq_lock);
 166
 167         if (b->irq_armed && list_empty(&b->signalers))
 168                 __intel_breadcrumbs_disarm_irq(b);
 169
 170         list_splice_init(&b->signaled_requests, &signal);
 171
 172         list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
 173                 GEM_BUG_ON(list_empty(&ce->signals));
 174
 175                 list_for_each_safe(pos, next, &ce->signals) {
 176                         struct i915_request *rq =
 177                                 list_entry(pos, typeof(*rq), signal_link);
 178
 179                         GEM_BUG_ON(!check_signal_order(ce, rq));
 180                         if (!__request_completed(rq))
 181                                 break;
 182
 183                         /*
 184                          * Queue for execution after dropping the signaling
 185                          * spinlock as the callback chain may end up adding
 186                          * more signalers to the same context or engine.
 187                          */
 188                         __signal_request(rq, &signal);
 189                 }
 190
 191                 /*
 192                  * We process the list deletion in bulk, only using a list_add
 193                  * (not list_move) above but keeping the status of
 194                  * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
 195                  */
 196                 if (!list_is_first(pos, &ce->signals)) {
 197                         /* Advance the list to the first incomplete request */
 198                         __list_del_many(&ce->signals, pos);
 199                         if (&ce->signals == pos) { /* now empty */
 200                                 list_del_init(&ce->signal_link);
 201                                 add_retire(b, ce->timeline);
 202                         }
 203                 }
 204         }
 205
 206         spin_unlock(&b->irq_lock);
 207
 208         list_for_each_safe(pos, next, &signal) {
 209                 struct i915_request *rq =
 210                         list_entry(pos, typeof(*rq), signal_link);
 211                 struct list_head cb_list;
 212
 213                 spin_lock(&rq->lock);
 214                 list_replace(&rq->fence.cb_list, &cb_list);
 215                 __dma_fence_signal__timestamp(&rq->fence, timestamp);
 216                 __dma_fence_signal__notify(&rq->fence, &cb_list);
 217                 spin_unlock(&rq->lock);
 218
 219                 i915_request_put(rq);
 220         }
 221 }
 222
 223 static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
 224 {
 225         struct intel_engine_cs *engine =
 226                 container_of(b, struct intel_engine_cs, breadcrumbs);
 227
 228         lockdep_assert_held(&b->irq_lock);
 229         if (b->irq_armed)
 230                 return true;
 231
 232         if (!intel_gt_pm_get_if_awake(engine->gt))
 233                 return false;
 234
 235         /*
 236          * The breadcrumb irq will be disarmed on the interrupt after the
 237          * waiters are signaled. This gives us a single interrupt window in
 238          * which we can add a new waiter and avoid the cost of re-enabling
 239          * the irq.
 240          */
 241         WRITE_ONCE(b->irq_armed, true);
 242
 243         /*
 244          * Since we are waiting on a request, the GPU should be busy
 245          * and should have its own rpm reference. This is tracked
 246          * by i915->gt.awake, we can forgo holding our own wakref
 247          * for the interrupt as before i915->gt.awake is released (when
 248          * the driver is idle) we disarm the breadcrumbs.
 249          */
 250
 251         if (!b->irq_enabled++)
 252                 irq_enable(engine);
 253
 254         return true;
 255 }
 256
 257 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
 258 {
 259         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 260
 261         spin_lock_init(&b->irq_lock);
 262         INIT_LIST_HEAD(&b->signalers);
 263         INIT_LIST_HEAD(&b->signaled_requests);
 264
 265         init_irq_work(&b->irq_work, signal_irq_work);
 266 }
 267
 268 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 269 {
 270         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 271         unsigned long flags;
 272
 273         spin_lock_irqsave(&b->irq_lock, flags);
 274
 275         if (b->irq_enabled)
 276                 irq_enable(engine);
 277         else
 278                 irq_disable(engine);
 279
 280         spin_unlock_irqrestore(&b->irq_lock, flags);
 281 }
 282
 283 void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine,
 284                                              struct intel_context *ce)
 285 {
 286         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 287         unsigned long flags;
 288
 289         spin_lock_irqsave(&b->irq_lock, flags);
 290         if (!list_empty(&ce->signals)) {
 291                 struct i915_request *rq, *next;
 292
 293                 /* Queue for executing the signal callbacks in the irq_work */
 294                 list_for_each_entry_safe(rq, next, &ce->signals, signal_link) {
 295                         GEM_BUG_ON(rq->engine != engine);
 296                         GEM_BUG_ON(!__request_completed(rq));
 297
 298                         __signal_request(rq, &b->signaled_requests);
 299                 }
 300
 301                 INIT_LIST_HEAD(&ce->signals);
 302                 list_del_init(&ce->signal_link);
 303
 304                 irq_work_queue(&b->irq_work);
 305         }
 306         spin_unlock_irqrestore(&b->irq_lock, flags);
 307 }
 308
 309 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 310 {
 311 }
 312
 313 bool i915_request_enable_breadcrumb(struct i915_request *rq)
 314 {
 315         lockdep_assert_held(&rq->lock);
 316
 317         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
 318                 return true;
 319
 320         if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
 321                 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
 322                 struct intel_context *ce = rq->context;
 323                 struct list_head *pos;
 324
 325                 spin_lock(&b->irq_lock);
 326
 327                 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
 328                         goto unlock;
 329
 330                 if (!__intel_breadcrumbs_arm_irq(b))
 331                         goto unlock;
 332
 333                 /*
 334                  * We keep the seqno in retirement order, so we can break
 335                  * inside intel_engine_signal_breadcrumbs as soon as we've
 336                  * passed the last completed request (or seen a request that
 337                  * hasn't event started). We could walk the timeline->requests,
 338                  * but keeping a separate signalers_list has the advantage of
 339                  * hopefully being much smaller than the full list and so
 340                  * provides faster iteration and detection when there are no
 341                  * more interrupts required for this context.
 342                  *
 343                  * We typically expect to add new signalers in order, so we
 344                  * start looking for our insertion point from the tail of
 345                  * the list.
 346                  */
 347                 list_for_each_prev(pos, &ce->signals) {
 348                         struct i915_request *it =
 349                                 list_entry(pos, typeof(*it), signal_link);
 350
 351                         if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
 352                                 break;
 353                 }
 354                 list_add(&rq->signal_link, pos);
 355                 if (pos == &ce->signals) /* catch transitions from empty list */
 356                         list_move_tail(&ce->signal_link, &b->signalers);
 357                 GEM_BUG_ON(!check_signal_order(ce, rq));
 358
 359                 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 360 unlock:
 361                 spin_unlock(&b->irq_lock);
 362         }
 363
 364         return !__request_completed(rq);
 365 }
 366
 367 void i915_request_cancel_breadcrumb(struct i915_request *rq)
 368 {
 369         struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
 370
 371         lockdep_assert_held(&rq->lock);
 372
 373         /*
 374          * We must wait for b->irq_lock so that we know the interrupt handler
 375          * has released its reference to the intel_context and has completed
 376          * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
 377          * required).
 378          */
 379         spin_lock(&b->irq_lock);
 380         if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
 381                 struct intel_context *ce = rq->context;
 382
 383                 list_del(&rq->signal_link);
 384                 if (list_empty(&ce->signals))
 385                         list_del_init(&ce->signal_link);
 386
 387                 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 388         }
 389         spin_unlock(&b->irq_lock);
 390 }
 391
 392 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
 393                                     struct drm_printer *p)
 394 {
 395         struct intel_breadcrumbs *b = &engine->breadcrumbs;
 396         struct intel_context *ce;
 397         struct i915_request *rq;
 398
 399         if (list_empty(&b->signalers))
 400                 return;
 401
 402         drm_printf(p, "Signals:\n");
 403
 404         spin_lock_irq(&b->irq_lock);
 405         list_for_each_entry(ce, &b->signalers, signal_link) {
 406                 list_for_each_entry(rq, &ce->signals, signal_link) {
 407                         drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
 408                                    rq->fence.context, rq->fence.seqno,
 409                                    i915_request_completed(rq) ? "!" :
 410                                    i915_request_started(rq) ? "*" :
 411                                    "",
 412                                    jiffies_to_msecs(jiffies - rq->emitted_jiffies));
 413                 }
 414         }
 415         spin_unlock_irq(&b->irq_lock);
 416 }