drivers/gpu/drm/i915/i915_sw_fence.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * (C) Copyright 2016 Intel Corporation
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/dma-fence.h>
   9 #include <linux/irq_work.h>
  10 #include <linux/dma-resv.h>
  11
  12 #include "i915_sw_fence.h"
  13 #include "i915_selftest.h"
  14
  15 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
  16 #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr)
  17 #else
  18 #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
  19 #endif
  20
  21 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
  22 static DEFINE_SPINLOCK(i915_sw_fence_lock);
  23 #endif
  24
  25 #define WQ_FLAG_BITS \
  26         BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags))
  27
  28 /* after WQ_FLAG_* for safety */
  29 #define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1)
  30 #define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2)
  31
  32 enum {
  33         DEBUG_FENCE_IDLE = 0,
  34         DEBUG_FENCE_NOTIFY,
  35 };
  36
  37 static void *i915_sw_fence_debug_hint(void *addr)
  38 {
  39         return (void *)(((struct i915_sw_fence *)addr)->fn);
  40 }
  41
  42 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
  43
  44 static const struct debug_obj_descr i915_sw_fence_debug_descr = {
  45         .name = "i915_sw_fence",
  46         .debug_hint = i915_sw_fence_debug_hint,
  47 };
  48
  49 static inline void debug_fence_init(struct i915_sw_fence *fence)
  50 {
  51         debug_object_init(fence, &i915_sw_fence_debug_descr);
  52 }
  53
  54 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  55 {
  56         debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
  57 }
  58
  59 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  60 {
  61         debug_object_activate(fence, &i915_sw_fence_debug_descr);
  62 }
  63
  64 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  65                                          int old, int new)
  66 {
  67         debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
  68 }
  69
  70 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  71 {
  72         debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
  73 }
  74
  75 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  76 {
  77         debug_object_destroy(fence, &i915_sw_fence_debug_descr);
  78 }
  79
  80 static inline void debug_fence_free(struct i915_sw_fence *fence)
  81 {
  82         debug_object_free(fence, &i915_sw_fence_debug_descr);
  83         smp_wmb(); /* flush the change in state before reallocation */
  84 }
  85
  86 static inline void debug_fence_assert(struct i915_sw_fence *fence)
  87 {
  88         debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
  89 }
  90
  91 #else
  92
  93 static inline void debug_fence_init(struct i915_sw_fence *fence)
  94 {
  95 }
  96
  97 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  98 {
  99 }
 100
 101 static inline void debug_fence_activate(struct i915_sw_fence *fence)
 102 {
 103 }
 104
 105 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
 106                                          int old, int new)
 107 {
 108 }
 109
 110 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
 111 {
 112 }
 113
 114 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
 115 {
 116 }
 117
 118 static inline void debug_fence_free(struct i915_sw_fence *fence)
 119 {
 120 }
 121
 122 static inline void debug_fence_assert(struct i915_sw_fence *fence)
 123 {
 124 }
 125
 126 #endif
 127
 128 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 129                                   enum i915_sw_fence_notify state)
 130 {
 131         return fence->fn(fence, state);
 132 }
 133
 134 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 135 void i915_sw_fence_fini(struct i915_sw_fence *fence)
 136 {
 137         debug_fence_free(fence);
 138 }
 139 #endif
 140
 141 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 142                                         struct list_head *continuation)
 143 {
 144         wait_queue_head_t *x = &fence->wait;
 145         wait_queue_entry_t *pos, *next;
 146         unsigned long flags;
 147
 148         debug_fence_deactivate(fence);
 149         atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 150
 151         /*
 152          * To prevent unbounded recursion as we traverse the graph of
 153          * i915_sw_fences, we move the entry list from this, the next ready
 154          * fence, to the tail of the original fence's entry list
 155          * (and so added to the list to be woken).
 156          */
 157
 158         spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 159         if (continuation) {
 160                 list_for_each_entry_safe(pos, next, &x->head, entry) {
 161                         if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
 162                                 list_move_tail(&pos->entry, continuation);
 163                         else
 164                                 pos->func(pos, TASK_NORMAL, 0, continuation);
 165                 }
 166         } else {
 167                 LIST_HEAD(extra);
 168
 169                 do {
 170                         list_for_each_entry_safe(pos, next, &x->head, entry) {
 171                                 int wake_flags;
 172
 173                                 wake_flags = 0;
 174                                 if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
 175                                         wake_flags = fence->error;
 176
 177                                 pos->func(pos, TASK_NORMAL, wake_flags, &extra);
 178                         }
 179
 180                         if (list_empty(&extra))
 181                                 break;
 182
 183                         list_splice_tail_init(&extra, &x->head);
 184                 } while (1);
 185         }
 186         spin_unlock_irqrestore(&x->lock, flags);
 187
 188         debug_fence_assert(fence);
 189 }
 190
 191 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 192                                      struct list_head *continuation)
 193 {
 194         debug_fence_assert(fence);
 195
 196         if (!atomic_dec_and_test(&fence->pending))
 197                 return;
 198
 199         debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 200
 201         if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 202                 return;
 203
 204         debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 205
 206         __i915_sw_fence_wake_up_all(fence, continuation);
 207
 208         debug_fence_destroy(fence);
 209         __i915_sw_fence_notify(fence, FENCE_FREE);
 210 }
 211
 212 void i915_sw_fence_complete(struct i915_sw_fence *fence)
 213 {
 214         debug_fence_assert(fence);
 215
 216         if (WARN_ON(i915_sw_fence_done(fence)))
 217                 return;
 218
 219         __i915_sw_fence_complete(fence, NULL);
 220 }
 221
 222 bool i915_sw_fence_await(struct i915_sw_fence *fence)
 223 {
 224         int pending;
 225
 226         /*
 227          * It is only safe to add a new await to the fence while it has
 228          * not yet been signaled (i.e. there are still existing signalers).
 229          */
 230         pending = atomic_read(&fence->pending);
 231         do {
 232                 if (pending < 1)
 233                         return false;
 234         } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));
 235
 236         return true;
 237 }
 238
 239 void __i915_sw_fence_init(struct i915_sw_fence *fence,
 240                           i915_sw_fence_notify_t fn,
 241                           const char *name,
 242                           struct lock_class_key *key)
 243 {
 244         BUG_ON(!fn);
 245
 246         __init_waitqueue_head(&fence->wait, name, key);
 247         fence->fn = fn;
 248 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
 249         fence->flags = 0;
 250 #endif
 251
 252         i915_sw_fence_reinit(fence);
 253 }
 254
 255 void i915_sw_fence_reinit(struct i915_sw_fence *fence)
 256 {
 257         debug_fence_init(fence);
 258
 259         atomic_set(&fence->pending, 1);
 260         fence->error = 0;
 261
 262         I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));
 263 }
 264
 265 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 266 {
 267         debug_fence_activate(fence);
 268         i915_sw_fence_complete(fence);
 269 }
 270
 271 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 272 {
 273         i915_sw_fence_set_error_once(wq->private, flags);
 274
 275         list_del(&wq->entry);
 276         __i915_sw_fence_complete(wq->private, key);
 277
 278         if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 279                 kfree(wq);
 280         return 0;
 281 }
 282
 283 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
 284 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 285                                     const struct i915_sw_fence * const signaler)
 286 {
 287         wait_queue_entry_t *wq;
 288
 289         if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 290                 return false;
 291
 292         if (fence == signaler)
 293                 return true;
 294
 295         list_for_each_entry(wq, &fence->wait.head, entry) {
 296                 if (wq->func != i915_sw_fence_wake)
 297                         continue;
 298
 299                 if (__i915_sw_fence_check_if_after(wq->private, signaler))
 300                         return true;
 301         }
 302
 303         return false;
 304 }
 305
 306 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
 307 {
 308         wait_queue_entry_t *wq;
 309
 310         if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 311                 return;
 312
 313         list_for_each_entry(wq, &fence->wait.head, entry) {
 314                 if (wq->func != i915_sw_fence_wake)
 315                         continue;
 316
 317                 __i915_sw_fence_clear_checked_bit(wq->private);
 318         }
 319 }
 320
 321 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 322                                   const struct i915_sw_fence * const signaler)
 323 {
 324         unsigned long flags;
 325         bool err;
 326
 327         spin_lock_irqsave(&i915_sw_fence_lock, flags);
 328         err = __i915_sw_fence_check_if_after(fence, signaler);
 329         __i915_sw_fence_clear_checked_bit(fence);
 330         spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
 331
 332         return err;
 333 }
 334 #else
 335 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 336                                          const struct i915_sw_fence * const signaler)
 337 {
 338         return false;
 339 }
 340 #endif
 341
 342 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 343                                           struct i915_sw_fence *signaler,
 344                                           wait_queue_entry_t *wq, gfp_t gfp)
 345 {
 346         unsigned int pending;
 347         unsigned long flags;
 348
 349         debug_fence_assert(fence);
 350         might_sleep_if(gfpflags_allow_blocking(gfp));
 351
 352         if (i915_sw_fence_done(signaler)) {
 353                 i915_sw_fence_set_error_once(fence, signaler->error);
 354                 return 0;
 355         }
 356
 357         debug_fence_assert(signaler);
 358
 359         /* The dependency graph must be acyclic. */
 360         if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 361                 return -EINVAL;
 362
 363         pending = I915_SW_FENCE_FLAG_FENCE;
 364         if (!wq) {
 365                 wq = kmalloc(sizeof(*wq), gfp);
 366                 if (!wq) {
 367                         if (!gfpflags_allow_blocking(gfp))
 368                                 return -ENOMEM;
 369
 370                         i915_sw_fence_wait(signaler);
 371                         i915_sw_fence_set_error_once(fence, signaler->error);
 372                         return 0;
 373                 }
 374
 375                 pending |= I915_SW_FENCE_FLAG_ALLOC;
 376         }
 377
 378         INIT_LIST_HEAD(&wq->entry);
 379         wq->flags = pending;
 380         wq->func = i915_sw_fence_wake;
 381         wq->private = fence;
 382
 383         i915_sw_fence_await(fence);
 384
 385         spin_lock_irqsave(&signaler->wait.lock, flags);
 386         if (likely(!i915_sw_fence_done(signaler))) {
 387                 __add_wait_queue_entry_tail(&signaler->wait, wq);
 388                 pending = 1;
 389         } else {
 390                 i915_sw_fence_wake(wq, 0, signaler->error, NULL);
 391                 pending = 0;
 392         }
 393         spin_unlock_irqrestore(&signaler->wait.lock, flags);
 394
 395         return pending;
 396 }
 397
 398 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 399                                  struct i915_sw_fence *signaler,
 400                                  wait_queue_entry_t *wq)
 401 {
 402         return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
 403 }
 404
 405 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 406                                      struct i915_sw_fence *signaler,
 407                                      gfp_t gfp)
 408 {
 409         return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 410 }
 411
 412 struct i915_sw_dma_fence_cb_timer {
 413         struct i915_sw_dma_fence_cb base;
 414         struct dma_fence *dma;
 415         struct timer_list timer;
 416         struct irq_work work;
 417         struct rcu_head rcu;
 418 };
 419
 420 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 421                                    struct dma_fence_cb *data)
 422 {
 423         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 424
 425         i915_sw_fence_set_error_once(cb->fence, dma->error);
 426         i915_sw_fence_complete(cb->fence);
 427         kfree(cb);
 428 }
 429
 430 static void timer_i915_sw_fence_wake(struct timer_list *t)
 431 {
 432         struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
 433         struct i915_sw_fence *fence;
 434
 435         fence = xchg(&cb->base.fence, NULL);
 436         if (!fence)
 437                 return;
 438
 439         pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n",
 440                   cb->dma->ops->get_driver_name(cb->dma),
 441                   cb->dma->ops->get_timeline_name(cb->dma),
 442                   cb->dma->seqno,
 443                   i915_sw_fence_debug_hint(fence));
 444
 445         i915_sw_fence_set_error_once(fence, -ETIMEDOUT);
 446         i915_sw_fence_complete(fence);
 447 }
 448
 449 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
 450                                          struct dma_fence_cb *data)
 451 {
 452         struct i915_sw_dma_fence_cb_timer *cb =
 453                 container_of(data, typeof(*cb), base.base);
 454         struct i915_sw_fence *fence;
 455
 456         fence = xchg(&cb->base.fence, NULL);
 457         if (fence) {
 458                 i915_sw_fence_set_error_once(fence, dma->error);
 459                 i915_sw_fence_complete(fence);
 460         }
 461
 462         irq_work_queue(&cb->work);
 463 }
 464
 465 static void irq_i915_sw_fence_work(struct irq_work *wrk)
 466 {
 467         struct i915_sw_dma_fence_cb_timer *cb =
 468                 container_of(wrk, typeof(*cb), work);
 469
 470         del_timer_sync(&cb->timer);
 471         dma_fence_put(cb->dma);
 472
 473         kfree_rcu(cb, rcu);
 474 }
 475
 476 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 477                                   struct dma_fence *dma,
 478                                   unsigned long timeout,
 479                                   gfp_t gfp)
 480 {
 481         struct i915_sw_dma_fence_cb *cb;
 482         dma_fence_func_t func;
 483         int ret;
 484
 485         debug_fence_assert(fence);
 486         might_sleep_if(gfpflags_allow_blocking(gfp));
 487
 488         if (dma_fence_is_signaled(dma)) {
 489                 i915_sw_fence_set_error_once(fence, dma->error);
 490                 return 0;
 491         }
 492
 493         cb = kmalloc(timeout ?
 494                      sizeof(struct i915_sw_dma_fence_cb_timer) :
 495                      sizeof(struct i915_sw_dma_fence_cb),
 496                      gfp);
 497         if (!cb) {
 498                 if (!gfpflags_allow_blocking(gfp))
 499                         return -ENOMEM;
 500
 501                 ret = dma_fence_wait(dma, false);
 502                 if (ret)
 503                         return ret;
 504
 505                 i915_sw_fence_set_error_once(fence, dma->error);
 506                 return 0;
 507         }
 508
 509         cb->fence = fence;
 510         i915_sw_fence_await(fence);
 511
 512         func = dma_i915_sw_fence_wake;
 513         if (timeout) {
 514                 struct i915_sw_dma_fence_cb_timer *timer =
 515                         container_of(cb, typeof(*timer), base);
 516
 517                 timer->dma = dma_fence_get(dma);
 518                 init_irq_work(&timer->work, irq_i915_sw_fence_work);
 519
 520                 timer_setup(&timer->timer,
 521                             timer_i915_sw_fence_wake, TIMER_IRQSAFE);
 522                 mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
 523
 524                 func = dma_i915_sw_fence_wake_timer;
 525         }
 526
 527         ret = dma_fence_add_callback(dma, &cb->base, func);
 528         if (ret == 0) {
 529                 ret = 1;
 530         } else {
 531                 func(dma, &cb->base);
 532                 if (ret == -ENOENT) /* fence already signaled */
 533                         ret = 0;
 534         }
 535
 536         return ret;
 537 }
 538
 539 static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
 540                                      struct dma_fence_cb *data)
 541 {
 542         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 543
 544         i915_sw_fence_set_error_once(cb->fence, dma->error);
 545         i915_sw_fence_complete(cb->fence);
 546 }
 547
 548 int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 549                                     struct dma_fence *dma,
 550                                     struct i915_sw_dma_fence_cb *cb)
 551 {
 552         int ret;
 553
 554         debug_fence_assert(fence);
 555
 556         if (dma_fence_is_signaled(dma)) {
 557                 i915_sw_fence_set_error_once(fence, dma->error);
 558                 return 0;
 559         }
 560
 561         cb->fence = fence;
 562         i915_sw_fence_await(fence);
 563
 564         ret = 1;
 565         if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) {
 566                 /* fence already signaled */
 567                 __dma_i915_sw_fence_wake(dma, &cb->base);
 568                 ret = 0;
 569         }
 570
 571         return ret;
 572 }
 573
 574 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 575                                     struct dma_resv *resv,
 576                                     const struct dma_fence_ops *exclude,
 577                                     bool write,
 578                                     unsigned long timeout,
 579                                     gfp_t gfp)
 580 {
 581         struct dma_resv_iter cursor;
 582         struct dma_fence *f;
 583         int ret = 0, pending;
 584
 585         debug_fence_assert(fence);
 586         might_sleep_if(gfpflags_allow_blocking(gfp));
 587
 588         dma_resv_iter_begin(&cursor, resv, write);
 589         dma_resv_for_each_fence_unlocked(&cursor, f) {
 590                 pending = i915_sw_fence_await_dma_fence(fence, f, timeout,
 591                                                         gfp);
 592                 if (pending < 0) {
 593                         ret = pending;
 594                         break;
 595                 }
 596
 597                 ret |= pending;
 598         }
 599         dma_resv_iter_end(&cursor);
 600         return ret;
 601 }
 602
 603 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 604 #include "selftests/lib_sw_fence.c"
 605 #include "selftests/i915_sw_fence.c"
 606 #endif