drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 /**
  60  * drm_syncobj_find - lookup and reference a sync object.
  61  * @file_private: drm file private pointer
  62  * @handle: sync object handle to lookup.
  63  *
  64  * Returns a reference to the syncobj pointed to by handle or NULL. The
  65  * reference must be released by calling drm_syncobj_put().
  66  */
  67 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  68                                      u32 handle)
  69 {
  70         struct drm_syncobj *syncobj;
  71
  72         spin_lock(&file_private->syncobj_table_lock);
  73
  74         /* Check if we currently have a reference on the object */
  75         syncobj = idr_find(&file_private->syncobj_idr, handle);
  76         if (syncobj)
  77                 drm_syncobj_get(syncobj);
  78
  79         spin_unlock(&file_private->syncobj_table_lock);
  80
  81         return syncobj;
  82 }
  83 EXPORT_SYMBOL(drm_syncobj_find);
  84
  85 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
  86                                             struct drm_syncobj_cb *cb,
  87                                             drm_syncobj_func_t func)
  88 {
  89         cb->func = func;
  90         list_add_tail(&cb->node, &syncobj->cb_list);
  91 }
  92
  93 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
  94                                                  struct dma_fence **fence,
  95                                                  struct drm_syncobj_cb *cb,
  96                                                  drm_syncobj_func_t func)
  97 {
  98         int ret;
  99
 100         *fence = drm_syncobj_fence_get(syncobj);
 101         if (*fence)
 102                 return 1;
 103
 104         spin_lock(&syncobj->lock);
 105         /* We've already tried once to get a fence and failed.  Now that we
 106          * have the lock, try one more time just to be sure we don't add a
 107          * callback when a fence has already been set.
 108          */
 109         if (syncobj->fence) {
 110                 *fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 111                                                                  lockdep_is_held(&syncobj->lock)));
 112                 ret = 1;
 113         } else {
 114                 *fence = NULL;
 115                 drm_syncobj_add_callback_locked(syncobj, cb, func);
 116                 ret = 0;
 117         }
 118         spin_unlock(&syncobj->lock);
 119
 120         return ret;
 121 }
 122
 123 /**
 124  * drm_syncobj_add_callback - adds a callback to syncobj::cb_list
 125  * @syncobj: Sync object to which to add the callback
 126  * @cb: Callback to add
 127  * @func: Func to use when initializing the drm_syncobj_cb struct
 128  *
 129  * This adds a callback to be called next time the fence is replaced
 130  */
 131 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
 132                               struct drm_syncobj_cb *cb,
 133                               drm_syncobj_func_t func)
 134 {
 135         spin_lock(&syncobj->lock);
 136         drm_syncobj_add_callback_locked(syncobj, cb, func);
 137         spin_unlock(&syncobj->lock);
 138 }
 139 EXPORT_SYMBOL(drm_syncobj_add_callback);
 140
 141 /**
 142  * drm_syncobj_add_callback - removes a callback to syncobj::cb_list
 143  * @syncobj: Sync object from which to remove the callback
 144  * @cb: Callback to remove
 145  */
 146 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
 147                                  struct drm_syncobj_cb *cb)
 148 {
 149         spin_lock(&syncobj->lock);
 150         list_del_init(&cb->node);
 151         spin_unlock(&syncobj->lock);
 152 }
 153 EXPORT_SYMBOL(drm_syncobj_remove_callback);
 154
 155 /**
 156  * drm_syncobj_replace_fence - replace fence in a sync object.
 157  * @syncobj: Sync object to replace fence in
 158  * @fence: fence to install in sync file.
 159  *
 160  * This replaces the fence on a sync object.
 161  */
 162 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 163                                struct dma_fence *fence)
 164 {
 165         struct dma_fence *old_fence;
 166         struct drm_syncobj_cb *cur, *tmp;
 167
 168         if (fence)
 169                 dma_fence_get(fence);
 170
 171         spin_lock(&syncobj->lock);
 172
 173         old_fence = rcu_dereference_protected(syncobj->fence,
 174                                               lockdep_is_held(&syncobj->lock));
 175         rcu_assign_pointer(syncobj->fence, fence);
 176
 177         if (fence != old_fence) {
 178                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
 179                         list_del_init(&cur->node);
 180                         cur->func(syncobj, cur);
 181                 }
 182         }
 183
 184         spin_unlock(&syncobj->lock);
 185
 186         dma_fence_put(old_fence);
 187 }
 188 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 189
 190 struct drm_syncobj_null_fence {
 191         struct dma_fence base;
 192         spinlock_t lock;
 193 };
 194
 195 static const char *drm_syncobj_null_fence_get_name(struct dma_fence *fence)
 196 {
 197         return "syncobjnull";
 198 }
 199
 200 static bool drm_syncobj_null_fence_enable_signaling(struct dma_fence *fence)
 201 {
 202     dma_fence_enable_sw_signaling(fence);
 203     return !dma_fence_is_signaled(fence);
 204 }
 205
 206 static const struct dma_fence_ops drm_syncobj_null_fence_ops = {
 207         .get_driver_name = drm_syncobj_null_fence_get_name,
 208         .get_timeline_name = drm_syncobj_null_fence_get_name,
 209         .enable_signaling = drm_syncobj_null_fence_enable_signaling,
 210         .wait = dma_fence_default_wait,
 211         .release = NULL,
 212 };
 213
 214 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 215 {
 216         struct drm_syncobj_null_fence *fence;
 217         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 218         if (fence == NULL)
 219                 return -ENOMEM;
 220
 221         spin_lock_init(&fence->lock);
 222         dma_fence_init(&fence->base, &drm_syncobj_null_fence_ops,
 223                        &fence->lock, 0, 0);
 224         dma_fence_signal(&fence->base);
 225
 226         drm_syncobj_replace_fence(syncobj, &fence->base);
 227
 228         dma_fence_put(&fence->base);
 229
 230         return 0;
 231 }
 232
 233 /**
 234  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 235  * @file_private: drm file private pointer
 236  * @handle: sync object handle to lookup.
 237  * @fence: out parameter for the fence
 238  *
 239  * This is just a convenience function that combines drm_syncobj_find() and
 240  * drm_syncobj_fence_get().
 241  *
 242  * Returns 0 on success or a negative error value on failure. On success @fence
 243  * contains a reference to the fence, which must be released by calling
 244  * dma_fence_put().
 245  */
 246 int drm_syncobj_find_fence(struct drm_file *file_private,
 247                            u32 handle,
 248                            struct dma_fence **fence)
 249 {
 250         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 251         int ret = 0;
 252
 253         if (!syncobj)
 254                 return -ENOENT;
 255
 256         *fence = drm_syncobj_fence_get(syncobj);
 257         if (!*fence) {
 258                 ret = -EINVAL;
 259         }
 260         drm_syncobj_put(syncobj);
 261         return ret;
 262 }
 263 EXPORT_SYMBOL(drm_syncobj_find_fence);
 264
 265 /**
 266  * drm_syncobj_free - free a sync object.
 267  * @kref: kref to free.
 268  *
 269  * Only to be called from kref_put in drm_syncobj_put.
 270  */
 271 void drm_syncobj_free(struct kref *kref)
 272 {
 273         struct drm_syncobj *syncobj = container_of(kref,
 274                                                    struct drm_syncobj,
 275                                                    refcount);
 276         drm_syncobj_replace_fence(syncobj, NULL);
 277         kfree(syncobj);
 278 }
 279 EXPORT_SYMBOL(drm_syncobj_free);
 280
 281 /**
 282  * drm_syncobj_create - create a new syncobj
 283  * @out_syncobj: returned syncobj
 284  * @flags: DRM_SYNCOBJ_* flags
 285  * @fence: if non-NULL, the syncobj will represent this fence
 286  *
 287  * This is the first function to create a sync object. After creating, drivers
 288  * probably want to make it available to userspace, either through
 289  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 290  *
 291  * Returns 0 on success or a negative error value on failure.
 292  */
 293 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 294                        struct dma_fence *fence)
 295 {
 296         int ret;
 297         struct drm_syncobj *syncobj;
 298
 299         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 300         if (!syncobj)
 301                 return -ENOMEM;
 302
 303         kref_init(&syncobj->refcount);
 304         INIT_LIST_HEAD(&syncobj->cb_list);
 305         spin_lock_init(&syncobj->lock);
 306
 307         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 308                 ret = drm_syncobj_assign_null_handle(syncobj);
 309                 if (ret < 0) {
 310                         drm_syncobj_put(syncobj);
 311                         return ret;
 312                 }
 313         }
 314
 315         if (fence)
 316                 drm_syncobj_replace_fence(syncobj, fence);
 317
 318         *out_syncobj = syncobj;
 319         return 0;
 320 }
 321 EXPORT_SYMBOL(drm_syncobj_create);
 322
 323 /**
 324  * drm_syncobj_get_handle - get a handle from a syncobj
 325  * @file_private: drm file private pointer
 326  * @syncobj: Sync object to export
 327  * @handle: out parameter with the new handle
 328  *
 329  * Exports a sync object created with drm_syncobj_create() as a handle on
 330  * @file_private to userspace.
 331  *
 332  * Returns 0 on success or a negative error value on failure.
 333  */
 334 int drm_syncobj_get_handle(struct drm_file *file_private,
 335                            struct drm_syncobj *syncobj, u32 *handle)
 336 {
 337         int ret;
 338
 339         /* take a reference to put in the idr */
 340         drm_syncobj_get(syncobj);
 341
 342         idr_preload(GFP_KERNEL);
 343         spin_lock(&file_private->syncobj_table_lock);
 344         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 345         spin_unlock(&file_private->syncobj_table_lock);
 346
 347         idr_preload_end();
 348
 349         if (ret < 0) {
 350                 drm_syncobj_put(syncobj);
 351                 return ret;
 352         }
 353
 354         *handle = ret;
 355         return 0;
 356 }
 357 EXPORT_SYMBOL(drm_syncobj_get_handle);
 358
 359 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 360                                         u32 *handle, uint32_t flags)
 361 {
 362         int ret;
 363         struct drm_syncobj *syncobj;
 364
 365         ret = drm_syncobj_create(&syncobj, flags, NULL);
 366         if (ret)
 367                 return ret;
 368
 369         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 370         drm_syncobj_put(syncobj);
 371         return ret;
 372 }
 373
 374 static int drm_syncobj_destroy(struct drm_file *file_private,
 375                                u32 handle)
 376 {
 377         struct drm_syncobj *syncobj;
 378
 379         spin_lock(&file_private->syncobj_table_lock);
 380         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 381         spin_unlock(&file_private->syncobj_table_lock);
 382
 383         if (!syncobj)
 384                 return -EINVAL;
 385
 386         drm_syncobj_put(syncobj);
 387         return 0;
 388 }
 389
 390 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 391 {
 392         struct drm_syncobj *syncobj = file->private_data;
 393
 394         drm_syncobj_put(syncobj);
 395         return 0;
 396 }
 397
 398 static const struct file_operations drm_syncobj_file_fops = {
 399         .release = drm_syncobj_file_release,
 400 };
 401
 402 /**
 403  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 404  * @syncobj: Sync object to export
 405  * @p_fd: out parameter with the new file descriptor
 406  *
 407  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 408  *
 409  * Returns 0 on success or a negative error value on failure.
 410  */
 411 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 412 {
 413         struct file *file;
 414         int fd;
 415
 416         fd = get_unused_fd_flags(O_CLOEXEC);
 417         if (fd < 0)
 418                 return fd;
 419
 420         file = anon_inode_getfile("syncobj_file",
 421                                   &drm_syncobj_file_fops,
 422                                   syncobj, 0);
 423         if (IS_ERR(file)) {
 424                 put_unused_fd(fd);
 425                 return PTR_ERR(file);
 426         }
 427
 428         drm_syncobj_get(syncobj);
 429         fd_install(fd, file);
 430
 431         *p_fd = fd;
 432         return 0;
 433 }
 434 EXPORT_SYMBOL(drm_syncobj_get_fd);
 435
 436 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 437                                     u32 handle, int *p_fd)
 438 {
 439         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 440         int ret;
 441
 442         if (!syncobj)
 443                 return -EINVAL;
 444
 445         ret = drm_syncobj_get_fd(syncobj, p_fd);
 446         drm_syncobj_put(syncobj);
 447         return ret;
 448 }
 449
 450 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 451                                     int fd, u32 *handle)
 452 {
 453         struct drm_syncobj *syncobj;
 454         struct file *file;
 455         int ret;
 456
 457         file = fget(fd);
 458         if (!file)
 459                 return -EINVAL;
 460
 461         if (file->f_op != &drm_syncobj_file_fops) {
 462                 fput(file);
 463                 return -EINVAL;
 464         }
 465
 466         /* take a reference to put in the idr */
 467         syncobj = file->private_data;
 468         drm_syncobj_get(syncobj);
 469
 470         idr_preload(GFP_KERNEL);
 471         spin_lock(&file_private->syncobj_table_lock);
 472         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 473         spin_unlock(&file_private->syncobj_table_lock);
 474         idr_preload_end();
 475
 476         if (ret > 0) {
 477                 *handle = ret;
 478                 ret = 0;
 479         } else
 480                 drm_syncobj_put(syncobj);
 481
 482         fput(file);
 483         return ret;
 484 }
 485
 486 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 487                                               int fd, int handle)
 488 {
 489         struct dma_fence *fence = sync_file_get_fence(fd);
 490         struct drm_syncobj *syncobj;
 491
 492         if (!fence)
 493                 return -EINVAL;
 494
 495         syncobj = drm_syncobj_find(file_private, handle);
 496         if (!syncobj) {
 497                 dma_fence_put(fence);
 498                 return -ENOENT;
 499         }
 500
 501         drm_syncobj_replace_fence(syncobj, fence);
 502         dma_fence_put(fence);
 503         drm_syncobj_put(syncobj);
 504         return 0;
 505 }
 506
 507 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 508                                         int handle, int *p_fd)
 509 {
 510         int ret;
 511         struct dma_fence *fence;
 512         struct sync_file *sync_file;
 513         int fd = get_unused_fd_flags(O_CLOEXEC);
 514
 515         if (fd < 0)
 516                 return fd;
 517
 518         ret = drm_syncobj_find_fence(file_private, handle, &fence);
 519         if (ret)
 520                 goto err_put_fd;
 521
 522         sync_file = sync_file_create(fence);
 523
 524         dma_fence_put(fence);
 525
 526         if (!sync_file) {
 527                 ret = -EINVAL;
 528                 goto err_put_fd;
 529         }
 530
 531         fd_install(fd, sync_file->file);
 532
 533         *p_fd = fd;
 534         return 0;
 535 err_put_fd:
 536         put_unused_fd(fd);
 537         return ret;
 538 }
 539 /**
 540  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 541  * @file_private: drm file-private structure to set up
 542  *
 543  * Called at device open time, sets up the structure for handling refcounting
 544  * of sync objects.
 545  */
 546 void
 547 drm_syncobj_open(struct drm_file *file_private)
 548 {
 549         idr_init(&file_private->syncobj_idr);
 550         spin_lock_init(&file_private->syncobj_table_lock);
 551 }
 552
 553 static int
 554 drm_syncobj_release_handle(int id, void *ptr, void *data)
 555 {
 556         struct drm_syncobj *syncobj = ptr;
 557
 558         drm_syncobj_put(syncobj);
 559         return 0;
 560 }
 561
 562 /**
 563  * drm_syncobj_release - release file-private sync object resources
 564  * @file_private: drm file-private structure to clean up
 565  *
 566  * Called at close time when the filp is going away.
 567  *
 568  * Releases any remaining references on objects by this filp.
 569  */
 570 void
 571 drm_syncobj_release(struct drm_file *file_private)
 572 {
 573         idr_for_each(&file_private->syncobj_idr,
 574                      &drm_syncobj_release_handle, file_private);
 575         idr_destroy(&file_private->syncobj_idr);
 576 }
 577
 578 int
 579 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 580                          struct drm_file *file_private)
 581 {
 582         struct drm_syncobj_create *args = data;
 583
 584         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 585                 return -ENODEV;
 586
 587         /* no valid flags yet */
 588         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 589                 return -EINVAL;
 590
 591         return drm_syncobj_create_as_handle(file_private,
 592                                             &args->handle, args->flags);
 593 }
 594
 595 int
 596 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 597                           struct drm_file *file_private)
 598 {
 599         struct drm_syncobj_destroy *args = data;
 600
 601         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 602                 return -ENODEV;
 603
 604         /* make sure padding is empty */
 605         if (args->pad)
 606                 return -EINVAL;
 607         return drm_syncobj_destroy(file_private, args->handle);
 608 }
 609
 610 int
 611 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 612                                    struct drm_file *file_private)
 613 {
 614         struct drm_syncobj_handle *args = data;
 615
 616         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 617                 return -ENODEV;
 618
 619         if (args->pad)
 620                 return -EINVAL;
 621
 622         if (args->flags != 0 &&
 623             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 624                 return -EINVAL;
 625
 626         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 627                 return drm_syncobj_export_sync_file(file_private, args->handle,
 628                                                     &args->fd);
 629
 630         return drm_syncobj_handle_to_fd(file_private, args->handle,
 631                                         &args->fd);
 632 }
 633
 634 int
 635 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 636                                    struct drm_file *file_private)
 637 {
 638         struct drm_syncobj_handle *args = data;
 639
 640         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 641                 return -ENODEV;
 642
 643         if (args->pad)
 644                 return -EINVAL;
 645
 646         if (args->flags != 0 &&
 647             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 648                 return -EINVAL;
 649
 650         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 651                 return drm_syncobj_import_sync_file_fence(file_private,
 652                                                           args->fd,
 653                                                           args->handle);
 654
 655         return drm_syncobj_fd_to_handle(file_private, args->fd,
 656                                         &args->handle);
 657 }
 658
 659 struct syncobj_wait_entry {
 660         struct task_struct *task;
 661         struct dma_fence *fence;
 662         struct dma_fence_cb fence_cb;
 663         struct drm_syncobj_cb syncobj_cb;
 664 };
 665
 666 static void syncobj_wait_fence_func(struct dma_fence *fence,
 667                                     struct dma_fence_cb *cb)
 668 {
 669         struct syncobj_wait_entry *wait =
 670                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 671
 672         wake_up_process(wait->task);
 673 }
 674
 675 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 676                                       struct drm_syncobj_cb *cb)
 677 {
 678         struct syncobj_wait_entry *wait =
 679                 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
 680
 681         /* This happens inside the syncobj lock */
 682         wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 683                                                               lockdep_is_held(&syncobj->lock)));
 684         wake_up_process(wait->task);
 685 }
 686
 687 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 688                                                   uint32_t count,
 689                                                   uint32_t flags,
 690                                                   signed long timeout,
 691                                                   uint32_t *idx)
 692 {
 693         struct syncobj_wait_entry *entries;
 694         struct dma_fence *fence;
 695         signed long ret;
 696         uint32_t signaled_count, i;
 697
 698         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 699         if (!entries)
 700                 return -ENOMEM;
 701
 702         /* Walk the list of sync objects and initialize entries.  We do
 703          * this up-front so that we can properly return -EINVAL if there is
 704          * a syncobj with a missing fence and then never have the chance of
 705          * returning -EINVAL again.
 706          */
 707         signaled_count = 0;
 708         for (i = 0; i < count; ++i) {
 709                 entries[i].task = current;
 710                 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
 711                 if (!entries[i].fence) {
 712                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 713                                 continue;
 714                         } else {
 715                                 ret = -EINVAL;
 716                                 goto cleanup_entries;
 717                         }
 718                 }
 719
 720                 if (dma_fence_is_signaled(entries[i].fence)) {
 721                         if (signaled_count == 0 && idx)
 722                                 *idx = i;
 723                         signaled_count++;
 724                 }
 725         }
 726
 727         /* Initialize ret to the max of timeout and 1.  That way, the
 728          * default return value indicates a successful wait and not a
 729          * timeout.
 730          */
 731         ret = max_t(signed long, timeout, 1);
 732
 733         if (signaled_count == count ||
 734             (signaled_count > 0 &&
 735              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 736                 goto cleanup_entries;
 737
 738         /* There's a very annoying laxness in the dma_fence API here, in
 739          * that backends are not required to automatically report when a
 740          * fence is signaled prior to fence->ops->enable_signaling() being
 741          * called.  So here if we fail to match signaled_count, we need to
 742          * fallthough and try a 0 timeout wait!
 743          */
 744
 745         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 746                 for (i = 0; i < count; ++i) {
 747                         drm_syncobj_fence_get_or_add_callback(syncobjs[i],
 748                                                               &entries[i].fence,
 749                                                               &entries[i].syncobj_cb,
 750                                                               syncobj_wait_syncobj_func);
 751                 }
 752         }
 753
 754         do {
 755                 set_current_state(TASK_INTERRUPTIBLE);
 756
 757                 signaled_count = 0;
 758                 for (i = 0; i < count; ++i) {
 759                         fence = entries[i].fence;
 760                         if (!fence)
 761                                 continue;
 762
 763                         if (dma_fence_is_signaled(fence) ||
 764                             (!entries[i].fence_cb.func &&
 765                              dma_fence_add_callback(fence,
 766                                                     &entries[i].fence_cb,
 767                                                     syncobj_wait_fence_func))) {
 768                                 /* The fence has been signaled */
 769                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 770                                         signaled_count++;
 771                                 } else {
 772                                         if (idx)
 773                                                 *idx = i;
 774                                         goto done_waiting;
 775                                 }
 776                         }
 777                 }
 778
 779                 if (signaled_count == count)
 780                         goto done_waiting;
 781
 782                 if (timeout == 0) {
 783                         /* If we are doing a 0 timeout wait and we got
 784                          * here, then we just timed out.
 785                          */
 786                         ret = 0;
 787                         goto done_waiting;
 788                 }
 789
 790                 ret = schedule_timeout(ret);
 791
 792                 if (ret > 0 && signal_pending(current))
 793                         ret = -ERESTARTSYS;
 794         } while (ret > 0);
 795
 796 done_waiting:
 797         __set_current_state(TASK_RUNNING);
 798
 799 cleanup_entries:
 800         for (i = 0; i < count; ++i) {
 801                 if (entries[i].syncobj_cb.func)
 802                         drm_syncobj_remove_callback(syncobjs[i],
 803                                                     &entries[i].syncobj_cb);
 804                 if (entries[i].fence_cb.func)
 805                         dma_fence_remove_callback(entries[i].fence,
 806                                                   &entries[i].fence_cb);
 807                 dma_fence_put(entries[i].fence);
 808         }
 809         kfree(entries);
 810
 811         return ret;
 812 }
 813
 814 /**
 815  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 816  *
 817  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 818  *
 819  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 820  */
 821 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 822 {
 823         ktime_t abs_timeout, now;
 824         u64 timeout_ns, timeout_jiffies64;
 825
 826         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 827         if (timeout_nsec == 0)
 828                 return 0;
 829
 830         abs_timeout = ns_to_ktime(timeout_nsec);
 831         now = ktime_get();
 832
 833         if (!ktime_after(abs_timeout, now))
 834                 return 0;
 835
 836         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 837
 838         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 839         /*  clamp timeout to avoid infinite timeout */
 840         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 841                 return MAX_SCHEDULE_TIMEOUT - 1;
 842
 843         return timeout_jiffies64 + 1;
 844 }
 845
 846 static int drm_syncobj_array_wait(struct drm_device *dev,
 847                                   struct drm_file *file_private,
 848                                   struct drm_syncobj_wait *wait,
 849                                   struct drm_syncobj **syncobjs)
 850 {
 851         signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 852         signed long ret = 0;
 853         uint32_t first = ~0;
 854
 855         ret = drm_syncobj_array_wait_timeout(syncobjs,
 856                                              wait->count_handles,
 857                                              wait->flags,
 858                                              timeout, &first);
 859         if (ret < 0)
 860                 return ret;
 861
 862         wait->first_signaled = first;
 863         if (ret == 0)
 864                 return -ETIME;
 865         return 0;
 866 }
 867
 868 static int drm_syncobj_array_find(struct drm_file *file_private,
 869                                   void __user *user_handles,
 870                                   uint32_t count_handles,
 871                                   struct drm_syncobj ***syncobjs_out)
 872 {
 873         uint32_t i, *handles;
 874         struct drm_syncobj **syncobjs;
 875         int ret;
 876
 877         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 878         if (handles == NULL)
 879                 return -ENOMEM;
 880
 881         if (copy_from_user(handles, user_handles,
 882                            sizeof(uint32_t) * count_handles)) {
 883                 ret = -EFAULT;
 884                 goto err_free_handles;
 885         }
 886
 887         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 888         if (syncobjs == NULL) {
 889                 ret = -ENOMEM;
 890                 goto err_free_handles;
 891         }
 892
 893         for (i = 0; i < count_handles; i++) {
 894                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 895                 if (!syncobjs[i]) {
 896                         ret = -ENOENT;
 897                         goto err_put_syncobjs;
 898                 }
 899         }
 900
 901         kfree(handles);
 902         *syncobjs_out = syncobjs;
 903         return 0;
 904
 905 err_put_syncobjs:
 906         while (i-- > 0)
 907                 drm_syncobj_put(syncobjs[i]);
 908         kfree(syncobjs);
 909 err_free_handles:
 910         kfree(handles);
 911
 912         return ret;
 913 }
 914
 915 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 916                                    uint32_t count)
 917 {
 918         uint32_t i;
 919         for (i = 0; i < count; i++)
 920                 drm_syncobj_put(syncobjs[i]);
 921         kfree(syncobjs);
 922 }
 923
 924 int
 925 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 926                        struct drm_file *file_private)
 927 {
 928         struct drm_syncobj_wait *args = data;
 929         struct drm_syncobj **syncobjs;
 930         int ret = 0;
 931
 932         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 933                 return -ENODEV;
 934
 935         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 936                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 937                 return -EINVAL;
 938
 939         if (args->count_handles == 0)
 940                 return -EINVAL;
 941
 942         ret = drm_syncobj_array_find(file_private,
 943                                      u64_to_user_ptr(args->handles),
 944                                      args->count_handles,
 945                                      &syncobjs);
 946         if (ret < 0)
 947                 return ret;
 948
 949         ret = drm_syncobj_array_wait(dev, file_private,
 950                                      args, syncobjs);
 951
 952         drm_syncobj_array_free(syncobjs, args->count_handles);
 953
 954         return ret;
 955 }
 956
 957 int
 958 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 959                         struct drm_file *file_private)
 960 {
 961         struct drm_syncobj_array *args = data;
 962         struct drm_syncobj **syncobjs;
 963         uint32_t i;
 964         int ret;
 965
 966         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 967                 return -ENODEV;
 968
 969         if (args->pad != 0)
 970                 return -EINVAL;
 971
 972         if (args->count_handles == 0)
 973                 return -EINVAL;
 974
 975         ret = drm_syncobj_array_find(file_private,
 976                                      u64_to_user_ptr(args->handles),
 977                                      args->count_handles,
 978                                      &syncobjs);
 979         if (ret < 0)
 980                 return ret;
 981
 982         for (i = 0; i < args->count_handles; i++)
 983                 drm_syncobj_replace_fence(syncobjs[i], NULL);
 984
 985         drm_syncobj_array_free(syncobjs, args->count_handles);
 986
 987         return 0;
 988 }
 989
 990 int
 991 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 992                          struct drm_file *file_private)
 993 {
 994         struct drm_syncobj_array *args = data;
 995         struct drm_syncobj **syncobjs;
 996         uint32_t i;
 997         int ret;
 998
 999         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1000                 return -ENODEV;
1001
1002         if (args->pad != 0)
1003                 return -EINVAL;
1004
1005         if (args->count_handles == 0)
1006                 return -EINVAL;
1007
1008         ret = drm_syncobj_array_find(file_private,
1009                                      u64_to_user_ptr(args->handles),
1010                                      args->count_handles,
1011                                      &syncobjs);
1012         if (ret < 0)
1013                 return ret;
1014
1015         for (i = 0; i < args->count_handles; i++) {
1016                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1017                 if (ret < 0)
1018                         break;
1019         }
1020
1021         drm_syncobj_array_free(syncobjs, args->count_handles);
1022
1023         return ret;
1024 }