drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 struct drm_syncobj_stub_fence {
  60         struct dma_fence base;
  61         spinlock_t lock;
  62 };
  63
  64 static const char *drm_syncobj_stub_fence_get_name(struct dma_fence *fence)
  65 {
  66         return "syncobjstub";
  67 }
  68
  69 static const struct dma_fence_ops drm_syncobj_stub_fence_ops = {
  70         .get_driver_name = drm_syncobj_stub_fence_get_name,
  71         .get_timeline_name = drm_syncobj_stub_fence_get_name,
  72 };
  73
  74
  75 /**
  76  * drm_syncobj_find - lookup and reference a sync object.
  77  * @file_private: drm file private pointer
  78  * @handle: sync object handle to lookup.
  79  *
  80  * Returns a reference to the syncobj pointed to by handle or NULL. The
  81  * reference must be released by calling drm_syncobj_put().
  82  */
  83 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  84                                      u32 handle)
  85 {
  86         struct drm_syncobj *syncobj;
  87
  88         spin_lock(&file_private->syncobj_table_lock);
  89
  90         /* Check if we currently have a reference on the object */
  91         syncobj = idr_find(&file_private->syncobj_idr, handle);
  92         if (syncobj)
  93                 drm_syncobj_get(syncobj);
  94
  95         spin_unlock(&file_private->syncobj_table_lock);
  96
  97         return syncobj;
  98 }
  99 EXPORT_SYMBOL(drm_syncobj_find);
 100
 101 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
 102                                             struct drm_syncobj_cb *cb,
 103                                             drm_syncobj_func_t func)
 104 {
 105         cb->func = func;
 106         list_add_tail(&cb->node, &syncobj->cb_list);
 107 }
 108
 109 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
 110                                                  struct dma_fence **fence,
 111                                                  struct drm_syncobj_cb *cb,
 112                                                  drm_syncobj_func_t func)
 113 {
 114         int ret;
 115
 116         *fence = drm_syncobj_fence_get(syncobj);
 117         if (*fence)
 118                 return 1;
 119
 120         spin_lock(&syncobj->lock);
 121         /* We've already tried once to get a fence and failed.  Now that we
 122          * have the lock, try one more time just to be sure we don't add a
 123          * callback when a fence has already been set.
 124          */
 125         if (syncobj->fence) {
 126                 *fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 127                                                                  lockdep_is_held(&syncobj->lock)));
 128                 ret = 1;
 129         } else {
 130                 *fence = NULL;
 131                 drm_syncobj_add_callback_locked(syncobj, cb, func);
 132                 ret = 0;
 133         }
 134         spin_unlock(&syncobj->lock);
 135
 136         return ret;
 137 }
 138
 139 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
 140                               struct drm_syncobj_cb *cb,
 141                               drm_syncobj_func_t func)
 142 {
 143         spin_lock(&syncobj->lock);
 144         drm_syncobj_add_callback_locked(syncobj, cb, func);
 145         spin_unlock(&syncobj->lock);
 146 }
 147
 148 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
 149                                  struct drm_syncobj_cb *cb)
 150 {
 151         spin_lock(&syncobj->lock);
 152         list_del_init(&cb->node);
 153         spin_unlock(&syncobj->lock);
 154 }
 155
 156 /**
 157  * drm_syncobj_replace_fence - replace fence in a sync object.
 158  * @syncobj: Sync object to replace fence in
 159  * @point: timeline point
 160  * @fence: fence to install in sync file.
 161  *
 162  * This replaces the fence on a sync object, or a timeline point fence.
 163  */
 164 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 165                                u64 point,
 166                                struct dma_fence *fence)
 167 {
 168         struct dma_fence *old_fence;
 169         struct drm_syncobj_cb *cur, *tmp;
 170
 171         if (fence)
 172                 dma_fence_get(fence);
 173
 174         spin_lock(&syncobj->lock);
 175
 176         old_fence = rcu_dereference_protected(syncobj->fence,
 177                                               lockdep_is_held(&syncobj->lock));
 178         rcu_assign_pointer(syncobj->fence, fence);
 179
 180         if (fence != old_fence) {
 181                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
 182                         list_del_init(&cur->node);
 183                         cur->func(syncobj, cur);
 184                 }
 185         }
 186
 187         spin_unlock(&syncobj->lock);
 188
 189         dma_fence_put(old_fence);
 190 }
 191 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 192
 193 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 194 {
 195         struct drm_syncobj_stub_fence *fence;
 196         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 197         if (fence == NULL)
 198                 return -ENOMEM;
 199
 200         spin_lock_init(&fence->lock);
 201         dma_fence_init(&fence->base, &drm_syncobj_stub_fence_ops,
 202                        &fence->lock, 0, 0);
 203         dma_fence_signal(&fence->base);
 204
 205         drm_syncobj_replace_fence(syncobj, 0, &fence->base);
 206
 207         dma_fence_put(&fence->base);
 208
 209         return 0;
 210 }
 211
 212 /**
 213  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 214  * @file_private: drm file private pointer
 215  * @handle: sync object handle to lookup.
 216  * @point: timeline point
 217  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 218  * @fence: out parameter for the fence
 219  *
 220  * This is just a convenience function that combines drm_syncobj_find() and
 221  * drm_syncobj_fence_get().
 222  *
 223  * Returns 0 on success or a negative error value on failure. On success @fence
 224  * contains a reference to the fence, which must be released by calling
 225  * dma_fence_put().
 226  */
 227 int drm_syncobj_find_fence(struct drm_file *file_private,
 228                            u32 handle, u64 point, u64 flags,
 229                            struct dma_fence **fence)
 230 {
 231         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 232         int ret = 0;
 233
 234         if (!syncobj)
 235                 return -ENOENT;
 236
 237         *fence = drm_syncobj_fence_get(syncobj);
 238         if (!*fence) {
 239                 ret = -EINVAL;
 240         }
 241         drm_syncobj_put(syncobj);
 242         return ret;
 243 }
 244 EXPORT_SYMBOL(drm_syncobj_find_fence);
 245
 246 /**
 247  * drm_syncobj_free - free a sync object.
 248  * @kref: kref to free.
 249  *
 250  * Only to be called from kref_put in drm_syncobj_put.
 251  */
 252 void drm_syncobj_free(struct kref *kref)
 253 {
 254         struct drm_syncobj *syncobj = container_of(kref,
 255                                                    struct drm_syncobj,
 256                                                    refcount);
 257         drm_syncobj_replace_fence(syncobj, 0, NULL);
 258         kfree(syncobj);
 259 }
 260 EXPORT_SYMBOL(drm_syncobj_free);
 261
 262 /**
 263  * drm_syncobj_create - create a new syncobj
 264  * @out_syncobj: returned syncobj
 265  * @flags: DRM_SYNCOBJ_* flags
 266  * @fence: if non-NULL, the syncobj will represent this fence
 267  *
 268  * This is the first function to create a sync object. After creating, drivers
 269  * probably want to make it available to userspace, either through
 270  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 271  *
 272  * Returns 0 on success or a negative error value on failure.
 273  */
 274 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 275                        struct dma_fence *fence)
 276 {
 277         int ret;
 278         struct drm_syncobj *syncobj;
 279
 280         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 281         if (!syncobj)
 282                 return -ENOMEM;
 283
 284         kref_init(&syncobj->refcount);
 285         INIT_LIST_HEAD(&syncobj->cb_list);
 286         spin_lock_init(&syncobj->lock);
 287
 288         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 289                 ret = drm_syncobj_assign_null_handle(syncobj);
 290                 if (ret < 0) {
 291                         drm_syncobj_put(syncobj);
 292                         return ret;
 293                 }
 294         }
 295
 296         if (fence)
 297                 drm_syncobj_replace_fence(syncobj, 0, fence);
 298
 299         *out_syncobj = syncobj;
 300         return 0;
 301 }
 302 EXPORT_SYMBOL(drm_syncobj_create);
 303
 304 /**
 305  * drm_syncobj_get_handle - get a handle from a syncobj
 306  * @file_private: drm file private pointer
 307  * @syncobj: Sync object to export
 308  * @handle: out parameter with the new handle
 309  *
 310  * Exports a sync object created with drm_syncobj_create() as a handle on
 311  * @file_private to userspace.
 312  *
 313  * Returns 0 on success or a negative error value on failure.
 314  */
 315 int drm_syncobj_get_handle(struct drm_file *file_private,
 316                            struct drm_syncobj *syncobj, u32 *handle)
 317 {
 318         int ret;
 319
 320         /* take a reference to put in the idr */
 321         drm_syncobj_get(syncobj);
 322
 323         idr_preload(GFP_KERNEL);
 324         spin_lock(&file_private->syncobj_table_lock);
 325         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 326         spin_unlock(&file_private->syncobj_table_lock);
 327
 328         idr_preload_end();
 329
 330         if (ret < 0) {
 331                 drm_syncobj_put(syncobj);
 332                 return ret;
 333         }
 334
 335         *handle = ret;
 336         return 0;
 337 }
 338 EXPORT_SYMBOL(drm_syncobj_get_handle);
 339
 340 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 341                                         u32 *handle, uint32_t flags)
 342 {
 343         int ret;
 344         struct drm_syncobj *syncobj;
 345
 346         ret = drm_syncobj_create(&syncobj, flags, NULL);
 347         if (ret)
 348                 return ret;
 349
 350         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 351         drm_syncobj_put(syncobj);
 352         return ret;
 353 }
 354
 355 static int drm_syncobj_destroy(struct drm_file *file_private,
 356                                u32 handle)
 357 {
 358         struct drm_syncobj *syncobj;
 359
 360         spin_lock(&file_private->syncobj_table_lock);
 361         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 362         spin_unlock(&file_private->syncobj_table_lock);
 363
 364         if (!syncobj)
 365                 return -EINVAL;
 366
 367         drm_syncobj_put(syncobj);
 368         return 0;
 369 }
 370
 371 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 372 {
 373         struct drm_syncobj *syncobj = file->private_data;
 374
 375         drm_syncobj_put(syncobj);
 376         return 0;
 377 }
 378
 379 static const struct file_operations drm_syncobj_file_fops = {
 380         .release = drm_syncobj_file_release,
 381 };
 382
 383 /**
 384  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 385  * @syncobj: Sync object to export
 386  * @p_fd: out parameter with the new file descriptor
 387  *
 388  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 389  *
 390  * Returns 0 on success or a negative error value on failure.
 391  */
 392 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 393 {
 394         struct file *file;
 395         int fd;
 396
 397         fd = get_unused_fd_flags(O_CLOEXEC);
 398         if (fd < 0)
 399                 return fd;
 400
 401         file = anon_inode_getfile("syncobj_file",
 402                                   &drm_syncobj_file_fops,
 403                                   syncobj, 0);
 404         if (IS_ERR(file)) {
 405                 put_unused_fd(fd);
 406                 return PTR_ERR(file);
 407         }
 408
 409         drm_syncobj_get(syncobj);
 410         fd_install(fd, file);
 411
 412         *p_fd = fd;
 413         return 0;
 414 }
 415 EXPORT_SYMBOL(drm_syncobj_get_fd);
 416
 417 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 418                                     u32 handle, int *p_fd)
 419 {
 420         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 421         int ret;
 422
 423         if (!syncobj)
 424                 return -EINVAL;
 425
 426         ret = drm_syncobj_get_fd(syncobj, p_fd);
 427         drm_syncobj_put(syncobj);
 428         return ret;
 429 }
 430
 431 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 432                                     int fd, u32 *handle)
 433 {
 434         struct drm_syncobj *syncobj;
 435         struct file *file;
 436         int ret;
 437
 438         file = fget(fd);
 439         if (!file)
 440                 return -EINVAL;
 441
 442         if (file->f_op != &drm_syncobj_file_fops) {
 443                 fput(file);
 444                 return -EINVAL;
 445         }
 446
 447         /* take a reference to put in the idr */
 448         syncobj = file->private_data;
 449         drm_syncobj_get(syncobj);
 450
 451         idr_preload(GFP_KERNEL);
 452         spin_lock(&file_private->syncobj_table_lock);
 453         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 454         spin_unlock(&file_private->syncobj_table_lock);
 455         idr_preload_end();
 456
 457         if (ret > 0) {
 458                 *handle = ret;
 459                 ret = 0;
 460         } else
 461                 drm_syncobj_put(syncobj);
 462
 463         fput(file);
 464         return ret;
 465 }
 466
 467 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 468                                               int fd, int handle)
 469 {
 470         struct dma_fence *fence = sync_file_get_fence(fd);
 471         struct drm_syncobj *syncobj;
 472
 473         if (!fence)
 474                 return -EINVAL;
 475
 476         syncobj = drm_syncobj_find(file_private, handle);
 477         if (!syncobj) {
 478                 dma_fence_put(fence);
 479                 return -ENOENT;
 480         }
 481
 482         drm_syncobj_replace_fence(syncobj, 0, fence);
 483         dma_fence_put(fence);
 484         drm_syncobj_put(syncobj);
 485         return 0;
 486 }
 487
 488 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 489                                         int handle, int *p_fd)
 490 {
 491         int ret;
 492         struct dma_fence *fence;
 493         struct sync_file *sync_file;
 494         int fd = get_unused_fd_flags(O_CLOEXEC);
 495
 496         if (fd < 0)
 497                 return fd;
 498
 499         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 500         if (ret)
 501                 goto err_put_fd;
 502
 503         sync_file = sync_file_create(fence);
 504
 505         dma_fence_put(fence);
 506
 507         if (!sync_file) {
 508                 ret = -EINVAL;
 509                 goto err_put_fd;
 510         }
 511
 512         fd_install(fd, sync_file->file);
 513
 514         *p_fd = fd;
 515         return 0;
 516 err_put_fd:
 517         put_unused_fd(fd);
 518         return ret;
 519 }
 520 /**
 521  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 522  * @file_private: drm file-private structure to set up
 523  *
 524  * Called at device open time, sets up the structure for handling refcounting
 525  * of sync objects.
 526  */
 527 void
 528 drm_syncobj_open(struct drm_file *file_private)
 529 {
 530         idr_init_base(&file_private->syncobj_idr, 1);
 531         spin_lock_init(&file_private->syncobj_table_lock);
 532 }
 533
 534 static int
 535 drm_syncobj_release_handle(int id, void *ptr, void *data)
 536 {
 537         struct drm_syncobj *syncobj = ptr;
 538
 539         drm_syncobj_put(syncobj);
 540         return 0;
 541 }
 542
 543 /**
 544  * drm_syncobj_release - release file-private sync object resources
 545  * @file_private: drm file-private structure to clean up
 546  *
 547  * Called at close time when the filp is going away.
 548  *
 549  * Releases any remaining references on objects by this filp.
 550  */
 551 void
 552 drm_syncobj_release(struct drm_file *file_private)
 553 {
 554         idr_for_each(&file_private->syncobj_idr,
 555                      &drm_syncobj_release_handle, file_private);
 556         idr_destroy(&file_private->syncobj_idr);
 557 }
 558
 559 int
 560 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 561                          struct drm_file *file_private)
 562 {
 563         struct drm_syncobj_create *args = data;
 564
 565         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 566                 return -EOPNOTSUPP;
 567
 568         /* no valid flags yet */
 569         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 570                 return -EINVAL;
 571
 572         return drm_syncobj_create_as_handle(file_private,
 573                                             &args->handle, args->flags);
 574 }
 575
 576 int
 577 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 578                           struct drm_file *file_private)
 579 {
 580         struct drm_syncobj_destroy *args = data;
 581
 582         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 583                 return -EOPNOTSUPP;
 584
 585         /* make sure padding is empty */
 586         if (args->pad)
 587                 return -EINVAL;
 588         return drm_syncobj_destroy(file_private, args->handle);
 589 }
 590
 591 int
 592 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 593                                    struct drm_file *file_private)
 594 {
 595         struct drm_syncobj_handle *args = data;
 596
 597         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 598                 return -EOPNOTSUPP;
 599
 600         if (args->pad)
 601                 return -EINVAL;
 602
 603         if (args->flags != 0 &&
 604             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 605                 return -EINVAL;
 606
 607         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 608                 return drm_syncobj_export_sync_file(file_private, args->handle,
 609                                                     &args->fd);
 610
 611         return drm_syncobj_handle_to_fd(file_private, args->handle,
 612                                         &args->fd);
 613 }
 614
 615 int
 616 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 617                                    struct drm_file *file_private)
 618 {
 619         struct drm_syncobj_handle *args = data;
 620
 621         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 622                 return -EOPNOTSUPP;
 623
 624         if (args->pad)
 625                 return -EINVAL;
 626
 627         if (args->flags != 0 &&
 628             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 629                 return -EINVAL;
 630
 631         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 632                 return drm_syncobj_import_sync_file_fence(file_private,
 633                                                           args->fd,
 634                                                           args->handle);
 635
 636         return drm_syncobj_fd_to_handle(file_private, args->fd,
 637                                         &args->handle);
 638 }
 639
 640 struct syncobj_wait_entry {
 641         struct task_struct *task;
 642         struct dma_fence *fence;
 643         struct dma_fence_cb fence_cb;
 644         struct drm_syncobj_cb syncobj_cb;
 645 };
 646
 647 static void syncobj_wait_fence_func(struct dma_fence *fence,
 648                                     struct dma_fence_cb *cb)
 649 {
 650         struct syncobj_wait_entry *wait =
 651                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 652
 653         wake_up_process(wait->task);
 654 }
 655
 656 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 657                                       struct drm_syncobj_cb *cb)
 658 {
 659         struct syncobj_wait_entry *wait =
 660                 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
 661
 662         /* This happens inside the syncobj lock */
 663         wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 664                                                               lockdep_is_held(&syncobj->lock)));
 665         wake_up_process(wait->task);
 666 }
 667
 668 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 669                                                   uint32_t count,
 670                                                   uint32_t flags,
 671                                                   signed long timeout,
 672                                                   uint32_t *idx)
 673 {
 674         struct syncobj_wait_entry *entries;
 675         struct dma_fence *fence;
 676         uint32_t signaled_count, i;
 677
 678         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 679         if (!entries)
 680                 return -ENOMEM;
 681
 682         /* Walk the list of sync objects and initialize entries.  We do
 683          * this up-front so that we can properly return -EINVAL if there is
 684          * a syncobj with a missing fence and then never have the chance of
 685          * returning -EINVAL again.
 686          */
 687         signaled_count = 0;
 688         for (i = 0; i < count; ++i) {
 689                 entries[i].task = current;
 690                 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
 691                 if (!entries[i].fence) {
 692                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 693                                 continue;
 694                         } else {
 695                                 timeout = -EINVAL;
 696                                 goto cleanup_entries;
 697                         }
 698                 }
 699
 700                 if (dma_fence_is_signaled(entries[i].fence)) {
 701                         if (signaled_count == 0 && idx)
 702                                 *idx = i;
 703                         signaled_count++;
 704                 }
 705         }
 706
 707         if (signaled_count == count ||
 708             (signaled_count > 0 &&
 709              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 710                 goto cleanup_entries;
 711
 712         /* There's a very annoying laxness in the dma_fence API here, in
 713          * that backends are not required to automatically report when a
 714          * fence is signaled prior to fence->ops->enable_signaling() being
 715          * called.  So here if we fail to match signaled_count, we need to
 716          * fallthough and try a 0 timeout wait!
 717          */
 718
 719         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 720                 for (i = 0; i < count; ++i) {
 721                         drm_syncobj_fence_get_or_add_callback(syncobjs[i],
 722                                                               &entries[i].fence,
 723                                                               &entries[i].syncobj_cb,
 724                                                               syncobj_wait_syncobj_func);
 725                 }
 726         }
 727
 728         do {
 729                 set_current_state(TASK_INTERRUPTIBLE);
 730
 731                 signaled_count = 0;
 732                 for (i = 0; i < count; ++i) {
 733                         fence = entries[i].fence;
 734                         if (!fence)
 735                                 continue;
 736
 737                         if (dma_fence_is_signaled(fence) ||
 738                             (!entries[i].fence_cb.func &&
 739                              dma_fence_add_callback(fence,
 740                                                     &entries[i].fence_cb,
 741                                                     syncobj_wait_fence_func))) {
 742                                 /* The fence has been signaled */
 743                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 744                                         signaled_count++;
 745                                 } else {
 746                                         if (idx)
 747                                                 *idx = i;
 748                                         goto done_waiting;
 749                                 }
 750                         }
 751                 }
 752
 753                 if (signaled_count == count)
 754                         goto done_waiting;
 755
 756                 if (timeout == 0) {
 757                         timeout = -ETIME;
 758                         goto done_waiting;
 759                 }
 760
 761                 if (signal_pending(current)) {
 762                         timeout = -ERESTARTSYS;
 763                         goto done_waiting;
 764                 }
 765
 766                 timeout = schedule_timeout(timeout);
 767         } while (1);
 768
 769 done_waiting:
 770         __set_current_state(TASK_RUNNING);
 771
 772 cleanup_entries:
 773         for (i = 0; i < count; ++i) {
 774                 if (entries[i].syncobj_cb.func)
 775                         drm_syncobj_remove_callback(syncobjs[i],
 776                                                     &entries[i].syncobj_cb);
 777                 if (entries[i].fence_cb.func)
 778                         dma_fence_remove_callback(entries[i].fence,
 779                                                   &entries[i].fence_cb);
 780                 dma_fence_put(entries[i].fence);
 781         }
 782         kfree(entries);
 783
 784         return timeout;
 785 }
 786
 787 /**
 788  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 789  *
 790  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 791  *
 792  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 793  */
 794 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 795 {
 796         ktime_t abs_timeout, now;
 797         u64 timeout_ns, timeout_jiffies64;
 798
 799         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 800         if (timeout_nsec == 0)
 801                 return 0;
 802
 803         abs_timeout = ns_to_ktime(timeout_nsec);
 804         now = ktime_get();
 805
 806         if (!ktime_after(abs_timeout, now))
 807                 return 0;
 808
 809         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 810
 811         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 812         /*  clamp timeout to avoid infinite timeout */
 813         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 814                 return MAX_SCHEDULE_TIMEOUT - 1;
 815
 816         return timeout_jiffies64 + 1;
 817 }
 818
 819 static int drm_syncobj_array_wait(struct drm_device *dev,
 820                                   struct drm_file *file_private,
 821                                   struct drm_syncobj_wait *wait,
 822                                   struct drm_syncobj **syncobjs)
 823 {
 824         signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 825         uint32_t first = ~0;
 826
 827         timeout = drm_syncobj_array_wait_timeout(syncobjs,
 828                                                  wait->count_handles,
 829                                                  wait->flags,
 830                                                  timeout, &first);
 831         if (timeout < 0)
 832                 return timeout;
 833
 834         wait->first_signaled = first;
 835         return 0;
 836 }
 837
 838 static int drm_syncobj_array_find(struct drm_file *file_private,
 839                                   void __user *user_handles,
 840                                   uint32_t count_handles,
 841                                   struct drm_syncobj ***syncobjs_out)
 842 {
 843         uint32_t i, *handles;
 844         struct drm_syncobj **syncobjs;
 845         int ret;
 846
 847         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 848         if (handles == NULL)
 849                 return -ENOMEM;
 850
 851         if (copy_from_user(handles, user_handles,
 852                            sizeof(uint32_t) * count_handles)) {
 853                 ret = -EFAULT;
 854                 goto err_free_handles;
 855         }
 856
 857         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 858         if (syncobjs == NULL) {
 859                 ret = -ENOMEM;
 860                 goto err_free_handles;
 861         }
 862
 863         for (i = 0; i < count_handles; i++) {
 864                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 865                 if (!syncobjs[i]) {
 866                         ret = -ENOENT;
 867                         goto err_put_syncobjs;
 868                 }
 869         }
 870
 871         kfree(handles);
 872         *syncobjs_out = syncobjs;
 873         return 0;
 874
 875 err_put_syncobjs:
 876         while (i-- > 0)
 877                 drm_syncobj_put(syncobjs[i]);
 878         kfree(syncobjs);
 879 err_free_handles:
 880         kfree(handles);
 881
 882         return ret;
 883 }
 884
 885 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 886                                    uint32_t count)
 887 {
 888         uint32_t i;
 889         for (i = 0; i < count; i++)
 890                 drm_syncobj_put(syncobjs[i]);
 891         kfree(syncobjs);
 892 }
 893
 894 int
 895 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 896                        struct drm_file *file_private)
 897 {
 898         struct drm_syncobj_wait *args = data;
 899         struct drm_syncobj **syncobjs;
 900         int ret = 0;
 901
 902         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 903                 return -EOPNOTSUPP;
 904
 905         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 906                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 907                 return -EINVAL;
 908
 909         if (args->count_handles == 0)
 910                 return -EINVAL;
 911
 912         ret = drm_syncobj_array_find(file_private,
 913                                      u64_to_user_ptr(args->handles),
 914                                      args->count_handles,
 915                                      &syncobjs);
 916         if (ret < 0)
 917                 return ret;
 918
 919         ret = drm_syncobj_array_wait(dev, file_private,
 920                                      args, syncobjs);
 921
 922         drm_syncobj_array_free(syncobjs, args->count_handles);
 923
 924         return ret;
 925 }
 926
 927 int
 928 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 929                         struct drm_file *file_private)
 930 {
 931         struct drm_syncobj_array *args = data;
 932         struct drm_syncobj **syncobjs;
 933         uint32_t i;
 934         int ret;
 935
 936         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 937                 return -EOPNOTSUPP;
 938
 939         if (args->pad != 0)
 940                 return -EINVAL;
 941
 942         if (args->count_handles == 0)
 943                 return -EINVAL;
 944
 945         ret = drm_syncobj_array_find(file_private,
 946                                      u64_to_user_ptr(args->handles),
 947                                      args->count_handles,
 948                                      &syncobjs);
 949         if (ret < 0)
 950                 return ret;
 951
 952         for (i = 0; i < args->count_handles; i++)
 953                 drm_syncobj_replace_fence(syncobjs[i], 0, NULL);
 954
 955         drm_syncobj_array_free(syncobjs, args->count_handles);
 956
 957         return 0;
 958 }
 959
 960 int
 961 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 962                          struct drm_file *file_private)
 963 {
 964         struct drm_syncobj_array *args = data;
 965         struct drm_syncobj **syncobjs;
 966         uint32_t i;
 967         int ret;
 968
 969         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 970                 return -EOPNOTSUPP;
 971
 972         if (args->pad != 0)
 973                 return -EINVAL;
 974
 975         if (args->count_handles == 0)
 976                 return -EINVAL;
 977
 978         ret = drm_syncobj_array_find(file_private,
 979                                      u64_to_user_ptr(args->handles),
 980                                      args->count_handles,
 981                                      &syncobjs);
 982         if (ret < 0)
 983                 return ret;
 984
 985         for (i = 0; i < args->count_handles; i++) {
 986                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
 987                 if (ret < 0)
 988                         break;
 989         }
 990
 991         drm_syncobj_array_free(syncobjs, args->count_handles);
 992
 993         return ret;
 994 }