drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
  33  * container for a synchronization primitive which can be used by userspace
  34  * to explicitly synchronize GPU commands, can be shared between userspace
  35  * processes, and can be shared between different DRM drivers.
  36  * Their primary use-case is to implement Vulkan fences and semaphores.
  37  * The syncobj userspace API provides ioctls for several operations:
  38  *
  39  *  - Creation and destruction of syncobjs
  40  *  - Import and export of syncobjs to/from a syncobj file descriptor
  41  *  - Import and export a syncobj's underlying fence to/from a sync file
  42  *  - Reset a syncobj (set its fence to NULL)
  43  *  - Signal a syncobj (set a trivially signaled fence)
  44  *  - Wait for a syncobj's fence to appear and be signaled
  45  *
  46  * The syncobj userspace API also provides operations to manipulate a syncobj
  47  * in terms of a timeline of struct &dma_fence_chain rather than a single
  48  * struct &dma_fence, through the following operations:
  49  *
  50  *   - Signal a given point on the timeline
  51  *   - Wait for a given point to appear and/or be signaled
  52  *   - Import and export from/to a given point of a timeline
  53  *
  54  * At it's core, a syncobj is simply a wrapper around a pointer to a struct
  55  * &dma_fence which may be NULL.
  56  * When a syncobj is first created, its pointer is either NULL or a pointer
  57  * to an already signaled fence depending on whether the
  58  * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
  59  * &DRM_IOCTL_SYNCOBJ_CREATE.
  60  *
  61  * If the syncobj is considered as a binary (its state is either signaled or
  62  * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
  63  * the syncobj, the syncobj's fence is replaced with a fence which will be
  64  * signaled by the completion of that work.
  65  * If the syncobj is considered as a timeline primitive, when GPU work is
  66  * enqueued in a DRM driver to signal the a given point of the syncobj, a new
  67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
  68  * pointing to the previous fence that was in the syncobj. The new struct
  69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
  70  * completion of the DRM driver's work and also any work associated with the
  71  * fence previously in the syncobj.
  72  *
  73  * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
  74  * time the work is enqueued, it waits on the syncobj's fence before
  75  * submitting the work to hardware. That fence is either :
  76  *
  77  *    - The syncobj's current fence if the syncobj is considered as a binary
  78  *      primitive.
  79  *    - The struct &dma_fence associated with a given point if the syncobj is
  80  *      considered as a timeline primitive.
  81  *
  82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
  83  * the enqueue operation is expected to fail.
  84  *
  85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
  86  * terms of the current fence at the time the ioctl is called by userspace
  87  * regardless of whether that operation is an immediate host-side operation
  88  * (signal or reset) or or an operation which is enqueued in some driver
  89  * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
  90  * to manipulate a syncobj from the host by resetting its pointer to NULL or
  91  * setting its pointer to a fence which is already signaled.
  92  *
  93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
  94  * terms of a u64 value referring to point in the timeline. See
  95  * dma_fence_chain_find_seqno() to see how a given point is found in the
  96  * timeline.
  97  *
  98  * Note that applications should be careful to always use timeline set of
  99  * ioctl() when dealing with syncobj considered as timeline. Using a binary
 100  * set of ioctl() with a syncobj considered as timeline could result incorrect
 101  * synchronization. The use of binary syncobj is supported through the
 102  * timeline set of ioctl() by using a point value of 0, this will reproduce
 103  * the behavior of the binary set of ioctl() (for example replace the
 104  * syncobj's fence when signaling).
 105  *
 106  *
 107  * Host-side wait on syncobjs
 108  * --------------------------
 109  *
 110  * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
 111  * host-side wait on all of the syncobj fences simultaneously.
 112  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
 113  * all of the syncobj fences to be signaled before it returns.
 114  * Otherwise, it returns once at least one syncobj fence has been signaled
 115  * and the index of a signaled fence is written back to the client.
 116  *
 117  * Unlike the enqueued GPU work dependencies which fail if they see a NULL
 118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
 119  * the host-side wait will first wait for the syncobj to receive a non-NULL
 120  * fence and then wait on that fence.
 121  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
 122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
 123  * Assuming the syncobj starts off with a NULL fence, this allows a client
 124  * to do a host wait in one thread (or process) which waits on GPU work
 125  * submitted in another thread (or process) without having to manually
 126  * synchronize between the two.
 127  * This requirement is inherited from the Vulkan fence API.
 128  *
 129  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set
 130  * a fence deadline hint on the backing fences before waiting, to provide the
 131  * fence signaler with an appropriate sense of urgency.  The deadline is
 132  * specified as an absolute &CLOCK_MONOTONIC value in units of ns.
 133  *
 134  * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
 135  * handles as well as an array of u64 points and does a host-side wait on all
 136  * of syncobj fences at the given points simultaneously.
 137  *
 138  * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
 139  * fence to materialize on the timeline without waiting for the fence to be
 140  * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
 141  * requirement is inherited from the wait-before-signal behavior required by
 142  * the Vulkan timeline semaphore API.
 143  *
 144  * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
 145  * blocking: an eventfd will be signaled when the syncobj is. This is useful to
 146  * integrate the wait in an event loop.
 147  *
 148  *
 149  * Import/export of syncobjs
 150  * -------------------------
 151  *
 152  * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
 153  * provide two mechanisms for import/export of syncobjs.
 154  *
 155  * The first lets the client import or export an entire syncobj to a file
 156  * descriptor.
 157  * These fd's are opaque and have no other use case, except passing the
 158  * syncobj between processes.
 159  * All exported file descriptors and any syncobj handles created as a
 160  * result of importing those file descriptors own a reference to the
 161  * same underlying struct &drm_syncobj and the syncobj can be used
 162  * persistently across all the processes with which it is shared.
 163  * The syncobj is freed only once the last reference is dropped.
 164  * Unlike dma-buf, importing a syncobj creates a new handle (with its own
 165  * reference) for every import instead of de-duplicating.
 166  * The primary use-case of this persistent import/export is for shared
 167  * Vulkan fences and semaphores.
 168  *
 169  * The second import/export mechanism, which is indicated by
 170  * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
 171  * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
 172  * import/export the syncobj's current fence from/to a &sync_file.
 173  * When a syncobj is exported to a sync file, that sync file wraps the
 174  * sycnobj's fence at the time of export and any later signal or reset
 175  * operations on the syncobj will not affect the exported sync file.
 176  * When a sync file is imported into a syncobj, the syncobj's fence is set
 177  * to the fence wrapped by that sync file.
 178  * Because sync files are immutable, resetting or signaling the syncobj
 179  * will not affect any sync files whose fences have been imported into the
 180  * syncobj.
 181  *
 182  *
 183  * Import/export of timeline points in timeline syncobjs
 184  * -----------------------------------------------------
 185  *
 186  * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
 187  * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
 188  * into another syncobj.
 189  *
 190  * Note that if you want to transfer a struct &dma_fence_chain from a given
 191  * point on a timeline syncobj from/into a binary syncobj, you can use the
 192  * point 0 to mean take/replace the fence in the syncobj.
 193  */
 194
 195 #include <linux/anon_inodes.h>
 196 #include <linux/dma-fence-unwrap.h>
 197 #include <linux/eventfd.h>
 198 #include <linux/file.h>
 199 #include <linux/fs.h>
 200 #include <linux/sched/signal.h>
 201 #include <linux/sync_file.h>
 202 #include <linux/uaccess.h>
 203
 204 #include <drm/drm.h>
 205 #include <drm/drm_drv.h>
 206 #include <drm/drm_file.h>
 207 #include <drm/drm_gem.h>
 208 #include <drm/drm_print.h>
 209 #include <drm/drm_syncobj.h>
 210 #include <drm/drm_utils.h>
 211
 212 #include "drm_internal.h"
 213
 214 struct syncobj_wait_entry {
 215         struct list_head node;
 216         struct task_struct *task;
 217         struct dma_fence *fence;
 218         struct dma_fence_cb fence_cb;
 219         u64    point;
 220 };
 221
 222 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 223                                       struct syncobj_wait_entry *wait);
 224
 225 struct syncobj_eventfd_entry {
 226         struct list_head node;
 227         struct dma_fence *fence;
 228         struct dma_fence_cb fence_cb;
 229         struct drm_syncobj *syncobj;
 230         struct eventfd_ctx *ev_fd_ctx;
 231         u64 point;
 232         u32 flags;
 233 };
 234
 235 static void
 236 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
 237                            struct syncobj_eventfd_entry *entry);
 238
 239 /**
 240  * drm_syncobj_find - lookup and reference a sync object.
 241  * @file_private: drm file private pointer
 242  * @handle: sync object handle to lookup.
 243  *
 244  * Returns a reference to the syncobj pointed to by handle or NULL. The
 245  * reference must be released by calling drm_syncobj_put().
 246  */
 247 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 248                                      u32 handle)
 249 {
 250         struct drm_syncobj *syncobj;
 251
 252         spin_lock(&file_private->syncobj_table_lock);
 253
 254         /* Check if we currently have a reference on the object */
 255         syncobj = idr_find(&file_private->syncobj_idr, handle);
 256         if (syncobj)
 257                 drm_syncobj_get(syncobj);
 258
 259         spin_unlock(&file_private->syncobj_table_lock);
 260
 261         return syncobj;
 262 }
 263 EXPORT_SYMBOL(drm_syncobj_find);
 264
 265 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 266                                        struct syncobj_wait_entry *wait)
 267 {
 268         struct dma_fence *fence;
 269
 270         if (wait->fence)
 271                 return;
 272
 273         spin_lock(&syncobj->lock);
 274         /* We've already tried once to get a fence and failed.  Now that we
 275          * have the lock, try one more time just to be sure we don't add a
 276          * callback when a fence has already been set.
 277          */
 278         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 279         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 280                 dma_fence_put(fence);
 281                 list_add_tail(&wait->node, &syncobj->cb_list);
 282         } else if (!fence) {
 283                 wait->fence = dma_fence_get_stub();
 284         } else {
 285                 wait->fence = fence;
 286         }
 287         spin_unlock(&syncobj->lock);
 288 }
 289
 290 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 291                                     struct syncobj_wait_entry *wait)
 292 {
 293         if (!wait->node.next)
 294                 return;
 295
 296         spin_lock(&syncobj->lock);
 297         list_del_init(&wait->node);
 298         spin_unlock(&syncobj->lock);
 299 }
 300
 301 static void
 302 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
 303 {
 304         eventfd_ctx_put(entry->ev_fd_ctx);
 305         dma_fence_put(entry->fence);
 306         /* This happens either inside the syncobj lock, or after the node has
 307          * already been removed from the list.
 308          */
 309         list_del(&entry->node);
 310         kfree(entry);
 311 }
 312
 313 static void
 314 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
 315                         struct syncobj_eventfd_entry *entry)
 316 {
 317         spin_lock(&syncobj->lock);
 318         list_add_tail(&entry->node, &syncobj->ev_fd_list);
 319         syncobj_eventfd_entry_func(syncobj, entry);
 320         spin_unlock(&syncobj->lock);
 321 }
 322
 323 /**
 324  * drm_syncobj_add_point - add new timeline point to the syncobj
 325  * @syncobj: sync object to add timeline point do
 326  * @chain: chain node to use to add the point
 327  * @fence: fence to encapsulate in the chain node
 328  * @point: sequence number to use for the point
 329  *
 330  * Add the chain node as new timeline point to the syncobj.
 331  */
 332 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 333                            struct dma_fence_chain *chain,
 334                            struct dma_fence *fence,
 335                            uint64_t point)
 336 {
 337         struct syncobj_wait_entry *wait_cur, *wait_tmp;
 338         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 339         struct dma_fence *prev;
 340
 341         dma_fence_get(fence);
 342
 343         spin_lock(&syncobj->lock);
 344
 345         prev = drm_syncobj_fence_get(syncobj);
 346         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 347         if (prev && prev->seqno >= point)
 348                 DRM_DEBUG("You are adding an unorder point to timeline!\n");
 349         dma_fence_chain_init(chain, prev, fence, point);
 350         rcu_assign_pointer(syncobj->fence, &chain->base);
 351
 352         list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
 353                 syncobj_wait_syncobj_func(syncobj, wait_cur);
 354         list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 355                 syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
 356         spin_unlock(&syncobj->lock);
 357
 358         /* Walk the chain once to trigger garbage collection */
 359         dma_fence_chain_for_each(fence, prev);
 360         dma_fence_put(prev);
 361 }
 362 EXPORT_SYMBOL(drm_syncobj_add_point);
 363
 364 /**
 365  * drm_syncobj_replace_fence - replace fence in a sync object.
 366  * @syncobj: Sync object to replace fence in
 367  * @fence: fence to install in sync file.
 368  *
 369  * This replaces the fence on a sync object.
 370  */
 371 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 372                                struct dma_fence *fence)
 373 {
 374         struct dma_fence *old_fence;
 375         struct syncobj_wait_entry *wait_cur, *wait_tmp;
 376         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 377
 378         if (fence)
 379                 dma_fence_get(fence);
 380
 381         spin_lock(&syncobj->lock);
 382
 383         old_fence = rcu_dereference_protected(syncobj->fence,
 384                                               lockdep_is_held(&syncobj->lock));
 385         rcu_assign_pointer(syncobj->fence, fence);
 386
 387         if (fence != old_fence) {
 388                 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
 389                         syncobj_wait_syncobj_func(syncobj, wait_cur);
 390                 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 391                         syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
 392         }
 393
 394         spin_unlock(&syncobj->lock);
 395
 396         dma_fence_put(old_fence);
 397 }
 398 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 399
 400 /**
 401  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 402  * @syncobj: sync object to assign the fence on
 403  *
 404  * Assign a already signaled stub fence to the sync object.
 405  */
 406 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 407 {
 408         struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());
 409
 410         if (!fence)
 411                 return -ENOMEM;
 412
 413         drm_syncobj_replace_fence(syncobj, fence);
 414         dma_fence_put(fence);
 415         return 0;
 416 }
 417
 418 /* 5s default for wait submission */
 419 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 420 /**
 421  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 422  * @file_private: drm file private pointer
 423  * @handle: sync object handle to lookup.
 424  * @point: timeline point
 425  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 426  * @fence: out parameter for the fence
 427  *
 428  * This is just a convenience function that combines drm_syncobj_find() and
 429  * drm_syncobj_fence_get().
 430  *
 431  * Returns 0 on success or a negative error value on failure. On success @fence
 432  * contains a reference to the fence, which must be released by calling
 433  * dma_fence_put().
 434  */
 435 int drm_syncobj_find_fence(struct drm_file *file_private,
 436                            u32 handle, u64 point, u64 flags,
 437                            struct dma_fence **fence)
 438 {
 439         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 440         struct syncobj_wait_entry wait;
 441         u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 442         int ret;
 443
 444         if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
 445                 return -EINVAL;
 446
 447         if (!syncobj)
 448                 return -ENOENT;
 449
 450         /* Waiting for userspace with locks help is illegal cause that can
 451          * trivial deadlock with page faults for example. Make lockdep complain
 452          * about it early on.
 453          */
 454         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 455                 might_sleep();
 456                 lockdep_assert_none_held_once();
 457         }
 458
 459         *fence = drm_syncobj_fence_get(syncobj);
 460
 461         if (*fence) {
 462                 ret = dma_fence_chain_find_seqno(fence, point);
 463                 if (!ret) {
 464                         /* If the requested seqno is already signaled
 465                          * drm_syncobj_find_fence may return a NULL
 466                          * fence. To make sure the recipient gets
 467                          * signalled, use a new fence instead.
 468                          */
 469                         if (!*fence)
 470                                 *fence = dma_fence_get_stub();
 471
 472                         goto out;
 473                 }
 474                 dma_fence_put(*fence);
 475         } else {
 476                 ret = -EINVAL;
 477         }
 478
 479         if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 480                 goto out;
 481
 482         memset(&wait, 0, sizeof(wait));
 483         wait.task = current;
 484         wait.point = point;
 485         drm_syncobj_fence_add_wait(syncobj, &wait);
 486
 487         do {
 488                 set_current_state(TASK_INTERRUPTIBLE);
 489                 if (wait.fence) {
 490                         ret = 0;
 491                         break;
 492                 }
 493                 if (timeout == 0) {
 494                         ret = -ETIME;
 495                         break;
 496                 }
 497
 498                 if (signal_pending(current)) {
 499                         ret = -ERESTARTSYS;
 500                         break;
 501                 }
 502
 503                 timeout = schedule_timeout(timeout);
 504         } while (1);
 505
 506         __set_current_state(TASK_RUNNING);
 507         *fence = wait.fence;
 508
 509         if (wait.node.next)
 510                 drm_syncobj_remove_wait(syncobj, &wait);
 511
 512 out:
 513         drm_syncobj_put(syncobj);
 514
 515         return ret;
 516 }
 517 EXPORT_SYMBOL(drm_syncobj_find_fence);
 518
 519 /**
 520  * drm_syncobj_free - free a sync object.
 521  * @kref: kref to free.
 522  *
 523  * Only to be called from kref_put in drm_syncobj_put.
 524  */
 525 void drm_syncobj_free(struct kref *kref)
 526 {
 527         struct drm_syncobj *syncobj = container_of(kref,
 528                                                    struct drm_syncobj,
 529                                                    refcount);
 530         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 531
 532         drm_syncobj_replace_fence(syncobj, NULL);
 533
 534         list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 535                 syncobj_eventfd_entry_free(ev_fd_cur);
 536
 537         kfree(syncobj);
 538 }
 539 EXPORT_SYMBOL(drm_syncobj_free);
 540
 541 /**
 542  * drm_syncobj_create - create a new syncobj
 543  * @out_syncobj: returned syncobj
 544  * @flags: DRM_SYNCOBJ_* flags
 545  * @fence: if non-NULL, the syncobj will represent this fence
 546  *
 547  * This is the first function to create a sync object. After creating, drivers
 548  * probably want to make it available to userspace, either through
 549  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 550  *
 551  * Returns 0 on success or a negative error value on failure.
 552  */
 553 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 554                        struct dma_fence *fence)
 555 {
 556         int ret;
 557         struct drm_syncobj *syncobj;
 558
 559         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 560         if (!syncobj)
 561                 return -ENOMEM;
 562
 563         kref_init(&syncobj->refcount);
 564         INIT_LIST_HEAD(&syncobj->cb_list);
 565         INIT_LIST_HEAD(&syncobj->ev_fd_list);
 566         spin_lock_init(&syncobj->lock);
 567
 568         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 569                 ret = drm_syncobj_assign_null_handle(syncobj);
 570                 if (ret < 0) {
 571                         drm_syncobj_put(syncobj);
 572                         return ret;
 573                 }
 574         }
 575
 576         if (fence)
 577                 drm_syncobj_replace_fence(syncobj, fence);
 578
 579         *out_syncobj = syncobj;
 580         return 0;
 581 }
 582 EXPORT_SYMBOL(drm_syncobj_create);
 583
 584 /**
 585  * drm_syncobj_get_handle - get a handle from a syncobj
 586  * @file_private: drm file private pointer
 587  * @syncobj: Sync object to export
 588  * @handle: out parameter with the new handle
 589  *
 590  * Exports a sync object created with drm_syncobj_create() as a handle on
 591  * @file_private to userspace.
 592  *
 593  * Returns 0 on success or a negative error value on failure.
 594  */
 595 int drm_syncobj_get_handle(struct drm_file *file_private,
 596                            struct drm_syncobj *syncobj, u32 *handle)
 597 {
 598         int ret;
 599
 600         /* take a reference to put in the idr */
 601         drm_syncobj_get(syncobj);
 602
 603         idr_preload(GFP_KERNEL);
 604         spin_lock(&file_private->syncobj_table_lock);
 605         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 606         spin_unlock(&file_private->syncobj_table_lock);
 607
 608         idr_preload_end();
 609
 610         if (ret < 0) {
 611                 drm_syncobj_put(syncobj);
 612                 return ret;
 613         }
 614
 615         *handle = ret;
 616         return 0;
 617 }
 618 EXPORT_SYMBOL(drm_syncobj_get_handle);
 619
 620 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 621                                         u32 *handle, uint32_t flags)
 622 {
 623         int ret;
 624         struct drm_syncobj *syncobj;
 625
 626         ret = drm_syncobj_create(&syncobj, flags, NULL);
 627         if (ret)
 628                 return ret;
 629
 630         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 631         drm_syncobj_put(syncobj);
 632         return ret;
 633 }
 634
 635 static int drm_syncobj_destroy(struct drm_file *file_private,
 636                                u32 handle)
 637 {
 638         struct drm_syncobj *syncobj;
 639
 640         spin_lock(&file_private->syncobj_table_lock);
 641         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 642         spin_unlock(&file_private->syncobj_table_lock);
 643
 644         if (!syncobj)
 645                 return -EINVAL;
 646
 647         drm_syncobj_put(syncobj);
 648         return 0;
 649 }
 650
 651 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 652 {
 653         struct drm_syncobj *syncobj = file->private_data;
 654
 655         drm_syncobj_put(syncobj);
 656         return 0;
 657 }
 658
 659 static const struct file_operations drm_syncobj_file_fops = {
 660         .release = drm_syncobj_file_release,
 661 };
 662
 663 /**
 664  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 665  * @syncobj: Sync object to export
 666  * @p_fd: out parameter with the new file descriptor
 667  *
 668  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 669  *
 670  * Returns 0 on success or a negative error value on failure.
 671  */
 672 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 673 {
 674         struct file *file;
 675         int fd;
 676
 677         fd = get_unused_fd_flags(O_CLOEXEC);
 678         if (fd < 0)
 679                 return fd;
 680
 681         file = anon_inode_getfile("syncobj_file",
 682                                   &drm_syncobj_file_fops,
 683                                   syncobj, 0);
 684         if (IS_ERR(file)) {
 685                 put_unused_fd(fd);
 686                 return PTR_ERR(file);
 687         }
 688
 689         drm_syncobj_get(syncobj);
 690         fd_install(fd, file);
 691
 692         *p_fd = fd;
 693         return 0;
 694 }
 695 EXPORT_SYMBOL(drm_syncobj_get_fd);
 696
 697 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 698                                     u32 handle, int *p_fd)
 699 {
 700         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 701         int ret;
 702
 703         if (!syncobj)
 704                 return -EINVAL;
 705
 706         ret = drm_syncobj_get_fd(syncobj, p_fd);
 707         drm_syncobj_put(syncobj);
 708         return ret;
 709 }
 710
 711 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 712                                     int fd, u32 *handle)
 713 {
 714         struct drm_syncobj *syncobj;
 715         struct fd f = fdget(fd);
 716         int ret;
 717
 718         if (!fd_file(f))
 719                 return -EINVAL;
 720
 721         if (fd_file(f)->f_op != &drm_syncobj_file_fops) {
 722                 fdput(f);
 723                 return -EINVAL;
 724         }
 725
 726         /* take a reference to put in the idr */
 727         syncobj = fd_file(f)->private_data;
 728         drm_syncobj_get(syncobj);
 729
 730         idr_preload(GFP_KERNEL);
 731         spin_lock(&file_private->syncobj_table_lock);
 732         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 733         spin_unlock(&file_private->syncobj_table_lock);
 734         idr_preload_end();
 735
 736         if (ret > 0) {
 737                 *handle = ret;
 738                 ret = 0;
 739         } else
 740                 drm_syncobj_put(syncobj);
 741
 742         fdput(f);
 743         return ret;
 744 }
 745
 746 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 747                                               int fd, int handle)
 748 {
 749         struct dma_fence *fence = sync_file_get_fence(fd);
 750         struct drm_syncobj *syncobj;
 751
 752         if (!fence)
 753                 return -EINVAL;
 754
 755         syncobj = drm_syncobj_find(file_private, handle);
 756         if (!syncobj) {
 757                 dma_fence_put(fence);
 758                 return -ENOENT;
 759         }
 760
 761         drm_syncobj_replace_fence(syncobj, fence);
 762         dma_fence_put(fence);
 763         drm_syncobj_put(syncobj);
 764         return 0;
 765 }
 766
 767 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 768                                         int handle, int *p_fd)
 769 {
 770         int ret;
 771         struct dma_fence *fence;
 772         struct sync_file *sync_file;
 773         int fd = get_unused_fd_flags(O_CLOEXEC);
 774
 775         if (fd < 0)
 776                 return fd;
 777
 778         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 779         if (ret)
 780                 goto err_put_fd;
 781
 782         sync_file = sync_file_create(fence);
 783
 784         dma_fence_put(fence);
 785
 786         if (!sync_file) {
 787                 ret = -EINVAL;
 788                 goto err_put_fd;
 789         }
 790
 791         fd_install(fd, sync_file->file);
 792
 793         *p_fd = fd;
 794         return 0;
 795 err_put_fd:
 796         put_unused_fd(fd);
 797         return ret;
 798 }
 799 /**
 800  * drm_syncobj_open - initializes syncobj file-private structures at devnode open time
 801  * @file_private: drm file-private structure to set up
 802  *
 803  * Called at device open time, sets up the structure for handling refcounting
 804  * of sync objects.
 805  */
 806 void
 807 drm_syncobj_open(struct drm_file *file_private)
 808 {
 809         idr_init_base(&file_private->syncobj_idr, 1);
 810         spin_lock_init(&file_private->syncobj_table_lock);
 811 }
 812
 813 static int
 814 drm_syncobj_release_handle(int id, void *ptr, void *data)
 815 {
 816         struct drm_syncobj *syncobj = ptr;
 817
 818         drm_syncobj_put(syncobj);
 819         return 0;
 820 }
 821
 822 /**
 823  * drm_syncobj_release - release file-private sync object resources
 824  * @file_private: drm file-private structure to clean up
 825  *
 826  * Called at close time when the filp is going away.
 827  *
 828  * Releases any remaining references on objects by this filp.
 829  */
 830 void
 831 drm_syncobj_release(struct drm_file *file_private)
 832 {
 833         idr_for_each(&file_private->syncobj_idr,
 834                      &drm_syncobj_release_handle, file_private);
 835         idr_destroy(&file_private->syncobj_idr);
 836 }
 837
 838 int
 839 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 840                          struct drm_file *file_private)
 841 {
 842         struct drm_syncobj_create *args = data;
 843
 844         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 845                 return -EOPNOTSUPP;
 846
 847         /* no valid flags yet */
 848         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 849                 return -EINVAL;
 850
 851         return drm_syncobj_create_as_handle(file_private,
 852                                             &args->handle, args->flags);
 853 }
 854
 855 int
 856 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 857                           struct drm_file *file_private)
 858 {
 859         struct drm_syncobj_destroy *args = data;
 860
 861         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 862                 return -EOPNOTSUPP;
 863
 864         /* make sure padding is empty */
 865         if (args->pad)
 866                 return -EINVAL;
 867         return drm_syncobj_destroy(file_private, args->handle);
 868 }
 869
 870 int
 871 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 872                                    struct drm_file *file_private)
 873 {
 874         struct drm_syncobj_handle *args = data;
 875
 876         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 877                 return -EOPNOTSUPP;
 878
 879         if (args->pad)
 880                 return -EINVAL;
 881
 882         if (args->flags != 0 &&
 883             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 884                 return -EINVAL;
 885
 886         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 887                 return drm_syncobj_export_sync_file(file_private, args->handle,
 888                                                     &args->fd);
 889
 890         return drm_syncobj_handle_to_fd(file_private, args->handle,
 891                                         &args->fd);
 892 }
 893
 894 int
 895 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 896                                    struct drm_file *file_private)
 897 {
 898         struct drm_syncobj_handle *args = data;
 899
 900         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 901                 return -EOPNOTSUPP;
 902
 903         if (args->pad)
 904                 return -EINVAL;
 905
 906         if (args->flags != 0 &&
 907             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 908                 return -EINVAL;
 909
 910         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 911                 return drm_syncobj_import_sync_file_fence(file_private,
 912                                                           args->fd,
 913                                                           args->handle);
 914
 915         return drm_syncobj_fd_to_handle(file_private, args->fd,
 916                                         &args->handle);
 917 }
 918
 919 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
 920                                             struct drm_syncobj_transfer *args)
 921 {
 922         struct drm_syncobj *timeline_syncobj = NULL;
 923         struct dma_fence *fence, *tmp;
 924         struct dma_fence_chain *chain;
 925         int ret;
 926
 927         timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 928         if (!timeline_syncobj) {
 929                 return -ENOENT;
 930         }
 931         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 932                                      args->src_point, args->flags,
 933                                      &tmp);
 934         if (ret)
 935                 goto err_put_timeline;
 936
 937         fence = dma_fence_unwrap_merge(tmp);
 938         dma_fence_put(tmp);
 939         if (!fence) {
 940                 ret = -ENOMEM;
 941                 goto err_put_timeline;
 942         }
 943
 944         chain = dma_fence_chain_alloc();
 945         if (!chain) {
 946                 ret = -ENOMEM;
 947                 goto err_free_fence;
 948         }
 949
 950         drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
 951 err_free_fence:
 952         dma_fence_put(fence);
 953 err_put_timeline:
 954         drm_syncobj_put(timeline_syncobj);
 955
 956         return ret;
 957 }
 958
 959 static int
 960 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
 961                                struct drm_syncobj_transfer *args)
 962 {
 963         struct drm_syncobj *binary_syncobj = NULL;
 964         struct dma_fence *fence;
 965         int ret;
 966
 967         binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 968         if (!binary_syncobj)
 969                 return -ENOENT;
 970         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 971                                      args->src_point, args->flags, &fence);
 972         if (ret)
 973                 goto err;
 974         drm_syncobj_replace_fence(binary_syncobj, fence);
 975         dma_fence_put(fence);
 976 err:
 977         drm_syncobj_put(binary_syncobj);
 978
 979         return ret;
 980 }
 981 int
 982 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
 983                            struct drm_file *file_private)
 984 {
 985         struct drm_syncobj_transfer *args = data;
 986         int ret;
 987
 988         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
 989                 return -EOPNOTSUPP;
 990
 991         if (args->pad)
 992                 return -EINVAL;
 993
 994         if (args->dst_point)
 995                 ret = drm_syncobj_transfer_to_timeline(file_private, args);
 996         else
 997                 ret = drm_syncobj_transfer_to_binary(file_private, args);
 998
 999         return ret;
1000 }
1001
1002 static void syncobj_wait_fence_func(struct dma_fence *fence,
1003                                     struct dma_fence_cb *cb)
1004 {
1005         struct syncobj_wait_entry *wait =
1006                 container_of(cb, struct syncobj_wait_entry, fence_cb);
1007
1008         wake_up_process(wait->task);
1009 }
1010
1011 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1012                                       struct syncobj_wait_entry *wait)
1013 {
1014         struct dma_fence *fence;
1015
1016         /* This happens inside the syncobj lock */
1017         fence = rcu_dereference_protected(syncobj->fence,
1018                                           lockdep_is_held(&syncobj->lock));
1019         dma_fence_get(fence);
1020         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
1021                 dma_fence_put(fence);
1022                 return;
1023         } else if (!fence) {
1024                 wait->fence = dma_fence_get_stub();
1025         } else {
1026                 wait->fence = fence;
1027         }
1028
1029         wake_up_process(wait->task);
1030         list_del_init(&wait->node);
1031 }
1032
1033 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1034                                                   void __user *user_points,
1035                                                   uint32_t count,
1036                                                   uint32_t flags,
1037                                                   signed long timeout,
1038                                                   uint32_t *idx,
1039                                                   ktime_t *deadline)
1040 {
1041         struct syncobj_wait_entry *entries;
1042         struct dma_fence *fence;
1043         uint64_t *points;
1044         uint32_t signaled_count, i;
1045
1046         if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1047                      DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1048                 might_sleep();
1049                 lockdep_assert_none_held_once();
1050         }
1051
1052         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1053         if (points == NULL)
1054                 return -ENOMEM;
1055
1056         if (!user_points) {
1057                 memset(points, 0, count * sizeof(uint64_t));
1058
1059         } else if (copy_from_user(points, user_points,
1060                                   sizeof(uint64_t) * count)) {
1061                 timeout = -EFAULT;
1062                 goto err_free_points;
1063         }
1064
1065         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1066         if (!entries) {
1067                 timeout = -ENOMEM;
1068                 goto err_free_points;
1069         }
1070         /* Walk the list of sync objects and initialize entries.  We do
1071          * this up-front so that we can properly return -EINVAL if there is
1072          * a syncobj with a missing fence and then never have the chance of
1073          * returning -EINVAL again.
1074          */
1075         signaled_count = 0;
1076         for (i = 0; i < count; ++i) {
1077                 struct dma_fence *fence;
1078
1079                 entries[i].task = current;
1080                 entries[i].point = points[i];
1081                 fence = drm_syncobj_fence_get(syncobjs[i]);
1082                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
1083                         dma_fence_put(fence);
1084                         if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1085                                      DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1086                                 continue;
1087                         } else {
1088                                 timeout = -EINVAL;
1089                                 goto cleanup_entries;
1090                         }
1091                 }
1092
1093                 if (fence)
1094                         entries[i].fence = fence;
1095                 else
1096                         entries[i].fence = dma_fence_get_stub();
1097
1098                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1099                     dma_fence_is_signaled(entries[i].fence)) {
1100                         if (signaled_count == 0 && idx)
1101                                 *idx = i;
1102                         signaled_count++;
1103                 }
1104         }
1105
1106         if (signaled_count == count ||
1107             (signaled_count > 0 &&
1108              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1109                 goto cleanup_entries;
1110
1111         /* There's a very annoying laxness in the dma_fence API here, in
1112          * that backends are not required to automatically report when a
1113          * fence is signaled prior to fence->ops->enable_signaling() being
1114          * called.  So here if we fail to match signaled_count, we need to
1115          * fallthough and try a 0 timeout wait!
1116          */
1117
1118         if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1119                      DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1120                 for (i = 0; i < count; ++i)
1121                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1122         }
1123
1124         if (deadline) {
1125                 for (i = 0; i < count; ++i) {
1126                         fence = entries[i].fence;
1127                         if (!fence)
1128                                 continue;
1129                         dma_fence_set_deadline(fence, *deadline);
1130                 }
1131         }
1132
1133         do {
1134                 set_current_state(TASK_INTERRUPTIBLE);
1135
1136                 signaled_count = 0;
1137                 for (i = 0; i < count; ++i) {
1138                         fence = entries[i].fence;
1139                         if (!fence)
1140                                 continue;
1141
1142                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1143                             dma_fence_is_signaled(fence) ||
1144                             (!entries[i].fence_cb.func &&
1145                              dma_fence_add_callback(fence,
1146                                                     &entries[i].fence_cb,
1147                                                     syncobj_wait_fence_func))) {
1148                                 /* The fence has been signaled */
1149                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1150                                         signaled_count++;
1151                                 } else {
1152                                         if (idx)
1153                                                 *idx = i;
1154                                         goto done_waiting;
1155                                 }
1156                         }
1157                 }
1158
1159                 if (signaled_count == count)
1160                         goto done_waiting;
1161
1162                 if (timeout == 0) {
1163                         timeout = -ETIME;
1164                         goto done_waiting;
1165                 }
1166
1167                 if (signal_pending(current)) {
1168                         timeout = -ERESTARTSYS;
1169                         goto done_waiting;
1170                 }
1171
1172                 timeout = schedule_timeout(timeout);
1173         } while (1);
1174
1175 done_waiting:
1176         __set_current_state(TASK_RUNNING);
1177
1178 cleanup_entries:
1179         for (i = 0; i < count; ++i) {
1180                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1181                 if (entries[i].fence_cb.func)
1182                         dma_fence_remove_callback(entries[i].fence,
1183                                                   &entries[i].fence_cb);
1184                 dma_fence_put(entries[i].fence);
1185         }
1186         kfree(entries);
1187
1188 err_free_points:
1189         kfree(points);
1190
1191         return timeout;
1192 }
1193
1194 /**
1195  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1196  *
1197  * @timeout_nsec: timeout nsec component in ns, 0 for poll
1198  *
1199  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1200  */
1201 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1202 {
1203         ktime_t abs_timeout, now;
1204         u64 timeout_ns, timeout_jiffies64;
1205
1206         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
1207         if (timeout_nsec == 0)
1208                 return 0;
1209
1210         abs_timeout = ns_to_ktime(timeout_nsec);
1211         now = ktime_get();
1212
1213         if (!ktime_after(abs_timeout, now))
1214                 return 0;
1215
1216         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1217
1218         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1219         /*  clamp timeout to avoid infinite timeout */
1220         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1221                 return MAX_SCHEDULE_TIMEOUT - 1;
1222
1223         return timeout_jiffies64 + 1;
1224 }
1225 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1226
1227 static int drm_syncobj_array_wait(struct drm_device *dev,
1228                                   struct drm_file *file_private,
1229                                   struct drm_syncobj_wait *wait,
1230                                   struct drm_syncobj_timeline_wait *timeline_wait,
1231                                   struct drm_syncobj **syncobjs, bool timeline,
1232                                   ktime_t *deadline)
1233 {
1234         signed long timeout = 0;
1235         uint32_t first = ~0;
1236
1237         if (!timeline) {
1238                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1239                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1240                                                          NULL,
1241                                                          wait->count_handles,
1242                                                          wait->flags,
1243                                                          timeout, &first,
1244                                                          deadline);
1245                 if (timeout < 0)
1246                         return timeout;
1247                 wait->first_signaled = first;
1248         } else {
1249                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1250                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1251                                                          u64_to_user_ptr(timeline_wait->points),
1252                                                          timeline_wait->count_handles,
1253                                                          timeline_wait->flags,
1254                                                          timeout, &first,
1255                                                          deadline);
1256                 if (timeout < 0)
1257                         return timeout;
1258                 timeline_wait->first_signaled = first;
1259         }
1260         return 0;
1261 }
1262
1263 static int drm_syncobj_array_find(struct drm_file *file_private,
1264                                   void __user *user_handles,
1265                                   uint32_t count_handles,
1266                                   struct drm_syncobj ***syncobjs_out)
1267 {
1268         uint32_t i, *handles;
1269         struct drm_syncobj **syncobjs;
1270         int ret;
1271
1272         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1273         if (handles == NULL)
1274                 return -ENOMEM;
1275
1276         if (copy_from_user(handles, user_handles,
1277                            sizeof(uint32_t) * count_handles)) {
1278                 ret = -EFAULT;
1279                 goto err_free_handles;
1280         }
1281
1282         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1283         if (syncobjs == NULL) {
1284                 ret = -ENOMEM;
1285                 goto err_free_handles;
1286         }
1287
1288         for (i = 0; i < count_handles; i++) {
1289                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1290                 if (!syncobjs[i]) {
1291                         ret = -ENOENT;
1292                         goto err_put_syncobjs;
1293                 }
1294         }
1295
1296         kfree(handles);
1297         *syncobjs_out = syncobjs;
1298         return 0;
1299
1300 err_put_syncobjs:
1301         while (i-- > 0)
1302                 drm_syncobj_put(syncobjs[i]);
1303         kfree(syncobjs);
1304 err_free_handles:
1305         kfree(handles);
1306
1307         return ret;
1308 }
1309
1310 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1311                                    uint32_t count)
1312 {
1313         uint32_t i;
1314
1315         for (i = 0; i < count; i++)
1316                 drm_syncobj_put(syncobjs[i]);
1317         kfree(syncobjs);
1318 }
1319
1320 int
1321 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1322                        struct drm_file *file_private)
1323 {
1324         struct drm_syncobj_wait *args = data;
1325         struct drm_syncobj **syncobjs;
1326         unsigned int possible_flags;
1327         ktime_t t, *tp = NULL;
1328         int ret = 0;
1329
1330         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1331                 return -EOPNOTSUPP;
1332
1333         possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1334                          DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1335                          DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1336
1337         if (args->flags & ~possible_flags)
1338                 return -EINVAL;
1339
1340         if (args->count_handles == 0)
1341                 return 0;
1342
1343         ret = drm_syncobj_array_find(file_private,
1344                                      u64_to_user_ptr(args->handles),
1345                                      args->count_handles,
1346                                      &syncobjs);
1347         if (ret < 0)
1348                 return ret;
1349
1350         if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1351                 t = ns_to_ktime(args->deadline_nsec);
1352                 tp = &t;
1353         }
1354
1355         ret = drm_syncobj_array_wait(dev, file_private,
1356                                      args, NULL, syncobjs, false, tp);
1357
1358         drm_syncobj_array_free(syncobjs, args->count_handles);
1359
1360         return ret;
1361 }
1362
1363 int
1364 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1365                                 struct drm_file *file_private)
1366 {
1367         struct drm_syncobj_timeline_wait *args = data;
1368         struct drm_syncobj **syncobjs;
1369         unsigned int possible_flags;
1370         ktime_t t, *tp = NULL;
1371         int ret = 0;
1372
1373         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1374                 return -EOPNOTSUPP;
1375
1376         possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1377                          DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1378                          DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE |
1379                          DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1380
1381         if (args->flags & ~possible_flags)
1382                 return -EINVAL;
1383
1384         if (args->count_handles == 0)
1385                 return 0;
1386
1387         ret = drm_syncobj_array_find(file_private,
1388                                      u64_to_user_ptr(args->handles),
1389                                      args->count_handles,
1390                                      &syncobjs);
1391         if (ret < 0)
1392                 return ret;
1393
1394         if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1395                 t = ns_to_ktime(args->deadline_nsec);
1396                 tp = &t;
1397         }
1398
1399         ret = drm_syncobj_array_wait(dev, file_private,
1400                                      NULL, args, syncobjs, true, tp);
1401
1402         drm_syncobj_array_free(syncobjs, args->count_handles);
1403
1404         return ret;
1405 }
1406
1407 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1408                                              struct dma_fence_cb *cb)
1409 {
1410         struct syncobj_eventfd_entry *entry =
1411                 container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1412
1413         eventfd_signal(entry->ev_fd_ctx);
1414         syncobj_eventfd_entry_free(entry);
1415 }
1416
1417 static void
1418 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1419                            struct syncobj_eventfd_entry *entry)
1420 {
1421         int ret;
1422         struct dma_fence *fence;
1423
1424         /* This happens inside the syncobj lock */
1425         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1426         if (!fence)
1427                 return;
1428
1429         ret = dma_fence_chain_find_seqno(&fence, entry->point);
1430         if (ret != 0) {
1431                 /* The given seqno has not been submitted yet. */
1432                 dma_fence_put(fence);
1433                 return;
1434         } else if (!fence) {
1435                 /* If dma_fence_chain_find_seqno returns 0 but sets the fence
1436                  * to NULL, it implies that the given seqno is signaled and a
1437                  * later seqno has already been submitted. Assign a stub fence
1438                  * so that the eventfd still gets signaled below.
1439                  */
1440                 fence = dma_fence_get_stub();
1441         }
1442
1443         list_del_init(&entry->node);
1444         entry->fence = fence;
1445
1446         if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1447                 eventfd_signal(entry->ev_fd_ctx);
1448                 syncobj_eventfd_entry_free(entry);
1449         } else {
1450                 ret = dma_fence_add_callback(fence, &entry->fence_cb,
1451                                              syncobj_eventfd_entry_fence_func);
1452                 if (ret == -ENOENT) {
1453                         eventfd_signal(entry->ev_fd_ctx);
1454                         syncobj_eventfd_entry_free(entry);
1455                 }
1456         }
1457 }
1458
1459 int
1460 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1461                           struct drm_file *file_private)
1462 {
1463         struct drm_syncobj_eventfd *args = data;
1464         struct drm_syncobj *syncobj;
1465         struct eventfd_ctx *ev_fd_ctx;
1466         struct syncobj_eventfd_entry *entry;
1467         int ret;
1468
1469         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1470                 return -EOPNOTSUPP;
1471
1472         if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1473                 return -EINVAL;
1474
1475         if (args->pad)
1476                 return -EINVAL;
1477
1478         syncobj = drm_syncobj_find(file_private, args->handle);
1479         if (!syncobj)
1480                 return -ENOENT;
1481
1482         ev_fd_ctx = eventfd_ctx_fdget(args->fd);
1483         if (IS_ERR(ev_fd_ctx)) {
1484                 ret = PTR_ERR(ev_fd_ctx);
1485                 goto err_fdget;
1486         }
1487
1488         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1489         if (!entry) {
1490                 ret = -ENOMEM;
1491                 goto err_kzalloc;
1492         }
1493         entry->syncobj = syncobj;
1494         entry->ev_fd_ctx = ev_fd_ctx;
1495         entry->point = args->point;
1496         entry->flags = args->flags;
1497
1498         drm_syncobj_add_eventfd(syncobj, entry);
1499         drm_syncobj_put(syncobj);
1500
1501         return 0;
1502
1503 err_kzalloc:
1504         eventfd_ctx_put(ev_fd_ctx);
1505 err_fdget:
1506         drm_syncobj_put(syncobj);
1507         return ret;
1508 }
1509
1510 int
1511 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1512                         struct drm_file *file_private)
1513 {
1514         struct drm_syncobj_array *args = data;
1515         struct drm_syncobj **syncobjs;
1516         uint32_t i;
1517         int ret;
1518
1519         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1520                 return -EOPNOTSUPP;
1521
1522         if (args->pad != 0)
1523                 return -EINVAL;
1524
1525         if (args->count_handles == 0)
1526                 return -EINVAL;
1527
1528         ret = drm_syncobj_array_find(file_private,
1529                                      u64_to_user_ptr(args->handles),
1530                                      args->count_handles,
1531                                      &syncobjs);
1532         if (ret < 0)
1533                 return ret;
1534
1535         for (i = 0; i < args->count_handles; i++)
1536                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1537
1538         drm_syncobj_array_free(syncobjs, args->count_handles);
1539
1540         return 0;
1541 }
1542
1543 int
1544 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1545                          struct drm_file *file_private)
1546 {
1547         struct drm_syncobj_array *args = data;
1548         struct drm_syncobj **syncobjs;
1549         uint32_t i;
1550         int ret;
1551
1552         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1553                 return -EOPNOTSUPP;
1554
1555         if (args->pad != 0)
1556                 return -EINVAL;
1557
1558         if (args->count_handles == 0)
1559                 return -EINVAL;
1560
1561         ret = drm_syncobj_array_find(file_private,
1562                                      u64_to_user_ptr(args->handles),
1563                                      args->count_handles,
1564                                      &syncobjs);
1565         if (ret < 0)
1566                 return ret;
1567
1568         for (i = 0; i < args->count_handles; i++) {
1569                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1570                 if (ret < 0)
1571                         break;
1572         }
1573
1574         drm_syncobj_array_free(syncobjs, args->count_handles);
1575
1576         return ret;
1577 }
1578
1579 int
1580 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1581                                   struct drm_file *file_private)
1582 {
1583         struct drm_syncobj_timeline_array *args = data;
1584         struct drm_syncobj **syncobjs;
1585         struct dma_fence_chain **chains;
1586         uint64_t *points;
1587         uint32_t i, j;
1588         int ret;
1589
1590         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1591                 return -EOPNOTSUPP;
1592
1593         if (args->flags != 0)
1594                 return -EINVAL;
1595
1596         if (args->count_handles == 0)
1597                 return -EINVAL;
1598
1599         ret = drm_syncobj_array_find(file_private,
1600                                      u64_to_user_ptr(args->handles),
1601                                      args->count_handles,
1602                                      &syncobjs);
1603         if (ret < 0)
1604                 return ret;
1605
1606         points = kmalloc_array(args->count_handles, sizeof(*points),
1607                                GFP_KERNEL);
1608         if (!points) {
1609                 ret = -ENOMEM;
1610                 goto out;
1611         }
1612         if (!u64_to_user_ptr(args->points)) {
1613                 memset(points, 0, args->count_handles * sizeof(uint64_t));
1614         } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1615                                   sizeof(uint64_t) * args->count_handles)) {
1616                 ret = -EFAULT;
1617                 goto err_points;
1618         }
1619
1620         chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1621         if (!chains) {
1622                 ret = -ENOMEM;
1623                 goto err_points;
1624         }
1625         for (i = 0; i < args->count_handles; i++) {
1626                 chains[i] = dma_fence_chain_alloc();
1627                 if (!chains[i]) {
1628                         for (j = 0; j < i; j++)
1629                                 dma_fence_chain_free(chains[j]);
1630                         ret = -ENOMEM;
1631                         goto err_chains;
1632                 }
1633         }
1634
1635         for (i = 0; i < args->count_handles; i++) {
1636                 struct dma_fence *fence = dma_fence_get_stub();
1637
1638                 drm_syncobj_add_point(syncobjs[i], chains[i],
1639                                       fence, points[i]);
1640                 dma_fence_put(fence);
1641         }
1642 err_chains:
1643         kfree(chains);
1644 err_points:
1645         kfree(points);
1646 out:
1647         drm_syncobj_array_free(syncobjs, args->count_handles);
1648
1649         return ret;
1650 }
1651
1652 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1653                             struct drm_file *file_private)
1654 {
1655         struct drm_syncobj_timeline_array *args = data;
1656         struct drm_syncobj **syncobjs;
1657         uint64_t __user *points = u64_to_user_ptr(args->points);
1658         uint32_t i;
1659         int ret;
1660
1661         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1662                 return -EOPNOTSUPP;
1663
1664         if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1665                 return -EINVAL;
1666
1667         if (args->count_handles == 0)
1668                 return -EINVAL;
1669
1670         ret = drm_syncobj_array_find(file_private,
1671                                      u64_to_user_ptr(args->handles),
1672                                      args->count_handles,
1673                                      &syncobjs);
1674         if (ret < 0)
1675                 return ret;
1676
1677         for (i = 0; i < args->count_handles; i++) {
1678                 struct dma_fence_chain *chain;
1679                 struct dma_fence *fence;
1680                 uint64_t point;
1681
1682                 fence = drm_syncobj_fence_get(syncobjs[i]);
1683                 chain = to_dma_fence_chain(fence);
1684                 if (chain) {
1685                         struct dma_fence *iter, *last_signaled =
1686                                 dma_fence_get(fence);
1687
1688                         if (args->flags &
1689                             DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1690                                 point = fence->seqno;
1691                         } else {
1692                                 dma_fence_chain_for_each(iter, fence) {
1693                                         if (iter->context != fence->context) {
1694                                                 dma_fence_put(iter);
1695                                                 /* It is most likely that timeline has
1696                                                 * unorder points. */
1697                                                 break;
1698                                         }
1699                                         dma_fence_put(last_signaled);
1700                                         last_signaled = dma_fence_get(iter);
1701                                 }
1702                                 point = dma_fence_is_signaled(last_signaled) ?
1703                                         last_signaled->seqno :
1704                                         to_dma_fence_chain(last_signaled)->prev_seqno;
1705                         }
1706                         dma_fence_put(last_signaled);
1707                 } else {
1708                         point = 0;
1709                 }
1710                 dma_fence_put(fence);
1711                 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1712                 ret = ret ? -EFAULT : 0;
1713                 if (ret)
1714                         break;
1715         }
1716         drm_syncobj_array_free(syncobjs, args->count_handles);
1717
1718         return ret;
1719 }