drivers/gpu/drm/xe/xe_sync.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include "xe_sync.h"
   7
   8 #include <linux/dma-fence-array.h>
   9 #include <linux/kthread.h>
  10 #include <linux/sched/mm.h>
  11 #include <linux/uaccess.h>
  12
  13 #include <drm/drm_print.h>
  14 #include <drm/drm_syncobj.h>
  15 #include <drm/xe_drm.h>
  16
  17 #include "xe_device_types.h"
  18 #include "xe_exec_queue.h"
  19 #include "xe_macros.h"
  20 #include "xe_sched_job_types.h"
  21
  22 struct xe_user_fence {
  23         struct xe_device *xe;
  24         struct kref refcount;
  25         struct dma_fence_cb cb;
  26         struct work_struct worker;
  27         struct mm_struct *mm;
  28         u64 __user *addr;
  29         u64 value;
  30         int signalled;
  31 };
  32
  33 static void user_fence_destroy(struct kref *kref)
  34 {
  35         struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
  36                                                  refcount);
  37
  38         mmdrop(ufence->mm);
  39         kfree(ufence);
  40 }
  41
  42 static void user_fence_get(struct xe_user_fence *ufence)
  43 {
  44         kref_get(&ufence->refcount);
  45 }
  46
  47 static void user_fence_put(struct xe_user_fence *ufence)
  48 {
  49         kref_put(&ufence->refcount, user_fence_destroy);
  50 }
  51
  52 static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
  53                                                u64 value)
  54 {
  55         struct xe_user_fence *ufence;
  56
  57         ufence = kmalloc(sizeof(*ufence), GFP_KERNEL);
  58         if (!ufence)
  59                 return NULL;
  60
  61         ufence->xe = xe;
  62         kref_init(&ufence->refcount);
  63         ufence->addr = u64_to_user_ptr(addr);
  64         ufence->value = value;
  65         ufence->mm = current->mm;
  66         mmgrab(ufence->mm);
  67
  68         return ufence;
  69 }
  70
  71 static void user_fence_worker(struct work_struct *w)
  72 {
  73         struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);
  74
  75         if (mmget_not_zero(ufence->mm)) {
  76                 kthread_use_mm(ufence->mm);
  77                 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
  78                         XE_WARN_ON("Copy to user failed");
  79                 kthread_unuse_mm(ufence->mm);
  80                 mmput(ufence->mm);
  81         }
  82
  83         wake_up_all(&ufence->xe->ufence_wq);
  84         WRITE_ONCE(ufence->signalled, 1);
  85         user_fence_put(ufence);
  86 }
  87
  88 static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
  89 {
  90         INIT_WORK(&ufence->worker, user_fence_worker);
  91         queue_work(ufence->xe->ordered_wq, &ufence->worker);
  92         dma_fence_put(fence);
  93 }
  94
  95 static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
  96 {
  97         struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);
  98
  99         kick_ufence(ufence, fence);
 100 }
 101
 102 int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
 103                         struct xe_sync_entry *sync,
 104                         struct drm_xe_sync __user *sync_user,
 105                         unsigned int flags)
 106 {
 107         struct drm_xe_sync sync_in;
 108         int err;
 109         bool exec = flags & SYNC_PARSE_FLAG_EXEC;
 110         bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
 111         bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
 112         bool signal;
 113
 114         if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
 115                 return -EFAULT;
 116
 117         if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
 118             XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
 119                 return -EINVAL;
 120
 121         signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
 122         switch (sync_in.type) {
 123         case DRM_XE_SYNC_TYPE_SYNCOBJ:
 124                 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
 125                         return -EOPNOTSUPP;
 126
 127                 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
 128                         return -EINVAL;
 129
 130                 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
 131                 if (XE_IOCTL_DBG(xe, !sync->syncobj))
 132                         return -ENOENT;
 133
 134                 if (!signal) {
 135                         sync->fence = drm_syncobj_fence_get(sync->syncobj);
 136                         if (XE_IOCTL_DBG(xe, !sync->fence))
 137                                 return -EINVAL;
 138                 }
 139                 break;
 140
 141         case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
 142                 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
 143                         return -EOPNOTSUPP;
 144
 145                 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
 146                         return -EINVAL;
 147
 148                 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
 149                         return -EINVAL;
 150
 151                 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
 152                 if (XE_IOCTL_DBG(xe, !sync->syncobj))
 153                         return -ENOENT;
 154
 155                 if (signal) {
 156                         sync->chain_fence = dma_fence_chain_alloc();
 157                         if (!sync->chain_fence)
 158                                 return -ENOMEM;
 159                 } else {
 160                         sync->fence = drm_syncobj_fence_get(sync->syncobj);
 161                         if (XE_IOCTL_DBG(xe, !sync->fence))
 162                                 return -EINVAL;
 163
 164                         err = dma_fence_chain_find_seqno(&sync->fence,
 165                                                          sync_in.timeline_value);
 166                         if (err)
 167                                 return err;
 168                 }
 169                 break;
 170
 171         case DRM_XE_SYNC_TYPE_USER_FENCE:
 172                 if (XE_IOCTL_DBG(xe, disallow_user_fence))
 173                         return -EOPNOTSUPP;
 174
 175                 if (XE_IOCTL_DBG(xe, !signal))
 176                         return -EOPNOTSUPP;
 177
 178                 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
 179                         return -EINVAL;
 180
 181                 if (exec) {
 182                         sync->addr = sync_in.addr;
 183                 } else {
 184                         sync->ufence = user_fence_create(xe, sync_in.addr,
 185                                                          sync_in.timeline_value);
 186                         if (XE_IOCTL_DBG(xe, !sync->ufence))
 187                                 return -ENOMEM;
 188                 }
 189
 190                 break;
 191
 192         default:
 193                 return -EINVAL;
 194         }
 195
 196         sync->type = sync_in.type;
 197         sync->flags = sync_in.flags;
 198         sync->timeline_value = sync_in.timeline_value;
 199
 200         return 0;
 201 }
 202
 203 int xe_sync_entry_wait(struct xe_sync_entry *sync)
 204 {
 205         if (sync->fence)
 206                 dma_fence_wait(sync->fence, true);
 207
 208         return 0;
 209 }
 210
 211 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
 212 {
 213         int err;
 214
 215         if (sync->fence) {
 216                 err = drm_sched_job_add_dependency(&job->drm,
 217                                                    dma_fence_get(sync->fence));
 218                 if (err) {
 219                         dma_fence_put(sync->fence);
 220                         return err;
 221                 }
 222         }
 223
 224         return 0;
 225 }
 226
 227 void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
 228 {
 229         if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
 230                 return;
 231
 232         if (sync->chain_fence) {
 233                 drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
 234                                       fence, sync->timeline_value);
 235                 /*
 236                  * The chain's ownership is transferred to the
 237                  * timeline.
 238                  */
 239                 sync->chain_fence = NULL;
 240         } else if (sync->syncobj) {
 241                 drm_syncobj_replace_fence(sync->syncobj, fence);
 242         } else if (sync->ufence) {
 243                 int err;
 244
 245                 dma_fence_get(fence);
 246                 user_fence_get(sync->ufence);
 247                 err = dma_fence_add_callback(fence, &sync->ufence->cb,
 248                                              user_fence_cb);
 249                 if (err == -ENOENT) {
 250                         kick_ufence(sync->ufence, fence);
 251                 } else if (err) {
 252                         XE_WARN_ON("failed to add user fence");
 253                         user_fence_put(sync->ufence);
 254                         dma_fence_put(fence);
 255                 }
 256         }
 257 }
 258
 259 void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
 260 {
 261         if (sync->syncobj)
 262                 drm_syncobj_put(sync->syncobj);
 263         if (sync->fence)
 264                 dma_fence_put(sync->fence);
 265         if (sync->chain_fence)
 266                 dma_fence_put(&sync->chain_fence->base);
 267         if (sync->ufence)
 268                 user_fence_put(sync->ufence);
 269 }
 270
 271 /**
 272  * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
 273  * @sync: input syncs
 274  * @num_sync: number of syncs
 275  * @q: exec queue
 276  * @vm: VM
 277  *
 278  * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
 279  * and return a composite fence of all in-fences + last fence. If no in-fences
 280  * return last fence on  input exec queue. Caller must drop reference to
 281  * returned fence.
 282  *
 283  * Return: fence on success, ERR_PTR(-ENOMEM) on failure
 284  */
 285 struct dma_fence *
 286 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
 287                      struct xe_exec_queue *q, struct xe_vm *vm)
 288 {
 289         struct dma_fence **fences = NULL;
 290         struct dma_fence_array *cf = NULL;
 291         struct dma_fence *fence;
 292         int i, num_in_fence = 0, current_fence = 0;
 293
 294         lockdep_assert_held(&vm->lock);
 295
 296         /* Count in-fences */
 297         for (i = 0; i < num_sync; ++i) {
 298                 if (sync[i].fence) {
 299                         ++num_in_fence;
 300                         fence = sync[i].fence;
 301                 }
 302         }
 303
 304         /* Easy case... */
 305         if (!num_in_fence) {
 306                 fence = xe_exec_queue_last_fence_get(q, vm);
 307                 return fence;
 308         }
 309
 310         /* Create composite fence */
 311         fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);
 312         if (!fences)
 313                 return ERR_PTR(-ENOMEM);
 314         for (i = 0; i < num_sync; ++i) {
 315                 if (sync[i].fence) {
 316                         dma_fence_get(sync[i].fence);
 317                         fences[current_fence++] = sync[i].fence;
 318                 }
 319         }
 320         fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm);
 321         cf = dma_fence_array_create(num_in_fence, fences,
 322                                     vm->composite_fence_ctx,
 323                                     vm->composite_fence_seqno++,
 324                                     false);
 325         if (!cf) {
 326                 --vm->composite_fence_seqno;
 327                 goto err_out;
 328         }
 329
 330         return &cf->base;
 331
 332 err_out:
 333         while (current_fence)
 334                 dma_fence_put(fences[--current_fence]);
 335         kfree(fences);
 336         kfree(cf);
 337
 338         return ERR_PTR(-ENOMEM);
 339 }
 340
 341 /**
 342  * xe_sync_ufence_get() - Get user fence from sync
 343  * @sync: input sync
 344  *
 345  * Get a user fence reference from sync.
 346  *
 347  * Return: xe_user_fence pointer with reference
 348  */
 349 struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
 350 {
 351         user_fence_get(sync->ufence);
 352
 353         return sync->ufence;
 354 }
 355
 356 /**
 357  * xe_sync_ufence_put() - Put user fence reference
 358  * @ufence: user fence reference
 359  *
 360  */
 361 void xe_sync_ufence_put(struct xe_user_fence *ufence)
 362 {
 363         user_fence_put(ufence);
 364 }
 365
 366 /**
 367  * xe_sync_ufence_get_status() - Get user fence status
 368  * @ufence: user fence
 369  *
 370  * Return: 1 if signalled, 0 not signalled, <0 on error
 371  */
 372 int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
 373 {
 374         return READ_ONCE(ufence->signalled);
 375 }