drivers/gpu/drm/xe/xe_exec_queue.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include "xe_exec_queue.h"
   7
   8 #include <linux/nospec.h>
   9
  10 #include <drm/drm_device.h>
  11 #include <drm/drm_drv.h>
  12 #include <drm/drm_file.h>
  13 #include <uapi/drm/xe_drm.h>
  14
  15 #include "xe_device.h"
  16 #include "xe_gt.h"
  17 #include "xe_hw_engine_class_sysfs.h"
  18 #include "xe_hw_engine_group.h"
  19 #include "xe_hw_fence.h"
  20 #include "xe_irq.h"
  21 #include "xe_lrc.h"
  22 #include "xe_macros.h"
  23 #include "xe_migrate.h"
  24 #include "xe_pm.h"
  25 #include "xe_ring_ops_types.h"
  26 #include "xe_trace.h"
  27 #include "xe_vm.h"
  28
  29 enum xe_exec_queue_sched_prop {
  30         XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
  31         XE_EXEC_QUEUE_TIMESLICE = 1,
  32         XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,
  33         XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,
  34 };
  35
  36 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
  37                                       u64 extensions, int ext_number);
  38
  39 static void __xe_exec_queue_free(struct xe_exec_queue *q)
  40 {
  41         if (q->vm)
  42                 xe_vm_put(q->vm);
  43
  44         if (q->xef)
  45                 xe_file_put(q->xef);
  46
  47         kfree(q);
  48 }
  49
  50 static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
  51                                                    struct xe_vm *vm,
  52                                                    u32 logical_mask,
  53                                                    u16 width, struct xe_hw_engine *hwe,
  54                                                    u32 flags, u64 extensions)
  55 {
  56         struct xe_exec_queue *q;
  57         struct xe_gt *gt = hwe->gt;
  58         int err;
  59
  60         /* only kernel queues can be permanent */
  61         XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));
  62
  63         q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL);
  64         if (!q)
  65                 return ERR_PTR(-ENOMEM);
  66
  67         kref_init(&q->refcount);
  68         q->flags = flags;
  69         q->hwe = hwe;
  70         q->gt = gt;
  71         q->class = hwe->class;
  72         q->width = width;
  73         q->msix_vec = XE_IRQ_DEFAULT_MSIX;
  74         q->logical_mask = logical_mask;
  75         q->fence_irq = &gt->fence_irq[hwe->class];
  76         q->ring_ops = gt->ring_ops[hwe->class];
  77         q->ops = gt->exec_queue_ops;
  78         INIT_LIST_HEAD(&q->lr.link);
  79         INIT_LIST_HEAD(&q->multi_gt_link);
  80         INIT_LIST_HEAD(&q->hw_engine_group_link);
  81
  82         q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
  83         q->sched_props.preempt_timeout_us =
  84                                 hwe->eclass->sched_props.preempt_timeout_us;
  85         q->sched_props.job_timeout_ms =
  86                                 hwe->eclass->sched_props.job_timeout_ms;
  87         if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&
  88             q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)
  89                 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;
  90         else
  91                 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;
  92
  93         if (vm)
  94                 q->vm = xe_vm_get(vm);
  95
  96         if (extensions) {
  97                 /*
  98                  * may set q->usm, must come before xe_lrc_create(),
  99                  * may overwrite q->sched_props, must come before q->ops->init()
 100                  */
 101                 err = exec_queue_user_extensions(xe, q, extensions, 0);
 102                 if (err) {
 103                         __xe_exec_queue_free(q);
 104                         return ERR_PTR(err);
 105                 }
 106         }
 107
 108         return q;
 109 }
 110
 111 static int __xe_exec_queue_init(struct xe_exec_queue *q)
 112 {
 113         struct xe_vm *vm = q->vm;
 114         int i, err;
 115
 116         if (vm) {
 117                 err = xe_vm_lock(vm, true);
 118                 if (err)
 119                         return err;
 120         }
 121
 122         for (i = 0; i < q->width; ++i) {
 123                 q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec);
 124                 if (IS_ERR(q->lrc[i])) {
 125                         err = PTR_ERR(q->lrc[i]);
 126                         goto err_unlock;
 127                 }
 128         }
 129
 130         if (vm)
 131                 xe_vm_unlock(vm);
 132
 133         err = q->ops->init(q);
 134         if (err)
 135                 goto err_lrc;
 136
 137         return 0;
 138
 139 err_unlock:
 140         if (vm)
 141                 xe_vm_unlock(vm);
 142 err_lrc:
 143         for (i = i - 1; i >= 0; --i)
 144                 xe_lrc_put(q->lrc[i]);
 145         return err;
 146 }
 147
 148 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
 149                                            u32 logical_mask, u16 width,
 150                                            struct xe_hw_engine *hwe, u32 flags,
 151                                            u64 extensions)
 152 {
 153         struct xe_exec_queue *q;
 154         int err;
 155
 156         q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,
 157                                   extensions);
 158         if (IS_ERR(q))
 159                 return q;
 160
 161         err = __xe_exec_queue_init(q);
 162         if (err)
 163                 goto err_post_alloc;
 164
 165         return q;
 166
 167 err_post_alloc:
 168         __xe_exec_queue_free(q);
 169         return ERR_PTR(err);
 170 }
 171
 172 struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,
 173                                                  struct xe_vm *vm,
 174                                                  enum xe_engine_class class,
 175                                                  u32 flags, u64 extensions)
 176 {
 177         struct xe_hw_engine *hwe, *hwe0 = NULL;
 178         enum xe_hw_engine_id id;
 179         u32 logical_mask = 0;
 180
 181         for_each_hw_engine(hwe, gt, id) {
 182                 if (xe_hw_engine_is_reserved(hwe))
 183                         continue;
 184
 185                 if (hwe->class == class) {
 186                         logical_mask |= BIT(hwe->logical_instance);
 187                         if (!hwe0)
 188                                 hwe0 = hwe;
 189                 }
 190         }
 191
 192         if (!logical_mask)
 193                 return ERR_PTR(-ENODEV);
 194
 195         return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions);
 196 }
 197
 198 /**
 199  * xe_exec_queue_create_bind() - Create bind exec queue.
 200  * @xe: Xe device.
 201  * @tile: tile which bind exec queue belongs to.
 202  * @flags: exec queue creation flags
 203  * @extensions: exec queue creation extensions
 204  *
 205  * Normalize bind exec queue creation. Bind exec queue is tied to migration VM
 206  * for access to physical memory required for page table programming. On a
 207  * faulting devices the reserved copy engine instance must be used to avoid
 208  * deadlocking (user binds cannot get stuck behind faults as kernel binds which
 209  * resolve faults depend on user binds). On non-faulting devices any copy engine
 210  * can be used.
 211  *
 212  * Returns exec queue on success, ERR_PTR on failure
 213  */
 214 struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
 215                                                 struct xe_tile *tile,
 216                                                 u32 flags, u64 extensions)
 217 {
 218         struct xe_gt *gt = tile->primary_gt;
 219         struct xe_exec_queue *q;
 220         struct xe_vm *migrate_vm;
 221
 222         migrate_vm = xe_migrate_get_vm(tile->migrate);
 223         if (xe->info.has_usm) {
 224                 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt,
 225                                                            XE_ENGINE_CLASS_COPY,
 226                                                            gt->usm.reserved_bcs_instance,
 227                                                            false);
 228
 229                 if (!hwe) {
 230                         xe_vm_put(migrate_vm);
 231                         return ERR_PTR(-EINVAL);
 232                 }
 233
 234                 q = xe_exec_queue_create(xe, migrate_vm,
 235                                          BIT(hwe->logical_instance), 1, hwe,
 236                                          flags, extensions);
 237         } else {
 238                 q = xe_exec_queue_create_class(xe, gt, migrate_vm,
 239                                                XE_ENGINE_CLASS_COPY, flags,
 240                                                extensions);
 241         }
 242         xe_vm_put(migrate_vm);
 243
 244         return q;
 245 }
 246 ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
 247
 248 void xe_exec_queue_destroy(struct kref *ref)
 249 {
 250         struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
 251         struct xe_exec_queue *eq, *next;
 252
 253         xe_exec_queue_last_fence_put_unlocked(q);
 254         if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {
 255                 list_for_each_entry_safe(eq, next, &q->multi_gt_list,
 256                                          multi_gt_link)
 257                         xe_exec_queue_put(eq);
 258         }
 259
 260         q->ops->fini(q);
 261 }
 262
 263 void xe_exec_queue_fini(struct xe_exec_queue *q)
 264 {
 265         int i;
 266
 267         /*
 268          * Before releasing our ref to lrc and xef, accumulate our run ticks
 269          * and wakeup any waiters.
 270          */
 271         xe_exec_queue_update_run_ticks(q);
 272         if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal))
 273                 wake_up_var(&q->xef->exec_queue.pending_removal);
 274
 275         for (i = 0; i < q->width; ++i)
 276                 xe_lrc_put(q->lrc[i]);
 277
 278         __xe_exec_queue_free(q);
 279 }
 280
 281 void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
 282 {
 283         switch (q->class) {
 284         case XE_ENGINE_CLASS_RENDER:
 285                 snprintf(q->name, sizeof(q->name), "rcs%d", instance);
 286                 break;
 287         case XE_ENGINE_CLASS_VIDEO_DECODE:
 288                 snprintf(q->name, sizeof(q->name), "vcs%d", instance);
 289                 break;
 290         case XE_ENGINE_CLASS_VIDEO_ENHANCE:
 291                 snprintf(q->name, sizeof(q->name), "vecs%d", instance);
 292                 break;
 293         case XE_ENGINE_CLASS_COPY:
 294                 snprintf(q->name, sizeof(q->name), "bcs%d", instance);
 295                 break;
 296         case XE_ENGINE_CLASS_COMPUTE:
 297                 snprintf(q->name, sizeof(q->name), "ccs%d", instance);
 298                 break;
 299         case XE_ENGINE_CLASS_OTHER:
 300                 snprintf(q->name, sizeof(q->name), "gsccs%d", instance);
 301                 break;
 302         default:
 303                 XE_WARN_ON(q->class);
 304         }
 305 }
 306
 307 struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id)
 308 {
 309         struct xe_exec_queue *q;
 310
 311         mutex_lock(&xef->exec_queue.lock);
 312         q = xa_load(&xef->exec_queue.xa, id);
 313         if (q)
 314                 xe_exec_queue_get(q);
 315         mutex_unlock(&xef->exec_queue.lock);
 316
 317         return q;
 318 }
 319
 320 enum xe_exec_queue_priority
 321 xe_exec_queue_device_get_max_priority(struct xe_device *xe)
 322 {
 323         return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :
 324                                        XE_EXEC_QUEUE_PRIORITY_NORMAL;
 325 }
 326
 327 static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,
 328                                    u64 value)
 329 {
 330         if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))
 331                 return -EINVAL;
 332
 333         if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))
 334                 return -EPERM;
 335
 336         q->sched_props.priority = value;
 337         return 0;
 338 }
 339
 340 static bool xe_exec_queue_enforce_schedule_limit(void)
 341 {
 342 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
 343         return true;
 344 #else
 345         return !capable(CAP_SYS_NICE);
 346 #endif
 347 }
 348
 349 static void
 350 xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,
 351                               enum xe_exec_queue_sched_prop prop,
 352                               u32 *min, u32 *max)
 353 {
 354         switch (prop) {
 355         case XE_EXEC_QUEUE_JOB_TIMEOUT:
 356                 *min = eclass->sched_props.job_timeout_min;
 357                 *max = eclass->sched_props.job_timeout_max;
 358                 break;
 359         case XE_EXEC_QUEUE_TIMESLICE:
 360                 *min = eclass->sched_props.timeslice_min;
 361                 *max = eclass->sched_props.timeslice_max;
 362                 break;
 363         case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
 364                 *min = eclass->sched_props.preempt_timeout_min;
 365                 *max = eclass->sched_props.preempt_timeout_max;
 366                 break;
 367         default:
 368                 break;
 369         }
 370 #if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
 371         if (capable(CAP_SYS_NICE)) {
 372                 switch (prop) {
 373                 case XE_EXEC_QUEUE_JOB_TIMEOUT:
 374                         *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;
 375                         *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;
 376                         break;
 377                 case XE_EXEC_QUEUE_TIMESLICE:
 378                         *min = XE_HW_ENGINE_TIMESLICE_MIN;
 379                         *max = XE_HW_ENGINE_TIMESLICE_MAX;
 380                         break;
 381                 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
 382                         *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;
 383                         *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;
 384                         break;
 385                 default:
 386                         break;
 387                 }
 388         }
 389 #endif
 390 }
 391
 392 static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,
 393                                     u64 value)
 394 {
 395         u32 min = 0, max = 0;
 396
 397         xe_exec_queue_get_prop_minmax(q->hwe->eclass,
 398                                       XE_EXEC_QUEUE_TIMESLICE, &min, &max);
 399
 400         if (xe_exec_queue_enforce_schedule_limit() &&
 401             !xe_hw_engine_timeout_in_range(value, min, max))
 402                 return -EINVAL;
 403
 404         q->sched_props.timeslice_us = value;
 405         return 0;
 406 }
 407
 408 typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
 409                                              struct xe_exec_queue *q,
 410                                              u64 value);
 411
 412 static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
 413         [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
 414         [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
 415 };
 416
 417 static int exec_queue_user_ext_set_property(struct xe_device *xe,
 418                                             struct xe_exec_queue *q,
 419                                             u64 extension)
 420 {
 421         u64 __user *address = u64_to_user_ptr(extension);
 422         struct drm_xe_ext_set_property ext;
 423         int err;
 424         u32 idx;
 425
 426         err = __copy_from_user(&ext, address, sizeof(ext));
 427         if (XE_IOCTL_DBG(xe, err))
 428                 return -EFAULT;
 429
 430         if (XE_IOCTL_DBG(xe, ext.property >=
 431                          ARRAY_SIZE(exec_queue_set_property_funcs)) ||
 432             XE_IOCTL_DBG(xe, ext.pad) ||
 433             XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
 434                          ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE))
 435                 return -EINVAL;
 436
 437         idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
 438         if (!exec_queue_set_property_funcs[idx])
 439                 return -EINVAL;
 440
 441         return exec_queue_set_property_funcs[idx](xe, q, ext.value);
 442 }
 443
 444 typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,
 445                                                struct xe_exec_queue *q,
 446                                                u64 extension);
 447
 448 static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
 449         [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
 450 };
 451
 452 #define MAX_USER_EXTENSIONS     16
 453 static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
 454                                       u64 extensions, int ext_number)
 455 {
 456         u64 __user *address = u64_to_user_ptr(extensions);
 457         struct drm_xe_user_extension ext;
 458         int err;
 459         u32 idx;
 460
 461         if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
 462                 return -E2BIG;
 463
 464         err = __copy_from_user(&ext, address, sizeof(ext));
 465         if (XE_IOCTL_DBG(xe, err))
 466                 return -EFAULT;
 467
 468         if (XE_IOCTL_DBG(xe, ext.pad) ||
 469             XE_IOCTL_DBG(xe, ext.name >=
 470                          ARRAY_SIZE(exec_queue_user_extension_funcs)))
 471                 return -EINVAL;
 472
 473         idx = array_index_nospec(ext.name,
 474                                  ARRAY_SIZE(exec_queue_user_extension_funcs));
 475         err = exec_queue_user_extension_funcs[idx](xe, q, extensions);
 476         if (XE_IOCTL_DBG(xe, err))
 477                 return err;
 478
 479         if (ext.next_extension)
 480                 return exec_queue_user_extensions(xe, q, ext.next_extension,
 481                                                   ++ext_number);
 482
 483         return 0;
 484 }
 485
 486 static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt,
 487                                       struct drm_xe_engine_class_instance *eci,
 488                                       u16 width, u16 num_placements)
 489 {
 490         int len = width * num_placements;
 491         int i, j, n;
 492         u16 class;
 493         u16 gt_id;
 494         u32 return_mask = 0, prev_mask;
 495
 496         if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) &&
 497                          len > 1))
 498                 return 0;
 499
 500         for (i = 0; i < width; ++i) {
 501                 u32 current_mask = 0;
 502
 503                 for (j = 0; j < num_placements; ++j) {
 504                         struct xe_hw_engine *hwe;
 505
 506                         n = j * width + i;
 507
 508                         hwe = xe_hw_engine_lookup(xe, eci[n]);
 509                         if (XE_IOCTL_DBG(xe, !hwe))
 510                                 return 0;
 511
 512                         if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe)))
 513                                 return 0;
 514
 515                         if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) ||
 516                             XE_IOCTL_DBG(xe, n && eci[n].engine_class != class))
 517                                 return 0;
 518
 519                         class = eci[n].engine_class;
 520                         gt_id = eci[n].gt_id;
 521
 522                         if (width == 1 || !i)
 523                                 return_mask |= BIT(eci[n].engine_instance);
 524                         current_mask |= BIT(eci[n].engine_instance);
 525                 }
 526
 527                 /* Parallel submissions must be logically contiguous */
 528                 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1))
 529                         return 0;
 530
 531                 prev_mask = current_mask;
 532         }
 533
 534         return return_mask;
 535 }
 536
 537 int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 538                                struct drm_file *file)
 539 {
 540         struct xe_device *xe = to_xe_device(dev);
 541         struct xe_file *xef = to_xe_file(file);
 542         struct drm_xe_exec_queue_create *args = data;
 543         struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE];
 544         struct drm_xe_engine_class_instance __user *user_eci =
 545                 u64_to_user_ptr(args->instances);
 546         struct xe_hw_engine *hwe;
 547         struct xe_vm *vm;
 548         struct xe_gt *gt;
 549         struct xe_tile *tile;
 550         struct xe_exec_queue *q = NULL;
 551         u32 logical_mask;
 552         u32 id;
 553         u32 len;
 554         int err;
 555
 556         if (XE_IOCTL_DBG(xe, args->flags) ||
 557             XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
 558                 return -EINVAL;
 559
 560         len = args->width * args->num_placements;
 561         if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE))
 562                 return -EINVAL;
 563
 564         err = __copy_from_user(eci, user_eci,
 565                                sizeof(struct drm_xe_engine_class_instance) *
 566                                len);
 567         if (XE_IOCTL_DBG(xe, err))
 568                 return -EFAULT;
 569
 570         if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count))
 571                 return -EINVAL;
 572
 573         if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) {
 574                 if (XE_IOCTL_DBG(xe, args->width != 1) ||
 575                     XE_IOCTL_DBG(xe, args->num_placements != 1) ||
 576                     XE_IOCTL_DBG(xe, eci[0].engine_instance != 0))
 577                         return -EINVAL;
 578
 579                 for_each_tile(tile, xe, id) {
 580                         struct xe_exec_queue *new;
 581                         u32 flags = EXEC_QUEUE_FLAG_VM;
 582
 583                         if (id)
 584                                 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD;
 585
 586                         new = xe_exec_queue_create_bind(xe, tile, flags,
 587                                                         args->extensions);
 588                         if (IS_ERR(new)) {
 589                                 err = PTR_ERR(new);
 590                                 if (q)
 591                                         goto put_exec_queue;
 592                                 return err;
 593                         }
 594                         if (id == 0)
 595                                 q = new;
 596                         else
 597                                 list_add_tail(&new->multi_gt_list,
 598                                               &q->multi_gt_link);
 599                 }
 600         } else {
 601                 gt = xe_device_get_gt(xe, eci[0].gt_id);
 602                 logical_mask = calc_validate_logical_mask(xe, gt, eci,
 603                                                           args->width,
 604                                                           args->num_placements);
 605                 if (XE_IOCTL_DBG(xe, !logical_mask))
 606                         return -EINVAL;
 607
 608                 hwe = xe_hw_engine_lookup(xe, eci[0]);
 609                 if (XE_IOCTL_DBG(xe, !hwe))
 610                         return -EINVAL;
 611
 612                 vm = xe_vm_lookup(xef, args->vm_id);
 613                 if (XE_IOCTL_DBG(xe, !vm))
 614                         return -ENOENT;
 615
 616                 err = down_read_interruptible(&vm->lock);
 617                 if (err) {
 618                         xe_vm_put(vm);
 619                         return err;
 620                 }
 621
 622                 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
 623                         up_read(&vm->lock);
 624                         xe_vm_put(vm);
 625                         return -ENOENT;
 626                 }
 627
 628                 q = xe_exec_queue_create(xe, vm, logical_mask,
 629                                          args->width, hwe, 0,
 630                                          args->extensions);
 631                 up_read(&vm->lock);
 632                 xe_vm_put(vm);
 633                 if (IS_ERR(q))
 634                         return PTR_ERR(q);
 635
 636                 if (xe_vm_in_preempt_fence_mode(vm)) {
 637                         q->lr.context = dma_fence_context_alloc(1);
 638
 639                         err = xe_vm_add_compute_exec_queue(vm, q);
 640                         if (XE_IOCTL_DBG(xe, err))
 641                                 goto put_exec_queue;
 642                 }
 643
 644                 if (q->vm && q->hwe->hw_engine_group) {
 645                         err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q);
 646                         if (err)
 647                                 goto put_exec_queue;
 648                 }
 649         }
 650
 651         q->xef = xe_file_get(xef);
 652
 653         /* user id alloc must always be last in ioctl to prevent UAF */
 654         err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL);
 655         if (err)
 656                 goto kill_exec_queue;
 657
 658         args->exec_queue_id = id;
 659
 660         return 0;
 661
 662 kill_exec_queue:
 663         xe_exec_queue_kill(q);
 664 put_exec_queue:
 665         xe_exec_queue_put(q);
 666         return err;
 667 }
 668
 669 int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
 670                                      struct drm_file *file)
 671 {
 672         struct xe_device *xe = to_xe_device(dev);
 673         struct xe_file *xef = to_xe_file(file);
 674         struct drm_xe_exec_queue_get_property *args = data;
 675         struct xe_exec_queue *q;
 676         int ret;
 677
 678         if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
 679                 return -EINVAL;
 680
 681         q = xe_exec_queue_lookup(xef, args->exec_queue_id);
 682         if (XE_IOCTL_DBG(xe, !q))
 683                 return -ENOENT;
 684
 685         switch (args->property) {
 686         case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN:
 687                 args->value = q->ops->reset_status(q);
 688                 ret = 0;
 689                 break;
 690         default:
 691                 ret = -EINVAL;
 692         }
 693
 694         xe_exec_queue_put(q);
 695
 696         return ret;
 697 }
 698
 699 /**
 700  * xe_exec_queue_is_lr() - Whether an exec_queue is long-running
 701  * @q: The exec_queue
 702  *
 703  * Return: True if the exec_queue is long-running, false otherwise.
 704  */
 705 bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
 706 {
 707         return q->vm && xe_vm_in_lr_mode(q->vm) &&
 708                 !(q->flags & EXEC_QUEUE_FLAG_VM);
 709 }
 710
 711 static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q)
 712 {
 713         return q->lrc[0]->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc[0]) - 1;
 714 }
 715
 716 /**
 717  * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full
 718  * @q: The exec_queue
 719  *
 720  * Return: True if the exec_queue's ring is full, false otherwise.
 721  */
 722 bool xe_exec_queue_ring_full(struct xe_exec_queue *q)
 723 {
 724         struct xe_lrc *lrc = q->lrc[0];
 725         s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES;
 726
 727         return xe_exec_queue_num_job_inflight(q) >= max_job;
 728 }
 729
 730 /**
 731  * xe_exec_queue_is_idle() - Whether an exec_queue is idle.
 732  * @q: The exec_queue
 733  *
 734  * FIXME: Need to determine what to use as the short-lived
 735  * timeline lock for the exec_queues, so that the return value
 736  * of this function becomes more than just an advisory
 737  * snapshot in time. The timeline lock must protect the
 738  * seqno from racing submissions on the same exec_queue.
 739  * Typically vm->resv, but user-created timeline locks use the migrate vm
 740  * and never grabs the migrate vm->resv so we have a race there.
 741  *
 742  * Return: True if the exec_queue is idle, false otherwise.
 743  */
 744 bool xe_exec_queue_is_idle(struct xe_exec_queue *q)
 745 {
 746         if (xe_exec_queue_is_parallel(q)) {
 747                 int i;
 748
 749                 for (i = 0; i < q->width; ++i) {
 750                         if (xe_lrc_seqno(q->lrc[i]) !=
 751                             q->lrc[i]->fence_ctx.next_seqno - 1)
 752                                 return false;
 753                 }
 754
 755                 return true;
 756         }
 757
 758         return xe_lrc_seqno(q->lrc[0]) ==
 759                 q->lrc[0]->fence_ctx.next_seqno - 1;
 760 }
 761
 762 /**
 763  * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue
 764  * from hw
 765  * @q: The exec queue
 766  *
 767  * Update the timestamp saved by HW for this exec queue and save run ticks
 768  * calculated by using the delta from last update.
 769  */
 770 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
 771 {
 772         struct xe_device *xe = gt_to_xe(q->gt);
 773         struct xe_lrc *lrc;
 774         u32 old_ts, new_ts;
 775         int idx;
 776
 777         /*
 778          * Jobs that are executed by kernel doesn't have a corresponding xe_file
 779          * and thus are not accounted.
 780          */
 781         if (!q->xef)
 782                 return;
 783
 784         /* Synchronize with unbind while holding the xe file open */
 785         if (!drm_dev_enter(&xe->drm, &idx))
 786                 return;
 787         /*
 788          * Only sample the first LRC. For parallel submission, all of them are
 789          * scheduled together and we compensate that below by multiplying by
 790          * width - this may introduce errors if that premise is not true and
 791          * they don't exit 100% aligned. On the other hand, looping through
 792          * the LRCs and reading them in different time could also introduce
 793          * errors.
 794          */
 795         lrc = q->lrc[0];
 796         new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
 797         q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;
 798
 799         drm_dev_exit(idx);
 800 }
 801
 802 /**
 803  * xe_exec_queue_kill - permanently stop all execution from an exec queue
 804  * @q: The exec queue
 805  *
 806  * This function permanently stops all activity on an exec queue. If the queue
 807  * is actively executing on the HW, it will be kicked off the engine; any
 808  * pending jobs are discarded and all future submissions are rejected.
 809  * This function is safe to call multiple times.
 810  */
 811 void xe_exec_queue_kill(struct xe_exec_queue *q)
 812 {
 813         struct xe_exec_queue *eq = q, *next;
 814
 815         list_for_each_entry_safe(eq, next, &eq->multi_gt_list,
 816                                  multi_gt_link) {
 817                 q->ops->kill(eq);
 818                 xe_vm_remove_compute_exec_queue(q->vm, eq);
 819         }
 820
 821         q->ops->kill(q);
 822         xe_vm_remove_compute_exec_queue(q->vm, q);
 823 }
 824
 825 int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
 826                                 struct drm_file *file)
 827 {
 828         struct xe_device *xe = to_xe_device(dev);
 829         struct xe_file *xef = to_xe_file(file);
 830         struct drm_xe_exec_queue_destroy *args = data;
 831         struct xe_exec_queue *q;
 832
 833         if (XE_IOCTL_DBG(xe, args->pad) ||
 834             XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
 835                 return -EINVAL;
 836
 837         mutex_lock(&xef->exec_queue.lock);
 838         q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id);
 839         if (q)
 840                 atomic_inc(&xef->exec_queue.pending_removal);
 841         mutex_unlock(&xef->exec_queue.lock);
 842
 843         if (XE_IOCTL_DBG(xe, !q))
 844                 return -ENOENT;
 845
 846         if (q->vm && q->hwe->hw_engine_group)
 847                 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
 848
 849         xe_exec_queue_kill(q);
 850
 851         trace_xe_exec_queue_close(q);
 852         xe_exec_queue_put(q);
 853
 854         return 0;
 855 }
 856
 857 static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
 858                                                     struct xe_vm *vm)
 859 {
 860         if (q->flags & EXEC_QUEUE_FLAG_VM) {
 861                 lockdep_assert_held(&vm->lock);
 862         } else {
 863                 xe_vm_assert_held(vm);
 864                 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem);
 865         }
 866 }
 867
 868 /**
 869  * xe_exec_queue_last_fence_put() - Drop ref to last fence
 870  * @q: The exec queue
 871  * @vm: The VM the engine does a bind or exec for
 872  */
 873 void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm)
 874 {
 875         xe_exec_queue_last_fence_lockdep_assert(q, vm);
 876
 877         xe_exec_queue_last_fence_put_unlocked(q);
 878 }
 879
 880 /**
 881  * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked
 882  * @q: The exec queue
 883  *
 884  * Only safe to be called from xe_exec_queue_destroy().
 885  */
 886 void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q)
 887 {
 888         if (q->last_fence) {
 889                 dma_fence_put(q->last_fence);
 890                 q->last_fence = NULL;
 891         }
 892 }
 893
 894 /**
 895  * xe_exec_queue_last_fence_get() - Get last fence
 896  * @q: The exec queue
 897  * @vm: The VM the engine does a bind or exec for
 898  *
 899  * Get last fence, takes a ref
 900  *
 901  * Returns: last fence if not signaled, dma fence stub if signaled
 902  */
 903 struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q,
 904                                                struct xe_vm *vm)
 905 {
 906         struct dma_fence *fence;
 907
 908         xe_exec_queue_last_fence_lockdep_assert(q, vm);
 909
 910         if (q->last_fence &&
 911             test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
 912                 xe_exec_queue_last_fence_put(q, vm);
 913
 914         fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
 915         dma_fence_get(fence);
 916         return fence;
 917 }
 918
 919 /**
 920  * xe_exec_queue_last_fence_get_for_resume() - Get last fence
 921  * @q: The exec queue
 922  * @vm: The VM the engine does a bind or exec for
 923  *
 924  * Get last fence, takes a ref. Only safe to be called in the context of
 925  * resuming the hw engine group's long-running exec queue, when the group
 926  * semaphore is held.
 927  *
 928  * Returns: last fence if not signaled, dma fence stub if signaled
 929  */
 930 struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q,
 931                                                           struct xe_vm *vm)
 932 {
 933         struct dma_fence *fence;
 934
 935         lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem);
 936
 937         if (q->last_fence &&
 938             test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
 939                 xe_exec_queue_last_fence_put_unlocked(q);
 940
 941         fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
 942         dma_fence_get(fence);
 943         return fence;
 944 }
 945
 946 /**
 947  * xe_exec_queue_last_fence_set() - Set last fence
 948  * @q: The exec queue
 949  * @vm: The VM the engine does a bind or exec for
 950  * @fence: The fence
 951  *
 952  * Set the last fence for the engine. Increases reference count for fence, when
 953  * closing engine xe_exec_queue_last_fence_put should be called.
 954  */
 955 void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm,
 956                                   struct dma_fence *fence)
 957 {
 958         xe_exec_queue_last_fence_lockdep_assert(q, vm);
 959
 960         xe_exec_queue_last_fence_put(q, vm);
 961         q->last_fence = dma_fence_get(fence);
 962 }
 963
 964 /**
 965  * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue
 966  * @q: The exec queue
 967  * @vm: The VM the engine does a bind or exec for
 968  *
 969  * Returns:
 970  * -ETIME if there exists an unsignalled last fence dependency, zero otherwise.
 971  */
 972 int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm)
 973 {
 974         struct dma_fence *fence;
 975         int err = 0;
 976
 977         fence = xe_exec_queue_last_fence_get(q, vm);
 978         if (fence) {
 979                 err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ?
 980                         0 : -ETIME;
 981                 dma_fence_put(fence);
 982         }
 983
 984         return err;
 985 }