drivers/accel/ivpu/ivpu_job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2024 Intel Corporation
   4  */
   5
   6 #include <drm/drm_file.h>
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/highmem.h>
  10 #include <linux/pci.h>
  11 #include <linux/module.h>
  12 #include <uapi/drm/ivpu_accel.h>
  13
  14 #include "ivpu_drv.h"
  15 #include "ivpu_fw.h"
  16 #include "ivpu_hw.h"
  17 #include "ivpu_ipc.h"
  18 #include "ivpu_job.h"
  19 #include "ivpu_jsm_msg.h"
  20 #include "ivpu_pm.h"
  21 #include "vpu_boot_api.h"
  22
  23 #define CMD_BUF_IDX          0
  24 #define JOB_ID_JOB_MASK      GENMASK(7, 0)
  25 #define JOB_ID_CONTEXT_MASK  GENMASK(31, 8)
  26 #define JOB_MAX_BUFFER_COUNT 65535
  27
  28 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
  29 {
  30         ivpu_hw_db_set(vdev, cmdq->db_id);
  31 }
  32
  33 static int ivpu_preemption_buffers_create(struct ivpu_device *vdev,
  34                                           struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  35 {
  36         u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE);
  37         u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE);
  38         struct ivpu_addr_range range;
  39
  40         if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW)
  41                 return 0;
  42
  43         range.start = vdev->hw->ranges.user.end - (primary_size * IVPU_NUM_CMDQS_PER_CTX);
  44         range.end = vdev->hw->ranges.user.end;
  45         cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, primary_size,
  46                                                    DRM_IVPU_BO_WC);
  47         if (!cmdq->primary_preempt_buf) {
  48                 ivpu_err(vdev, "Failed to create primary preemption buffer\n");
  49                 return -ENOMEM;
  50         }
  51
  52         range.start = vdev->hw->ranges.shave.end - (secondary_size * IVPU_NUM_CMDQS_PER_CTX);
  53         range.end = vdev->hw->ranges.shave.end;
  54         cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, secondary_size,
  55                                                      DRM_IVPU_BO_WC);
  56         if (!cmdq->secondary_preempt_buf) {
  57                 ivpu_err(vdev, "Failed to create secondary preemption buffer\n");
  58                 goto err_free_primary;
  59         }
  60
  61         return 0;
  62
  63 err_free_primary:
  64         ivpu_bo_free(cmdq->primary_preempt_buf);
  65         return -ENOMEM;
  66 }
  67
  68 static void ivpu_preemption_buffers_free(struct ivpu_device *vdev,
  69                                          struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  70 {
  71         if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW)
  72                 return;
  73
  74         drm_WARN_ON(&vdev->drm, !cmdq->primary_preempt_buf);
  75         drm_WARN_ON(&vdev->drm, !cmdq->secondary_preempt_buf);
  76         ivpu_bo_free(cmdq->primary_preempt_buf);
  77         ivpu_bo_free(cmdq->secondary_preempt_buf);
  78 }
  79
  80 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv)
  81 {
  82         struct xa_limit db_xa_limit = {.max = IVPU_MAX_DB, .min = IVPU_MIN_DB};
  83         struct ivpu_device *vdev = file_priv->vdev;
  84         struct ivpu_cmdq *cmdq;
  85         int ret;
  86
  87         cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
  88         if (!cmdq)
  89                 return NULL;
  90
  91         ret = xa_alloc(&vdev->db_xa, &cmdq->db_id, NULL, db_xa_limit, GFP_KERNEL);
  92         if (ret) {
  93                 ivpu_err(vdev, "Failed to allocate doorbell id: %d\n", ret);
  94                 goto err_free_cmdq;
  95         }
  96
  97         cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
  98         if (!cmdq->mem)
  99                 goto err_erase_xa;
 100
 101         ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);
 102         if (ret)
 103                 goto err_free_cmdq_mem;
 104
 105         return cmdq;
 106
 107 err_free_cmdq_mem:
 108         ivpu_bo_free(cmdq->mem);
 109 err_erase_xa:
 110         xa_erase(&vdev->db_xa, cmdq->db_id);
 111 err_free_cmdq:
 112         kfree(cmdq);
 113         return NULL;
 114 }
 115
 116 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 117 {
 118         if (!cmdq)
 119                 return;
 120
 121         ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);
 122         ivpu_bo_free(cmdq->mem);
 123         xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
 124         kfree(cmdq);
 125 }
 126
 127 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,
 128                               u8 priority)
 129 {
 130         struct ivpu_device *vdev = file_priv->vdev;
 131         int ret;
 132
 133         ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->db_id,
 134                                        task_pid_nr(current), engine,
 135                                        cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 136         if (ret)
 137                 return ret;
 138
 139         ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->db_id,
 140                                                         priority);
 141         if (ret)
 142                 return ret;
 143
 144         return 0;
 145 }
 146
 147 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 148 {
 149         struct ivpu_device *vdev = file_priv->vdev;
 150         int ret;
 151
 152         if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW)
 153                 ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->db_id, cmdq->db_id,
 154                                                cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 155         else
 156                 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
 157                                            cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 158
 159         if (!ret)
 160                 ivpu_dbg(vdev, JOB, "DB %d registered to ctx %d\n", cmdq->db_id, file_priv->ctx.id);
 161
 162         return ret;
 163 }
 164
 165 static int
 166 ivpu_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine, u8 priority)
 167 {
 168         struct ivpu_device *vdev = file_priv->vdev;
 169         struct vpu_job_queue_header *jobq_header;
 170         int ret;
 171
 172         lockdep_assert_held(&file_priv->lock);
 173
 174         if (cmdq->db_registered)
 175                 return 0;
 176
 177         cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
 178                                   sizeof(struct vpu_job_queue_entry));
 179
 180         cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
 181         jobq_header = &cmdq->jobq->header;
 182         jobq_header->engine_idx = engine;
 183         jobq_header->head = 0;
 184         jobq_header->tail = 0;
 185         wmb(); /* Flush WC buffer for jobq->header */
 186
 187         if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {
 188                 ret = ivpu_hws_cmdq_init(file_priv, cmdq, engine, priority);
 189                 if (ret)
 190                         return ret;
 191         }
 192
 193         ret = ivpu_register_db(file_priv, cmdq);
 194         if (ret)
 195                 return ret;
 196
 197         cmdq->db_registered = true;
 198
 199         return 0;
 200 }
 201
 202 static int ivpu_cmdq_fini(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 203 {
 204         struct ivpu_device *vdev = file_priv->vdev;
 205         int ret;
 206
 207         lockdep_assert_held(&file_priv->lock);
 208
 209         if (!cmdq->db_registered)
 210                 return 0;
 211
 212         cmdq->db_registered = false;
 213
 214         if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {
 215                 ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->db_id);
 216                 if (!ret)
 217                         ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->db_id);
 218         }
 219
 220         ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id);
 221         if (!ret)
 222                 ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);
 223
 224         return 0;
 225 }
 226
 227 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine,
 228                                            u8 priority)
 229 {
 230         int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
 231         struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];
 232         int ret;
 233
 234         lockdep_assert_held(&file_priv->lock);
 235
 236         if (!cmdq) {
 237                 cmdq = ivpu_cmdq_alloc(file_priv);
 238                 if (!cmdq)
 239                         return NULL;
 240                 file_priv->cmdq[cmdq_idx] = cmdq;
 241         }
 242
 243         ret = ivpu_cmdq_init(file_priv, cmdq, engine, priority);
 244         if (ret)
 245                 return NULL;
 246
 247         return cmdq;
 248 }
 249
 250 static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine, u8 priority)
 251 {
 252         int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
 253         struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];
 254
 255         lockdep_assert_held(&file_priv->lock);
 256
 257         if (cmdq) {
 258                 file_priv->cmdq[cmdq_idx] = NULL;
 259                 ivpu_cmdq_fini(file_priv, cmdq);
 260                 ivpu_cmdq_free(file_priv, cmdq);
 261         }
 262 }
 263
 264 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
 265 {
 266         u16 engine;
 267         u8 priority;
 268
 269         lockdep_assert_held(&file_priv->lock);
 270
 271         for (engine = 0; engine < IVPU_NUM_ENGINES; engine++)
 272                 for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++)
 273                         ivpu_cmdq_release_locked(file_priv, engine, priority);
 274 }
 275
 276 /*
 277  * Mark the doorbell as unregistered
 278  * This function needs to be called when the VPU hardware is restarted
 279  * and FW loses job queue state. The next time job queue is used it
 280  * will be registered again.
 281  */
 282 static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv)
 283 {
 284         u16 engine;
 285         u8 priority;
 286
 287         mutex_lock(&file_priv->lock);
 288
 289         for (engine = 0; engine < IVPU_NUM_ENGINES; engine++) {
 290                 for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++) {
 291                         int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
 292                         struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];
 293
 294                         if (cmdq)
 295                                 cmdq->db_registered = false;
 296                 }
 297         }
 298
 299         mutex_unlock(&file_priv->lock);
 300 }
 301
 302 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
 303 {
 304         struct ivpu_file_priv *file_priv;
 305         unsigned long ctx_id;
 306
 307         mutex_lock(&vdev->context_list_lock);
 308
 309         xa_for_each(&vdev->context_xa, ctx_id, file_priv)
 310                 ivpu_cmdq_reset(file_priv);
 311
 312         mutex_unlock(&vdev->context_list_lock);
 313 }
 314
 315 static void ivpu_cmdq_fini_all(struct ivpu_file_priv *file_priv)
 316 {
 317         u16 engine;
 318         u8 priority;
 319
 320         for (engine = 0; engine < IVPU_NUM_ENGINES; engine++) {
 321                 for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++) {
 322                         int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);
 323
 324                         if (file_priv->cmdq[cmdq_idx])
 325                                 ivpu_cmdq_fini(file_priv, file_priv->cmdq[cmdq_idx]);
 326                 }
 327         }
 328 }
 329
 330 void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv)
 331 {
 332         struct ivpu_device *vdev = file_priv->vdev;
 333
 334         lockdep_assert_held(&file_priv->lock);
 335
 336         ivpu_cmdq_fini_all(file_priv);
 337
 338         if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_OS)
 339                 ivpu_jsm_context_release(vdev, file_priv->ctx.id);
 340 }
 341
 342 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
 343 {
 344         struct ivpu_device *vdev = job->vdev;
 345         struct vpu_job_queue_header *header = &cmdq->jobq->header;
 346         struct vpu_job_queue_entry *entry;
 347         u32 tail = READ_ONCE(header->tail);
 348         u32 next_entry = (tail + 1) % cmdq->entry_count;
 349
 350         /* Check if there is space left in job queue */
 351         if (next_entry == header->head) {
 352                 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",
 353                          job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);
 354                 return -EBUSY;
 355         }
 356
 357         entry = &cmdq->jobq->job[tail];
 358         entry->batch_buf_addr = job->cmd_buf_vpu_addr;
 359         entry->job_id = job->job_id;
 360         entry->flags = 0;
 361         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
 362                 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
 363
 364         if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW &&
 365             (unlikely(!(ivpu_test_mode & IVPU_TEST_MODE_PREEMPTION_DISABLE)))) {
 366                 entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr;
 367                 entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf);
 368                 entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr;
 369                 entry->secondary_preempt_buf_size = ivpu_bo_size(cmdq->secondary_preempt_buf);
 370         }
 371
 372         wmb(); /* Ensure that tail is updated after filling entry */
 373         header->tail = next_entry;
 374         wmb(); /* Flush WC buffer for jobq header */
 375
 376         return 0;
 377 }
 378
 379 struct ivpu_fence {
 380         struct dma_fence base;
 381         spinlock_t lock; /* protects base */
 382         struct ivpu_device *vdev;
 383 };
 384
 385 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
 386 {
 387         return container_of(fence, struct ivpu_fence, base);
 388 }
 389
 390 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
 391 {
 392         return DRIVER_NAME;
 393 }
 394
 395 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
 396 {
 397         struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
 398
 399         return dev_name(ivpu_fence->vdev->drm.dev);
 400 }
 401
 402 static const struct dma_fence_ops ivpu_fence_ops = {
 403         .get_driver_name = ivpu_fence_get_driver_name,
 404         .get_timeline_name = ivpu_fence_get_timeline_name,
 405 };
 406
 407 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
 408 {
 409         struct ivpu_fence *fence;
 410
 411         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 412         if (!fence)
 413                 return NULL;
 414
 415         fence->vdev = vdev;
 416         spin_lock_init(&fence->lock);
 417         dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
 418
 419         return &fence->base;
 420 }
 421
 422 static void ivpu_job_destroy(struct ivpu_job *job)
 423 {
 424         struct ivpu_device *vdev = job->vdev;
 425         u32 i;
 426
 427         ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",
 428                  job->job_id, job->file_priv->ctx.id, job->engine_idx);
 429
 430         for (i = 0; i < job->bo_count; i++)
 431                 if (job->bos[i])
 432                         drm_gem_object_put(&job->bos[i]->base.base);
 433
 434         dma_fence_put(job->done_fence);
 435         ivpu_file_priv_put(&job->file_priv);
 436         kfree(job);
 437 }
 438
 439 static struct ivpu_job *
 440 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
 441 {
 442         struct ivpu_device *vdev = file_priv->vdev;
 443         struct ivpu_job *job;
 444
 445         job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
 446         if (!job)
 447                 return NULL;
 448
 449         job->vdev = vdev;
 450         job->engine_idx = engine_idx;
 451         job->bo_count = bo_count;
 452         job->done_fence = ivpu_fence_create(vdev);
 453         if (!job->done_fence) {
 454                 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
 455                 goto err_free_job;
 456         }
 457
 458         job->file_priv = ivpu_file_priv_get(file_priv);
 459
 460         ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
 461         return job;
 462
 463 err_free_job:
 464         kfree(job);
 465         return NULL;
 466 }
 467
 468 static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id)
 469 {
 470         struct ivpu_job *job;
 471
 472         xa_lock(&vdev->submitted_jobs_xa);
 473         job = __xa_erase(&vdev->submitted_jobs_xa, job_id);
 474
 475         if (xa_empty(&vdev->submitted_jobs_xa) && job) {
 476                 vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts),
 477                                             vdev->busy_time);
 478         }
 479
 480         xa_unlock(&vdev->submitted_jobs_xa);
 481
 482         return job;
 483 }
 484
 485 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 486 {
 487         struct ivpu_job *job;
 488
 489         job = ivpu_job_remove_from_submitted_jobs(vdev, job_id);
 490         if (!job)
 491                 return -ENOENT;
 492
 493         if (job->file_priv->has_mmu_faults)
 494                 job_status = DRM_IVPU_JOB_STATUS_ABORTED;
 495
 496         job->bos[CMD_BUF_IDX]->job_status = job_status;
 497         dma_fence_signal(job->done_fence);
 498
 499         ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
 500                  job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
 501
 502         ivpu_job_destroy(job);
 503         ivpu_stop_job_timeout_detection(vdev);
 504
 505         ivpu_rpm_put(vdev);
 506         return 0;
 507 }
 508
 509 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
 510 {
 511         struct ivpu_job *job;
 512         unsigned long id;
 513
 514         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 515                 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
 516 }
 517
 518 static int ivpu_job_submit(struct ivpu_job *job, u8 priority)
 519 {
 520         struct ivpu_file_priv *file_priv = job->file_priv;
 521         struct ivpu_device *vdev = job->vdev;
 522         struct xa_limit job_id_range;
 523         struct ivpu_cmdq *cmdq;
 524         bool is_first_job;
 525         int ret;
 526
 527         ret = ivpu_rpm_get(vdev);
 528         if (ret < 0)
 529                 return ret;
 530
 531         mutex_lock(&file_priv->lock);
 532
 533         cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx, priority);
 534         if (!cmdq) {
 535                 ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d engine %d prio %d\n",
 536                                       file_priv->ctx.id, job->engine_idx, priority);
 537                 ret = -EINVAL;
 538                 goto err_unlock_file_priv;
 539         }
 540
 541         job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
 542         job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;
 543
 544         xa_lock(&vdev->submitted_jobs_xa);
 545         is_first_job = xa_empty(&vdev->submitted_jobs_xa);
 546         ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);
 547         if (ret) {
 548                 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",
 549                          file_priv->ctx.id);
 550                 ret = -EBUSY;
 551                 goto err_unlock_submitted_jobs_xa;
 552         }
 553
 554         ret = ivpu_cmdq_push_job(cmdq, job);
 555         if (ret)
 556                 goto err_erase_xa;
 557
 558         ivpu_start_job_timeout_detection(vdev);
 559
 560         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
 561                 cmdq->jobq->header.head = cmdq->jobq->header.tail;
 562                 wmb(); /* Flush WC buffer for jobq header */
 563         } else {
 564                 ivpu_cmdq_ring_db(vdev, cmdq);
 565                 if (is_first_job)
 566                         vdev->busy_start_ts = ktime_get();
 567         }
 568
 569         ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d prio %d addr 0x%llx next %d\n",
 570                  job->job_id, file_priv->ctx.id, job->engine_idx, priority,
 571                  job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);
 572
 573         xa_unlock(&vdev->submitted_jobs_xa);
 574
 575         mutex_unlock(&file_priv->lock);
 576
 577         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW))
 578                 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
 579
 580         return 0;
 581
 582 err_erase_xa:
 583         __xa_erase(&vdev->submitted_jobs_xa, job->job_id);
 584 err_unlock_submitted_jobs_xa:
 585         xa_unlock(&vdev->submitted_jobs_xa);
 586 err_unlock_file_priv:
 587         mutex_unlock(&file_priv->lock);
 588         ivpu_rpm_put(vdev);
 589         return ret;
 590 }
 591
 592 static int
 593 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
 594                                 u32 buf_count, u32 commands_offset)
 595 {
 596         struct ivpu_file_priv *file_priv = file->driver_priv;
 597         struct ivpu_device *vdev = file_priv->vdev;
 598         struct ww_acquire_ctx acquire_ctx;
 599         enum dma_resv_usage usage;
 600         struct ivpu_bo *bo;
 601         int ret;
 602         u32 i;
 603
 604         for (i = 0; i < buf_count; i++) {
 605                 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
 606
 607                 if (!obj)
 608                         return -ENOENT;
 609
 610                 job->bos[i] = to_ivpu_bo(obj);
 611
 612                 ret = ivpu_bo_pin(job->bos[i]);
 613                 if (ret)
 614                         return ret;
 615         }
 616
 617         bo = job->bos[CMD_BUF_IDX];
 618         if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
 619                 ivpu_warn(vdev, "Buffer is already in use\n");
 620                 return -EBUSY;
 621         }
 622
 623         if (commands_offset >= ivpu_bo_size(bo)) {
 624                 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
 625                 return -EINVAL;
 626         }
 627
 628         job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
 629
 630         ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
 631                                         &acquire_ctx);
 632         if (ret) {
 633                 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
 634                 return ret;
 635         }
 636
 637         for (i = 0; i < buf_count; i++) {
 638                 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
 639                 if (ret) {
 640                         ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
 641                         goto unlock_reservations;
 642                 }
 643         }
 644
 645         for (i = 0; i < buf_count; i++) {
 646                 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
 647                 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
 648         }
 649
 650 unlock_reservations:
 651         drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
 652
 653         wmb(); /* Flush write combining buffers */
 654
 655         return ret;
 656 }
 657
 658 static inline u8 ivpu_job_to_hws_priority(struct ivpu_file_priv *file_priv, u8 priority)
 659 {
 660         if (priority == DRM_IVPU_JOB_PRIORITY_DEFAULT)
 661                 return DRM_IVPU_JOB_PRIORITY_NORMAL;
 662
 663         return priority - 1;
 664 }
 665
 666 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 667 {
 668         struct ivpu_file_priv *file_priv = file->driver_priv;
 669         struct ivpu_device *vdev = file_priv->vdev;
 670         struct drm_ivpu_submit *params = data;
 671         struct ivpu_job *job;
 672         u32 *buf_handles;
 673         int idx, ret;
 674         u8 priority;
 675
 676         if (params->engine > DRM_IVPU_ENGINE_COPY)
 677                 return -EINVAL;
 678
 679         if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
 680                 return -EINVAL;
 681
 682         if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)
 683                 return -EINVAL;
 684
 685         if (!IS_ALIGNED(params->commands_offset, 8))
 686                 return -EINVAL;
 687
 688         if (!file_priv->ctx.id)
 689                 return -EINVAL;
 690
 691         if (file_priv->has_mmu_faults)
 692                 return -EBADFD;
 693
 694         buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);
 695         if (!buf_handles)
 696                 return -ENOMEM;
 697
 698         ret = copy_from_user(buf_handles,
 699                              (void __user *)params->buffers_ptr,
 700                              params->buffer_count * sizeof(u32));
 701         if (ret) {
 702                 ret = -EFAULT;
 703                 goto err_free_handles;
 704         }
 705
 706         if (!drm_dev_enter(&vdev->drm, &idx)) {
 707                 ret = -ENODEV;
 708                 goto err_free_handles;
 709         }
 710
 711         ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
 712                  file_priv->ctx.id, params->buffer_count);
 713
 714         job = ivpu_job_create(file_priv, params->engine, params->buffer_count);
 715         if (!job) {
 716                 ivpu_err(vdev, "Failed to create job\n");
 717                 ret = -ENOMEM;
 718                 goto err_exit_dev;
 719         }
 720
 721         ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
 722                                               params->commands_offset);
 723         if (ret) {
 724                 ivpu_err(vdev, "Failed to prepare job: %d\n", ret);
 725                 goto err_destroy_job;
 726         }
 727
 728         priority = ivpu_job_to_hws_priority(file_priv, params->priority);
 729
 730         down_read(&vdev->pm->reset_lock);
 731         ret = ivpu_job_submit(job, priority);
 732         up_read(&vdev->pm->reset_lock);
 733         if (ret)
 734                 goto err_signal_fence;
 735
 736         drm_dev_exit(idx);
 737         kfree(buf_handles);
 738         return ret;
 739
 740 err_signal_fence:
 741         dma_fence_signal(job->done_fence);
 742 err_destroy_job:
 743         ivpu_job_destroy(job);
 744 err_exit_dev:
 745         drm_dev_exit(idx);
 746 err_free_handles:
 747         kfree(buf_handles);
 748         return ret;
 749 }
 750
 751 static void
 752 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 753                        struct vpu_jsm_msg *jsm_msg)
 754 {
 755         struct vpu_ipc_msg_payload_job_done *payload;
 756         int ret;
 757
 758         if (!jsm_msg) {
 759                 ivpu_err(vdev, "IPC message has no JSM payload\n");
 760                 return;
 761         }
 762
 763         if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
 764                 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
 765                 return;
 766         }
 767
 768         payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
 769         ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
 770         if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
 771                 ivpu_start_job_timeout_detection(vdev);
 772 }
 773
 774 void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
 775 {
 776         ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
 777                               VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
 778 }
 779
 780 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
 781 {
 782         ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
 783 }