drivers/gpu/drm/v3d/v3d_sched.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2018 Broadcom */
   3
   4 /**
   5  * DOC: Broadcom V3D scheduling
   6  *
   7  * The shared DRM GPU scheduler is used to coordinate submitting jobs
   8  * to the hardware.  Each DRM fd (roughly a client process) gets its
   9  * own scheduler entity, which will process jobs in order.  The GPU
  10  * scheduler will round-robin between clients to submit the next job.
  11  *
  12  * For simplicity, and in order to keep latency low for interactive
  13  * jobs when bulk background jobs are queued up, we submit a new job
  14  * to the HW only when it has completed the last one, instead of
  15  * filling up the CT[01]Q FIFOs with jobs.  Similarly, we use
  16  * drm_sched_job_add_dependency() to manage the dependency between bin and
  17  * render, instead of having the clients submit jobs using the HW's
  18  * semaphores to interlock between them.
  19  */
  20
  21 #include <linux/sched/clock.h>
  22 #include <linux/kthread.h>
  23
  24 #include <drm/drm_syncobj.h>
  25
  26 #include "v3d_drv.h"
  27 #include "v3d_regs.h"
  28 #include "v3d_trace.h"
  29
  30 #define V3D_CSD_CFG012_WG_COUNT_SHIFT 16
  31
  32 static struct v3d_job *
  33 to_v3d_job(struct drm_sched_job *sched_job)
  34 {
  35         return container_of(sched_job, struct v3d_job, base);
  36 }
  37
  38 static struct v3d_bin_job *
  39 to_bin_job(struct drm_sched_job *sched_job)
  40 {
  41         return container_of(sched_job, struct v3d_bin_job, base.base);
  42 }
  43
  44 static struct v3d_render_job *
  45 to_render_job(struct drm_sched_job *sched_job)
  46 {
  47         return container_of(sched_job, struct v3d_render_job, base.base);
  48 }
  49
  50 static struct v3d_tfu_job *
  51 to_tfu_job(struct drm_sched_job *sched_job)
  52 {
  53         return container_of(sched_job, struct v3d_tfu_job, base.base);
  54 }
  55
  56 static struct v3d_csd_job *
  57 to_csd_job(struct drm_sched_job *sched_job)
  58 {
  59         return container_of(sched_job, struct v3d_csd_job, base.base);
  60 }
  61
  62 static struct v3d_cpu_job *
  63 to_cpu_job(struct drm_sched_job *sched_job)
  64 {
  65         return container_of(sched_job, struct v3d_cpu_job, base.base);
  66 }
  67
  68 static void
  69 v3d_sched_job_free(struct drm_sched_job *sched_job)
  70 {
  71         struct v3d_job *job = to_v3d_job(sched_job);
  72
  73         v3d_job_cleanup(job);
  74 }
  75
  76 void
  77 v3d_timestamp_query_info_free(struct v3d_timestamp_query_info *query_info,
  78                               unsigned int count)
  79 {
  80         if (query_info->queries) {
  81                 unsigned int i;
  82
  83                 for (i = 0; i < count; i++)
  84                         drm_syncobj_put(query_info->queries[i].syncobj);
  85
  86                 kvfree(query_info->queries);
  87         }
  88 }
  89
  90 void
  91 v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,
  92                                 unsigned int count)
  93 {
  94         if (query_info->queries) {
  95                 unsigned int i;
  96
  97                 for (i = 0; i < count; i++) {
  98                         drm_syncobj_put(query_info->queries[i].syncobj);
  99                         kvfree(query_info->queries[i].kperfmon_ids);
 100                 }
 101
 102                 kvfree(query_info->queries);
 103         }
 104 }
 105
 106 static void
 107 v3d_cpu_job_free(struct drm_sched_job *sched_job)
 108 {
 109         struct v3d_cpu_job *job = to_cpu_job(sched_job);
 110
 111         v3d_timestamp_query_info_free(&job->timestamp_query,
 112                                       job->timestamp_query.count);
 113
 114         v3d_performance_query_info_free(&job->performance_query,
 115                                         job->performance_query.count);
 116
 117         v3d_job_cleanup(&job->base);
 118 }
 119
 120 static void
 121 v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
 122 {
 123         if (job->perfmon != v3d->active_perfmon)
 124                 v3d_perfmon_stop(v3d, v3d->active_perfmon, true);
 125
 126         if (job->perfmon && v3d->active_perfmon != job->perfmon)
 127                 v3d_perfmon_start(v3d, job->perfmon);
 128 }
 129
 130 static void
 131 v3d_job_start_stats(struct v3d_job *job, enum v3d_queue queue)
 132 {
 133         struct v3d_dev *v3d = job->v3d;
 134         struct v3d_file_priv *file = job->file->driver_priv;
 135         struct v3d_stats *global_stats = &v3d->queue[queue].stats;
 136         struct v3d_stats *local_stats = &file->stats[queue];
 137         u64 now = local_clock();
 138
 139         write_seqcount_begin(&local_stats->lock);
 140         local_stats->start_ns = now;
 141         write_seqcount_end(&local_stats->lock);
 142
 143         write_seqcount_begin(&global_stats->lock);
 144         global_stats->start_ns = now;
 145         write_seqcount_end(&global_stats->lock);
 146 }
 147
 148 static void
 149 v3d_stats_update(struct v3d_stats *stats, u64 now)
 150 {
 151         write_seqcount_begin(&stats->lock);
 152         stats->enabled_ns += now - stats->start_ns;
 153         stats->jobs_completed++;
 154         stats->start_ns = 0;
 155         write_seqcount_end(&stats->lock);
 156 }
 157
 158 void
 159 v3d_job_update_stats(struct v3d_job *job, enum v3d_queue queue)
 160 {
 161         struct v3d_dev *v3d = job->v3d;
 162         struct v3d_file_priv *file = job->file->driver_priv;
 163         struct v3d_stats *global_stats = &v3d->queue[queue].stats;
 164         struct v3d_stats *local_stats = &file->stats[queue];
 165         u64 now = local_clock();
 166
 167         v3d_stats_update(local_stats, now);
 168         v3d_stats_update(global_stats, now);
 169 }
 170
 171 static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 172 {
 173         struct v3d_bin_job *job = to_bin_job(sched_job);
 174         struct v3d_dev *v3d = job->base.v3d;
 175         struct drm_device *dev = &v3d->drm;
 176         struct dma_fence *fence;
 177         unsigned long irqflags;
 178
 179         if (unlikely(job->base.base.s_fence->finished.error))
 180                 return NULL;
 181
 182         /* Lock required around bin_job update vs
 183          * v3d_overflow_mem_work().
 184          */
 185         spin_lock_irqsave(&v3d->job_lock, irqflags);
 186         v3d->bin_job = job;
 187         /* Clear out the overflow allocation, so we don't
 188          * reuse the overflow attached to a previous job.
 189          */
 190         V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);
 191         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
 192
 193         v3d_invalidate_caches(v3d);
 194
 195         fence = v3d_fence_create(v3d, V3D_BIN);
 196         if (IS_ERR(fence))
 197                 return NULL;
 198
 199         if (job->base.irq_fence)
 200                 dma_fence_put(job->base.irq_fence);
 201         job->base.irq_fence = dma_fence_get(fence);
 202
 203         trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 204                             job->start, job->end);
 205
 206         v3d_job_start_stats(&job->base, V3D_BIN);
 207         v3d_switch_perfmon(v3d, &job->base);
 208
 209         /* Set the current and end address of the control list.
 210          * Writing the end register is what starts the job.
 211          */
 212         if (job->qma) {
 213                 V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, job->qma);
 214                 V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, job->qms);
 215         }
 216         if (job->qts) {
 217                 V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,
 218                                V3D_CLE_CT0QTS_ENABLE |
 219                                job->qts);
 220         }
 221         V3D_CORE_WRITE(0, V3D_CLE_CT0QBA, job->start);
 222         V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);
 223
 224         return fence;
 225 }
 226
 227 static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 228 {
 229         struct v3d_render_job *job = to_render_job(sched_job);
 230         struct v3d_dev *v3d = job->base.v3d;
 231         struct drm_device *dev = &v3d->drm;
 232         struct dma_fence *fence;
 233
 234         if (unlikely(job->base.base.s_fence->finished.error))
 235                 return NULL;
 236
 237         v3d->render_job = job;
 238
 239         /* Can we avoid this flush?  We need to be careful of
 240          * scheduling, though -- imagine job0 rendering to texture and
 241          * job1 reading, and them being executed as bin0, bin1,
 242          * render0, render1, so that render1's flush at bin time
 243          * wasn't enough.
 244          */
 245         v3d_invalidate_caches(v3d);
 246
 247         fence = v3d_fence_create(v3d, V3D_RENDER);
 248         if (IS_ERR(fence))
 249                 return NULL;
 250
 251         if (job->base.irq_fence)
 252                 dma_fence_put(job->base.irq_fence);
 253         job->base.irq_fence = dma_fence_get(fence);
 254
 255         trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 256                             job->start, job->end);
 257
 258         v3d_job_start_stats(&job->base, V3D_RENDER);
 259         v3d_switch_perfmon(v3d, &job->base);
 260
 261         /* XXX: Set the QCFG */
 262
 263         /* Set the current and end address of the control list.
 264          * Writing the end register is what starts the job.
 265          */
 266         V3D_CORE_WRITE(0, V3D_CLE_CT1QBA, job->start);
 267         V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);
 268
 269         return fence;
 270 }
 271
 272 static struct dma_fence *
 273 v3d_tfu_job_run(struct drm_sched_job *sched_job)
 274 {
 275         struct v3d_tfu_job *job = to_tfu_job(sched_job);
 276         struct v3d_dev *v3d = job->base.v3d;
 277         struct drm_device *dev = &v3d->drm;
 278         struct dma_fence *fence;
 279
 280         fence = v3d_fence_create(v3d, V3D_TFU);
 281         if (IS_ERR(fence))
 282                 return NULL;
 283
 284         v3d->tfu_job = job;
 285         if (job->base.irq_fence)
 286                 dma_fence_put(job->base.irq_fence);
 287         job->base.irq_fence = dma_fence_get(fence);
 288
 289         trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 290
 291         v3d_job_start_stats(&job->base, V3D_TFU);
 292
 293         V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);
 294         V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);
 295         V3D_WRITE(V3D_TFU_ICA(v3d->ver), job->args.ica);
 296         V3D_WRITE(V3D_TFU_IUA(v3d->ver), job->args.iua);
 297         V3D_WRITE(V3D_TFU_IOA(v3d->ver), job->args.ioa);
 298         if (v3d->ver >= 71)
 299                 V3D_WRITE(V3D_V7_TFU_IOC, job->args.v71.ioc);
 300         V3D_WRITE(V3D_TFU_IOS(v3d->ver), job->args.ios);
 301         V3D_WRITE(V3D_TFU_COEF0(v3d->ver), job->args.coef[0]);
 302         if (v3d->ver >= 71 || (job->args.coef[0] & V3D_TFU_COEF0_USECOEF)) {
 303                 V3D_WRITE(V3D_TFU_COEF1(v3d->ver), job->args.coef[1]);
 304                 V3D_WRITE(V3D_TFU_COEF2(v3d->ver), job->args.coef[2]);
 305                 V3D_WRITE(V3D_TFU_COEF3(v3d->ver), job->args.coef[3]);
 306         }
 307         /* ICFG kicks off the job. */
 308         V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC);
 309
 310         return fence;
 311 }
 312
 313 static struct dma_fence *
 314 v3d_csd_job_run(struct drm_sched_job *sched_job)
 315 {
 316         struct v3d_csd_job *job = to_csd_job(sched_job);
 317         struct v3d_dev *v3d = job->base.v3d;
 318         struct drm_device *dev = &v3d->drm;
 319         struct dma_fence *fence;
 320         int i, csd_cfg0_reg;
 321
 322         v3d->csd_job = job;
 323
 324         v3d_invalidate_caches(v3d);
 325
 326         fence = v3d_fence_create(v3d, V3D_CSD);
 327         if (IS_ERR(fence))
 328                 return NULL;
 329
 330         if (job->base.irq_fence)
 331                 dma_fence_put(job->base.irq_fence);
 332         job->base.irq_fence = dma_fence_get(fence);
 333
 334         trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 335
 336         v3d_job_start_stats(&job->base, V3D_CSD);
 337         v3d_switch_perfmon(v3d, &job->base);
 338
 339         csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);
 340         for (i = 1; i <= 6; i++)
 341                 V3D_CORE_WRITE(0, csd_cfg0_reg + 4 * i, job->args.cfg[i]);
 342
 343         /* Although V3D 7.1 has an eighth configuration register, we are not
 344          * using it. Therefore, make sure it remains unused.
 345          *
 346          * XXX: Set the CFG7 register
 347          */
 348         if (v3d->ver >= 71)
 349                 V3D_CORE_WRITE(0, V3D_V7_CSD_QUEUED_CFG7, 0);
 350
 351         /* CFG0 write kicks off the job. */
 352         V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]);
 353
 354         return fence;
 355 }
 356
 357 static void
 358 v3d_rewrite_csd_job_wg_counts_from_indirect(struct v3d_cpu_job *job)
 359 {
 360         struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;
 361         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 362         struct v3d_bo *indirect = to_v3d_bo(indirect_csd->indirect);
 363         struct drm_v3d_submit_csd *args = &indirect_csd->job->args;
 364         u32 *wg_counts;
 365
 366         v3d_get_bo_vaddr(bo);
 367         v3d_get_bo_vaddr(indirect);
 368
 369         wg_counts = (uint32_t *)(bo->vaddr + indirect_csd->offset);
 370
 371         if (wg_counts[0] == 0 || wg_counts[1] == 0 || wg_counts[2] == 0)
 372                 return;
 373
 374         args->cfg[0] = wg_counts[0] << V3D_CSD_CFG012_WG_COUNT_SHIFT;
 375         args->cfg[1] = wg_counts[1] << V3D_CSD_CFG012_WG_COUNT_SHIFT;
 376         args->cfg[2] = wg_counts[2] << V3D_CSD_CFG012_WG_COUNT_SHIFT;
 377         args->cfg[4] = DIV_ROUND_UP(indirect_csd->wg_size, 16) *
 378                        (wg_counts[0] * wg_counts[1] * wg_counts[2]) - 1;
 379
 380         for (int i = 0; i < 3; i++) {
 381                 /* 0xffffffff indicates that the uniform rewrite is not needed */
 382                 if (indirect_csd->wg_uniform_offsets[i] != 0xffffffff) {
 383                         u32 uniform_idx = indirect_csd->wg_uniform_offsets[i];
 384                         ((uint32_t *)indirect->vaddr)[uniform_idx] = wg_counts[i];
 385                 }
 386         }
 387
 388         v3d_put_bo_vaddr(indirect);
 389         v3d_put_bo_vaddr(bo);
 390 }
 391
 392 static void
 393 v3d_timestamp_query(struct v3d_cpu_job *job)
 394 {
 395         struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;
 396         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 397         u8 *value_addr;
 398
 399         v3d_get_bo_vaddr(bo);
 400
 401         for (int i = 0; i < timestamp_query->count; i++) {
 402                 value_addr = ((u8 *)bo->vaddr) + timestamp_query->queries[i].offset;
 403                 *((u64 *)value_addr) = i == 0 ? ktime_get_ns() : 0ull;
 404
 405                 drm_syncobj_replace_fence(timestamp_query->queries[i].syncobj,
 406                                           job->base.done_fence);
 407         }
 408
 409         v3d_put_bo_vaddr(bo);
 410 }
 411
 412 static void
 413 v3d_reset_timestamp_queries(struct v3d_cpu_job *job)
 414 {
 415         struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;
 416         struct v3d_timestamp_query *queries = timestamp_query->queries;
 417         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 418         u8 *value_addr;
 419
 420         v3d_get_bo_vaddr(bo);
 421
 422         for (int i = 0; i < timestamp_query->count; i++) {
 423                 value_addr = ((u8 *)bo->vaddr) + queries[i].offset;
 424                 *((u64 *)value_addr) = 0;
 425
 426                 drm_syncobj_replace_fence(queries[i].syncobj, NULL);
 427         }
 428
 429         v3d_put_bo_vaddr(bo);
 430 }
 431
 432 static void write_to_buffer_32(u32 *dst, unsigned int idx, u32 value)
 433 {
 434         dst[idx] = value;
 435 }
 436
 437 static void write_to_buffer_64(u64 *dst, unsigned int idx, u64 value)
 438 {
 439         dst[idx] = value;
 440 }
 441
 442 static void
 443 write_to_buffer(void *dst, unsigned int idx, bool do_64bit, u64 value)
 444 {
 445         if (do_64bit)
 446                 write_to_buffer_64(dst, idx, value);
 447         else
 448                 write_to_buffer_32(dst, idx, value);
 449 }
 450
 451 static void
 452 v3d_copy_query_results(struct v3d_cpu_job *job)
 453 {
 454         struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;
 455         struct v3d_timestamp_query *queries = timestamp_query->queries;
 456         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 457         struct v3d_bo *timestamp = to_v3d_bo(job->base.bo[1]);
 458         struct v3d_copy_query_results_info *copy = &job->copy;
 459         struct dma_fence *fence;
 460         u8 *query_addr;
 461         bool available, write_result;
 462         u8 *data;
 463         int i;
 464
 465         v3d_get_bo_vaddr(bo);
 466         v3d_get_bo_vaddr(timestamp);
 467
 468         data = ((u8 *)bo->vaddr) + copy->offset;
 469
 470         for (i = 0; i < timestamp_query->count; i++) {
 471                 fence = drm_syncobj_fence_get(queries[i].syncobj);
 472                 available = fence ? dma_fence_is_signaled(fence) : false;
 473
 474                 write_result = available || copy->do_partial;
 475                 if (write_result) {
 476                         query_addr = ((u8 *)timestamp->vaddr) + queries[i].offset;
 477                         write_to_buffer(data, 0, copy->do_64bit, *((u64 *)query_addr));
 478                 }
 479
 480                 if (copy->availability_bit)
 481                         write_to_buffer(data, 1, copy->do_64bit, available ? 1u : 0u);
 482
 483                 data += copy->stride;
 484
 485                 dma_fence_put(fence);
 486         }
 487
 488         v3d_put_bo_vaddr(timestamp);
 489         v3d_put_bo_vaddr(bo);
 490 }
 491
 492 static void
 493 v3d_reset_performance_queries(struct v3d_cpu_job *job)
 494 {
 495         struct v3d_performance_query_info *performance_query = &job->performance_query;
 496         struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;
 497         struct v3d_dev *v3d = job->base.v3d;
 498         struct v3d_perfmon *perfmon;
 499
 500         for (int i = 0; i < performance_query->count; i++) {
 501                 for (int j = 0; j < performance_query->nperfmons; j++) {
 502                         perfmon = v3d_perfmon_find(v3d_priv,
 503                                                    performance_query->queries[i].kperfmon_ids[j]);
 504                         if (!perfmon) {
 505                                 DRM_DEBUG("Failed to find perfmon.");
 506                                 continue;
 507                         }
 508
 509                         v3d_perfmon_stop(v3d, perfmon, false);
 510
 511                         memset(perfmon->values, 0, perfmon->ncounters * sizeof(u64));
 512
 513                         v3d_perfmon_put(perfmon);
 514                 }
 515
 516                 drm_syncobj_replace_fence(performance_query->queries[i].syncobj, NULL);
 517         }
 518 }
 519
 520 static void
 521 v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data,
 522                                    unsigned int query)
 523 {
 524         struct v3d_performance_query_info *performance_query =
 525                                                 &job->performance_query;
 526         struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;
 527         struct v3d_performance_query *perf_query =
 528                         &performance_query->queries[query];
 529         struct v3d_dev *v3d = job->base.v3d;
 530         unsigned int i, j, offset;
 531
 532         for (i = 0, offset = 0;
 533              i < performance_query->nperfmons;
 534              i++, offset += DRM_V3D_MAX_PERF_COUNTERS) {
 535                 struct v3d_perfmon *perfmon;
 536
 537                 perfmon = v3d_perfmon_find(v3d_priv,
 538                                            perf_query->kperfmon_ids[i]);
 539                 if (!perfmon) {
 540                         DRM_DEBUG("Failed to find perfmon.");
 541                         continue;
 542                 }
 543
 544                 v3d_perfmon_stop(v3d, perfmon, true);
 545
 546                 if (job->copy.do_64bit) {
 547                         for (j = 0; j < perfmon->ncounters; j++)
 548                                 write_to_buffer_64(data, offset + j,
 549                                                    perfmon->values[j]);
 550                 } else {
 551                         for (j = 0; j < perfmon->ncounters; j++)
 552                                 write_to_buffer_32(data, offset + j,
 553                                                    perfmon->values[j]);
 554                 }
 555
 556                 v3d_perfmon_put(perfmon);
 557         }
 558 }
 559
 560 static void
 561 v3d_copy_performance_query(struct v3d_cpu_job *job)
 562 {
 563         struct v3d_performance_query_info *performance_query = &job->performance_query;
 564         struct v3d_copy_query_results_info *copy = &job->copy;
 565         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 566         struct dma_fence *fence;
 567         bool available, write_result;
 568         u8 *data;
 569
 570         v3d_get_bo_vaddr(bo);
 571
 572         data = ((u8 *)bo->vaddr) + copy->offset;
 573
 574         for (int i = 0; i < performance_query->count; i++) {
 575                 fence = drm_syncobj_fence_get(performance_query->queries[i].syncobj);
 576                 available = fence ? dma_fence_is_signaled(fence) : false;
 577
 578                 write_result = available || copy->do_partial;
 579                 if (write_result)
 580                         v3d_write_performance_query_result(job, data, i);
 581
 582                 if (copy->availability_bit)
 583                         write_to_buffer(data, performance_query->ncounters,
 584                                         copy->do_64bit, available ? 1u : 0u);
 585
 586                 data += copy->stride;
 587
 588                 dma_fence_put(fence);
 589         }
 590
 591         v3d_put_bo_vaddr(bo);
 592 }
 593
 594 static const v3d_cpu_job_fn cpu_job_function[] = {
 595         [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect,
 596         [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query,
 597         [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries,
 598         [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = v3d_copy_query_results,
 599         [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = v3d_reset_performance_queries,
 600         [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = v3d_copy_performance_query,
 601 };
 602
 603 static struct dma_fence *
 604 v3d_cpu_job_run(struct drm_sched_job *sched_job)
 605 {
 606         struct v3d_cpu_job *job = to_cpu_job(sched_job);
 607         struct v3d_dev *v3d = job->base.v3d;
 608
 609         v3d->cpu_job = job;
 610
 611         if (job->job_type >= ARRAY_SIZE(cpu_job_function)) {
 612                 DRM_DEBUG_DRIVER("Unknown CPU job: %d\n", job->job_type);
 613                 return NULL;
 614         }
 615
 616         v3d_job_start_stats(&job->base, V3D_CPU);
 617         trace_v3d_cpu_job_begin(&v3d->drm, job->job_type);
 618
 619         cpu_job_function[job->job_type](job);
 620
 621         trace_v3d_cpu_job_end(&v3d->drm, job->job_type);
 622         v3d_job_update_stats(&job->base, V3D_CPU);
 623
 624         return NULL;
 625 }
 626
 627 static struct dma_fence *
 628 v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 629 {
 630         struct v3d_job *job = to_v3d_job(sched_job);
 631         struct v3d_dev *v3d = job->v3d;
 632
 633         v3d_job_start_stats(job, V3D_CACHE_CLEAN);
 634
 635         v3d_clean_caches(v3d);
 636
 637         v3d_job_update_stats(job, V3D_CACHE_CLEAN);
 638
 639         return NULL;
 640 }
 641
 642 static enum drm_gpu_sched_stat
 643 v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)
 644 {
 645         enum v3d_queue q;
 646
 647         mutex_lock(&v3d->reset_lock);
 648
 649         /* block scheduler */
 650         for (q = 0; q < V3D_MAX_QUEUES; q++)
 651                 drm_sched_stop(&v3d->queue[q].sched, sched_job);
 652
 653         if (sched_job)
 654                 drm_sched_increase_karma(sched_job);
 655
 656         /* get the GPU back into the init state */
 657         v3d_reset(v3d);
 658
 659         for (q = 0; q < V3D_MAX_QUEUES; q++)
 660                 drm_sched_resubmit_jobs(&v3d->queue[q].sched);
 661
 662         /* Unblock schedulers and restart their jobs. */
 663         for (q = 0; q < V3D_MAX_QUEUES; q++) {
 664                 drm_sched_start(&v3d->queue[q].sched);
 665         }
 666
 667         mutex_unlock(&v3d->reset_lock);
 668
 669         return DRM_GPU_SCHED_STAT_NOMINAL;
 670 }
 671
 672 /* If the current address or return address have changed, then the GPU
 673  * has probably made progress and we should delay the reset.  This
 674  * could fail if the GPU got in an infinite loop in the CL, but that
 675  * is pretty unlikely outside of an i-g-t testcase.
 676  */
 677 static enum drm_gpu_sched_stat
 678 v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q,
 679                     u32 *timedout_ctca, u32 *timedout_ctra)
 680 {
 681         struct v3d_job *job = to_v3d_job(sched_job);
 682         struct v3d_dev *v3d = job->v3d;
 683         u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(q));
 684         u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(q));
 685
 686         if (*timedout_ctca != ctca || *timedout_ctra != ctra) {
 687                 *timedout_ctca = ctca;
 688                 *timedout_ctra = ctra;
 689                 return DRM_GPU_SCHED_STAT_NOMINAL;
 690         }
 691
 692         return v3d_gpu_reset_for_timeout(v3d, sched_job);
 693 }
 694
 695 static enum drm_gpu_sched_stat
 696 v3d_bin_job_timedout(struct drm_sched_job *sched_job)
 697 {
 698         struct v3d_bin_job *job = to_bin_job(sched_job);
 699
 700         return v3d_cl_job_timedout(sched_job, V3D_BIN,
 701                                    &job->timedout_ctca, &job->timedout_ctra);
 702 }
 703
 704 static enum drm_gpu_sched_stat
 705 v3d_render_job_timedout(struct drm_sched_job *sched_job)
 706 {
 707         struct v3d_render_job *job = to_render_job(sched_job);
 708
 709         return v3d_cl_job_timedout(sched_job, V3D_RENDER,
 710                                    &job->timedout_ctca, &job->timedout_ctra);
 711 }
 712
 713 static enum drm_gpu_sched_stat
 714 v3d_generic_job_timedout(struct drm_sched_job *sched_job)
 715 {
 716         struct v3d_job *job = to_v3d_job(sched_job);
 717
 718         return v3d_gpu_reset_for_timeout(job->v3d, sched_job);
 719 }
 720
 721 static enum drm_gpu_sched_stat
 722 v3d_csd_job_timedout(struct drm_sched_job *sched_job)
 723 {
 724         struct v3d_csd_job *job = to_csd_job(sched_job);
 725         struct v3d_dev *v3d = job->base.v3d;
 726         u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4(v3d->ver));
 727
 728         /* If we've made progress, skip reset and let the timer get
 729          * rearmed.
 730          */
 731         if (job->timedout_batches != batches) {
 732                 job->timedout_batches = batches;
 733                 return DRM_GPU_SCHED_STAT_NOMINAL;
 734         }
 735
 736         return v3d_gpu_reset_for_timeout(v3d, sched_job);
 737 }
 738
 739 static const struct drm_sched_backend_ops v3d_bin_sched_ops = {
 740         .run_job = v3d_bin_job_run,
 741         .timedout_job = v3d_bin_job_timedout,
 742         .free_job = v3d_sched_job_free,
 743 };
 744
 745 static const struct drm_sched_backend_ops v3d_render_sched_ops = {
 746         .run_job = v3d_render_job_run,
 747         .timedout_job = v3d_render_job_timedout,
 748         .free_job = v3d_sched_job_free,
 749 };
 750
 751 static const struct drm_sched_backend_ops v3d_tfu_sched_ops = {
 752         .run_job = v3d_tfu_job_run,
 753         .timedout_job = v3d_generic_job_timedout,
 754         .free_job = v3d_sched_job_free,
 755 };
 756
 757 static const struct drm_sched_backend_ops v3d_csd_sched_ops = {
 758         .run_job = v3d_csd_job_run,
 759         .timedout_job = v3d_csd_job_timedout,
 760         .free_job = v3d_sched_job_free
 761 };
 762
 763 static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {
 764         .run_job = v3d_cache_clean_job_run,
 765         .timedout_job = v3d_generic_job_timedout,
 766         .free_job = v3d_sched_job_free
 767 };
 768
 769 static const struct drm_sched_backend_ops v3d_cpu_sched_ops = {
 770         .run_job = v3d_cpu_job_run,
 771         .timedout_job = v3d_generic_job_timedout,
 772         .free_job = v3d_cpu_job_free
 773 };
 774
 775 int
 776 v3d_sched_init(struct v3d_dev *v3d)
 777 {
 778         int hw_jobs_limit = 1;
 779         int job_hang_limit = 0;
 780         int hang_limit_ms = 500;
 781         int ret;
 782
 783         ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,
 784                              &v3d_bin_sched_ops, NULL,
 785                              DRM_SCHED_PRIORITY_COUNT,
 786                              hw_jobs_limit, job_hang_limit,
 787                              msecs_to_jiffies(hang_limit_ms), NULL,
 788                              NULL, "v3d_bin", v3d->drm.dev);
 789         if (ret)
 790                 return ret;
 791
 792         ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,
 793                              &v3d_render_sched_ops, NULL,
 794                              DRM_SCHED_PRIORITY_COUNT,
 795                              hw_jobs_limit, job_hang_limit,
 796                              msecs_to_jiffies(hang_limit_ms), NULL,
 797                              NULL, "v3d_render", v3d->drm.dev);
 798         if (ret)
 799                 goto fail;
 800
 801         ret = drm_sched_init(&v3d->queue[V3D_TFU].sched,
 802                              &v3d_tfu_sched_ops, NULL,
 803                              DRM_SCHED_PRIORITY_COUNT,
 804                              hw_jobs_limit, job_hang_limit,
 805                              msecs_to_jiffies(hang_limit_ms), NULL,
 806                              NULL, "v3d_tfu", v3d->drm.dev);
 807         if (ret)
 808                 goto fail;
 809
 810         if (v3d_has_csd(v3d)) {
 811                 ret = drm_sched_init(&v3d->queue[V3D_CSD].sched,
 812                                      &v3d_csd_sched_ops, NULL,
 813                                      DRM_SCHED_PRIORITY_COUNT,
 814                                      hw_jobs_limit, job_hang_limit,
 815                                      msecs_to_jiffies(hang_limit_ms), NULL,
 816                                      NULL, "v3d_csd", v3d->drm.dev);
 817                 if (ret)
 818                         goto fail;
 819
 820                 ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched,
 821                                      &v3d_cache_clean_sched_ops, NULL,
 822                                      DRM_SCHED_PRIORITY_COUNT,
 823                                      hw_jobs_limit, job_hang_limit,
 824                                      msecs_to_jiffies(hang_limit_ms), NULL,
 825                                      NULL, "v3d_cache_clean", v3d->drm.dev);
 826                 if (ret)
 827                         goto fail;
 828         }
 829
 830         ret = drm_sched_init(&v3d->queue[V3D_CPU].sched,
 831                              &v3d_cpu_sched_ops, NULL,
 832                              DRM_SCHED_PRIORITY_COUNT,
 833                              1, job_hang_limit,
 834                              msecs_to_jiffies(hang_limit_ms), NULL,
 835                              NULL, "v3d_cpu", v3d->drm.dev);
 836         if (ret)
 837                 goto fail;
 838
 839         return 0;
 840
 841 fail:
 842         v3d_sched_fini(v3d);
 843         return ret;
 844 }
 845
 846 void
 847 v3d_sched_fini(struct v3d_dev *v3d)
 848 {
 849         enum v3d_queue q;
 850
 851         for (q = 0; q < V3D_MAX_QUEUES; q++) {
 852                 if (v3d->queue[q].sched.ready)
 853                         drm_sched_fini(&v3d->queue[q].sched);
 854         }
 855 }