drivers/gpu/drm/v3d/v3d_sched.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2018 Broadcom */
   3
   4 /**
   5  * DOC: Broadcom V3D scheduling
   6  *
   7  * The shared DRM GPU scheduler is used to coordinate submitting jobs
   8  * to the hardware. Each DRM fd (roughly a client process) gets its
   9  * own scheduler entity, which will process jobs in order. The GPU
  10  * scheduler will schedule the clients with a FIFO scheduling algorithm.
  11  *
  12  * For simplicity, and in order to keep latency low for interactive
  13  * jobs when bulk background jobs are queued up, we submit a new job
  14  * to the HW only when it has completed the last one, instead of
  15  * filling up the CT[01]Q FIFOs with jobs. Similarly, we use
  16  * `drm_sched_job_add_dependency()` to manage the dependency between bin
  17  * and render, instead of having the clients submit jobs using the HW's
  18  * semaphores to interlock between them.
  19  */
  20
  21 #include <linux/sched/clock.h>
  22 #include <linux/kthread.h>
  23
  24 #include <drm/drm_syncobj.h>
  25
  26 #include "v3d_drv.h"
  27 #include "v3d_regs.h"
  28 #include "v3d_trace.h"
  29
  30 #define V3D_CSD_CFG012_WG_COUNT_SHIFT 16
  31
  32 static struct v3d_job *
  33 to_v3d_job(struct drm_sched_job *sched_job)
  34 {
  35         return container_of(sched_job, struct v3d_job, base);
  36 }
  37
  38 static struct v3d_bin_job *
  39 to_bin_job(struct drm_sched_job *sched_job)
  40 {
  41         return container_of(sched_job, struct v3d_bin_job, base.base);
  42 }
  43
  44 static struct v3d_render_job *
  45 to_render_job(struct drm_sched_job *sched_job)
  46 {
  47         return container_of(sched_job, struct v3d_render_job, base.base);
  48 }
  49
  50 static struct v3d_tfu_job *
  51 to_tfu_job(struct drm_sched_job *sched_job)
  52 {
  53         return container_of(sched_job, struct v3d_tfu_job, base.base);
  54 }
  55
  56 static struct v3d_csd_job *
  57 to_csd_job(struct drm_sched_job *sched_job)
  58 {
  59         return container_of(sched_job, struct v3d_csd_job, base.base);
  60 }
  61
  62 static struct v3d_cpu_job *
  63 to_cpu_job(struct drm_sched_job *sched_job)
  64 {
  65         return container_of(sched_job, struct v3d_cpu_job, base.base);
  66 }
  67
  68 static void
  69 v3d_sched_job_free(struct drm_sched_job *sched_job)
  70 {
  71         struct v3d_job *job = to_v3d_job(sched_job);
  72
  73         v3d_job_cleanup(job);
  74 }
  75
  76 void
  77 v3d_timestamp_query_info_free(struct v3d_timestamp_query_info *query_info,
  78                               unsigned int count)
  79 {
  80         if (query_info->queries) {
  81                 unsigned int i;
  82
  83                 for (i = 0; i < count; i++)
  84                         drm_syncobj_put(query_info->queries[i].syncobj);
  85
  86                 kvfree(query_info->queries);
  87         }
  88 }
  89
  90 void
  91 v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,
  92                                 unsigned int count)
  93 {
  94         if (query_info->queries) {
  95                 unsigned int i;
  96
  97                 for (i = 0; i < count; i++) {
  98                         drm_syncobj_put(query_info->queries[i].syncobj);
  99                         kvfree(query_info->queries[i].kperfmon_ids);
 100                 }
 101
 102                 kvfree(query_info->queries);
 103         }
 104 }
 105
 106 static void
 107 v3d_cpu_job_free(struct drm_sched_job *sched_job)
 108 {
 109         struct v3d_cpu_job *job = to_cpu_job(sched_job);
 110
 111         v3d_timestamp_query_info_free(&job->timestamp_query,
 112                                       job->timestamp_query.count);
 113
 114         v3d_performance_query_info_free(&job->performance_query,
 115                                         job->performance_query.count);
 116
 117         v3d_job_cleanup(&job->base);
 118 }
 119
 120 static void
 121 v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
 122 {
 123         struct v3d_perfmon *perfmon = v3d->global_perfmon;
 124
 125         if (!perfmon)
 126                 perfmon = job->perfmon;
 127
 128         if (perfmon == v3d->active_perfmon)
 129                 return;
 130
 131         if (perfmon != v3d->active_perfmon)
 132                 v3d_perfmon_stop(v3d, v3d->active_perfmon, true);
 133
 134         if (perfmon && v3d->active_perfmon != perfmon)
 135                 v3d_perfmon_start(v3d, perfmon);
 136 }
 137
 138 static void
 139 v3d_job_start_stats(struct v3d_job *job, enum v3d_queue queue)
 140 {
 141         struct v3d_dev *v3d = job->v3d;
 142         struct v3d_file_priv *file = job->file->driver_priv;
 143         struct v3d_stats *global_stats = &v3d->queue[queue].stats;
 144         struct v3d_stats *local_stats = &file->stats[queue];
 145         u64 now = local_clock();
 146         unsigned long flags;
 147
 148         /*
 149          * We only need to disable local interrupts to appease lockdep who
 150          * otherwise would think v3d_job_start_stats vs v3d_stats_update has an
 151          * unsafe in-irq vs no-irq-off usage problem. This is a false positive
 152          * because all the locks are per queue and stats type, and all jobs are
 153          * completely one at a time serialised. More specifically:
 154          *
 155          * 1. Locks for GPU queues are updated from interrupt handlers under a
 156          *    spin lock and started here with preemption disabled.
 157          *
 158          * 2. Locks for CPU queues are updated from the worker with preemption
 159          *    disabled and equally started here with preemption disabled.
 160          *
 161          * Therefore both are consistent.
 162          *
 163          * 3. Because next job can only be queued after the previous one has
 164          *    been signaled, and locks are per queue, there is also no scope for
 165          *    the start part to race with the update part.
 166          */
 167         if (IS_ENABLED(CONFIG_LOCKDEP))
 168                 local_irq_save(flags);
 169         else
 170                 preempt_disable();
 171
 172         write_seqcount_begin(&local_stats->lock);
 173         local_stats->start_ns = now;
 174         write_seqcount_end(&local_stats->lock);
 175
 176         write_seqcount_begin(&global_stats->lock);
 177         global_stats->start_ns = now;
 178         write_seqcount_end(&global_stats->lock);
 179
 180         if (IS_ENABLED(CONFIG_LOCKDEP))
 181                 local_irq_restore(flags);
 182         else
 183                 preempt_enable();
 184 }
 185
 186 static void
 187 v3d_stats_update(struct v3d_stats *stats, u64 now)
 188 {
 189         write_seqcount_begin(&stats->lock);
 190         stats->enabled_ns += now - stats->start_ns;
 191         stats->jobs_completed++;
 192         stats->start_ns = 0;
 193         write_seqcount_end(&stats->lock);
 194 }
 195
 196 void
 197 v3d_job_update_stats(struct v3d_job *job, enum v3d_queue queue)
 198 {
 199         struct v3d_dev *v3d = job->v3d;
 200         struct v3d_file_priv *file = job->file->driver_priv;
 201         struct v3d_stats *global_stats = &v3d->queue[queue].stats;
 202         struct v3d_stats *local_stats = &file->stats[queue];
 203         u64 now = local_clock();
 204         unsigned long flags;
 205
 206         /* See comment in v3d_job_start_stats() */
 207         if (IS_ENABLED(CONFIG_LOCKDEP))
 208                 local_irq_save(flags);
 209         else
 210                 preempt_disable();
 211
 212         v3d_stats_update(local_stats, now);
 213         v3d_stats_update(global_stats, now);
 214
 215         if (IS_ENABLED(CONFIG_LOCKDEP))
 216                 local_irq_restore(flags);
 217         else
 218                 preempt_enable();
 219 }
 220
 221 static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 222 {
 223         struct v3d_bin_job *job = to_bin_job(sched_job);
 224         struct v3d_dev *v3d = job->base.v3d;
 225         struct drm_device *dev = &v3d->drm;
 226         struct dma_fence *fence;
 227         unsigned long irqflags;
 228
 229         if (unlikely(job->base.base.s_fence->finished.error))
 230                 return NULL;
 231
 232         /* Lock required around bin_job update vs
 233          * v3d_overflow_mem_work().
 234          */
 235         spin_lock_irqsave(&v3d->job_lock, irqflags);
 236         v3d->bin_job = job;
 237         /* Clear out the overflow allocation, so we don't
 238          * reuse the overflow attached to a previous job.
 239          */
 240         V3D_CORE_WRITE(0, V3D_PTB_BPOS, 0);
 241         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
 242
 243         v3d_invalidate_caches(v3d);
 244
 245         fence = v3d_fence_create(v3d, V3D_BIN);
 246         if (IS_ERR(fence))
 247                 return NULL;
 248
 249         if (job->base.irq_fence)
 250                 dma_fence_put(job->base.irq_fence);
 251         job->base.irq_fence = dma_fence_get(fence);
 252
 253         trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 254                             job->start, job->end);
 255
 256         v3d_job_start_stats(&job->base, V3D_BIN);
 257         v3d_switch_perfmon(v3d, &job->base);
 258
 259         /* Set the current and end address of the control list.
 260          * Writing the end register is what starts the job.
 261          */
 262         if (job->qma) {
 263                 V3D_CORE_WRITE(0, V3D_CLE_CT0QMA, job->qma);
 264                 V3D_CORE_WRITE(0, V3D_CLE_CT0QMS, job->qms);
 265         }
 266         if (job->qts) {
 267                 V3D_CORE_WRITE(0, V3D_CLE_CT0QTS,
 268                                V3D_CLE_CT0QTS_ENABLE |
 269                                job->qts);
 270         }
 271         V3D_CORE_WRITE(0, V3D_CLE_CT0QBA, job->start);
 272         V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);
 273
 274         return fence;
 275 }
 276
 277 static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 278 {
 279         struct v3d_render_job *job = to_render_job(sched_job);
 280         struct v3d_dev *v3d = job->base.v3d;
 281         struct drm_device *dev = &v3d->drm;
 282         struct dma_fence *fence;
 283
 284         if (unlikely(job->base.base.s_fence->finished.error))
 285                 return NULL;
 286
 287         v3d->render_job = job;
 288
 289         /* Can we avoid this flush?  We need to be careful of
 290          * scheduling, though -- imagine job0 rendering to texture and
 291          * job1 reading, and them being executed as bin0, bin1,
 292          * render0, render1, so that render1's flush at bin time
 293          * wasn't enough.
 294          */
 295         v3d_invalidate_caches(v3d);
 296
 297         fence = v3d_fence_create(v3d, V3D_RENDER);
 298         if (IS_ERR(fence))
 299                 return NULL;
 300
 301         if (job->base.irq_fence)
 302                 dma_fence_put(job->base.irq_fence);
 303         job->base.irq_fence = dma_fence_get(fence);
 304
 305         trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 306                             job->start, job->end);
 307
 308         v3d_job_start_stats(&job->base, V3D_RENDER);
 309         v3d_switch_perfmon(v3d, &job->base);
 310
 311         /* XXX: Set the QCFG */
 312
 313         /* Set the current and end address of the control list.
 314          * Writing the end register is what starts the job.
 315          */
 316         V3D_CORE_WRITE(0, V3D_CLE_CT1QBA, job->start);
 317         V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);
 318
 319         return fence;
 320 }
 321
 322 static struct dma_fence *
 323 v3d_tfu_job_run(struct drm_sched_job *sched_job)
 324 {
 325         struct v3d_tfu_job *job = to_tfu_job(sched_job);
 326         struct v3d_dev *v3d = job->base.v3d;
 327         struct drm_device *dev = &v3d->drm;
 328         struct dma_fence *fence;
 329
 330         fence = v3d_fence_create(v3d, V3D_TFU);
 331         if (IS_ERR(fence))
 332                 return NULL;
 333
 334         v3d->tfu_job = job;
 335         if (job->base.irq_fence)
 336                 dma_fence_put(job->base.irq_fence);
 337         job->base.irq_fence = dma_fence_get(fence);
 338
 339         trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 340
 341         v3d_job_start_stats(&job->base, V3D_TFU);
 342
 343         V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);
 344         V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);
 345         V3D_WRITE(V3D_TFU_ICA(v3d->ver), job->args.ica);
 346         V3D_WRITE(V3D_TFU_IUA(v3d->ver), job->args.iua);
 347         V3D_WRITE(V3D_TFU_IOA(v3d->ver), job->args.ioa);
 348         if (v3d->ver >= 71)
 349                 V3D_WRITE(V3D_V7_TFU_IOC, job->args.v71.ioc);
 350         V3D_WRITE(V3D_TFU_IOS(v3d->ver), job->args.ios);
 351         V3D_WRITE(V3D_TFU_COEF0(v3d->ver), job->args.coef[0]);
 352         if (v3d->ver >= 71 || (job->args.coef[0] & V3D_TFU_COEF0_USECOEF)) {
 353                 V3D_WRITE(V3D_TFU_COEF1(v3d->ver), job->args.coef[1]);
 354                 V3D_WRITE(V3D_TFU_COEF2(v3d->ver), job->args.coef[2]);
 355                 V3D_WRITE(V3D_TFU_COEF3(v3d->ver), job->args.coef[3]);
 356         }
 357         /* ICFG kicks off the job. */
 358         V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC);
 359
 360         return fence;
 361 }
 362
 363 static struct dma_fence *
 364 v3d_csd_job_run(struct drm_sched_job *sched_job)
 365 {
 366         struct v3d_csd_job *job = to_csd_job(sched_job);
 367         struct v3d_dev *v3d = job->base.v3d;
 368         struct drm_device *dev = &v3d->drm;
 369         struct dma_fence *fence;
 370         int i, csd_cfg0_reg;
 371
 372         v3d->csd_job = job;
 373
 374         v3d_invalidate_caches(v3d);
 375
 376         fence = v3d_fence_create(v3d, V3D_CSD);
 377         if (IS_ERR(fence))
 378                 return NULL;
 379
 380         if (job->base.irq_fence)
 381                 dma_fence_put(job->base.irq_fence);
 382         job->base.irq_fence = dma_fence_get(fence);
 383
 384         trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 385
 386         v3d_job_start_stats(&job->base, V3D_CSD);
 387         v3d_switch_perfmon(v3d, &job->base);
 388
 389         csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);
 390         for (i = 1; i <= 6; i++)
 391                 V3D_CORE_WRITE(0, csd_cfg0_reg + 4 * i, job->args.cfg[i]);
 392
 393         /* Although V3D 7.1 has an eighth configuration register, we are not
 394          * using it. Therefore, make sure it remains unused.
 395          *
 396          * XXX: Set the CFG7 register
 397          */
 398         if (v3d->ver >= 71)
 399                 V3D_CORE_WRITE(0, V3D_V7_CSD_QUEUED_CFG7, 0);
 400
 401         /* CFG0 write kicks off the job. */
 402         V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]);
 403
 404         return fence;
 405 }
 406
 407 static void
 408 v3d_rewrite_csd_job_wg_counts_from_indirect(struct v3d_cpu_job *job)
 409 {
 410         struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;
 411         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 412         struct v3d_bo *indirect = to_v3d_bo(indirect_csd->indirect);
 413         struct drm_v3d_submit_csd *args = &indirect_csd->job->args;
 414         u32 *wg_counts;
 415
 416         v3d_get_bo_vaddr(bo);
 417         v3d_get_bo_vaddr(indirect);
 418
 419         wg_counts = (uint32_t *)(bo->vaddr + indirect_csd->offset);
 420
 421         if (wg_counts[0] == 0 || wg_counts[1] == 0 || wg_counts[2] == 0)
 422                 return;
 423
 424         args->cfg[0] = wg_counts[0] << V3D_CSD_CFG012_WG_COUNT_SHIFT;
 425         args->cfg[1] = wg_counts[1] << V3D_CSD_CFG012_WG_COUNT_SHIFT;
 426         args->cfg[2] = wg_counts[2] << V3D_CSD_CFG012_WG_COUNT_SHIFT;
 427         args->cfg[4] = DIV_ROUND_UP(indirect_csd->wg_size, 16) *
 428                        (wg_counts[0] * wg_counts[1] * wg_counts[2]) - 1;
 429
 430         for (int i = 0; i < 3; i++) {
 431                 /* 0xffffffff indicates that the uniform rewrite is not needed */
 432                 if (indirect_csd->wg_uniform_offsets[i] != 0xffffffff) {
 433                         u32 uniform_idx = indirect_csd->wg_uniform_offsets[i];
 434                         ((uint32_t *)indirect->vaddr)[uniform_idx] = wg_counts[i];
 435                 }
 436         }
 437
 438         v3d_put_bo_vaddr(indirect);
 439         v3d_put_bo_vaddr(bo);
 440 }
 441
 442 static void
 443 v3d_timestamp_query(struct v3d_cpu_job *job)
 444 {
 445         struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;
 446         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 447         u8 *value_addr;
 448
 449         v3d_get_bo_vaddr(bo);
 450
 451         for (int i = 0; i < timestamp_query->count; i++) {
 452                 value_addr = ((u8 *)bo->vaddr) + timestamp_query->queries[i].offset;
 453                 *((u64 *)value_addr) = i == 0 ? ktime_get_ns() : 0ull;
 454
 455                 drm_syncobj_replace_fence(timestamp_query->queries[i].syncobj,
 456                                           job->base.done_fence);
 457         }
 458
 459         v3d_put_bo_vaddr(bo);
 460 }
 461
 462 static void
 463 v3d_reset_timestamp_queries(struct v3d_cpu_job *job)
 464 {
 465         struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;
 466         struct v3d_timestamp_query *queries = timestamp_query->queries;
 467         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 468         u8 *value_addr;
 469
 470         v3d_get_bo_vaddr(bo);
 471
 472         for (int i = 0; i < timestamp_query->count; i++) {
 473                 value_addr = ((u8 *)bo->vaddr) + queries[i].offset;
 474                 *((u64 *)value_addr) = 0;
 475
 476                 drm_syncobj_replace_fence(queries[i].syncobj, NULL);
 477         }
 478
 479         v3d_put_bo_vaddr(bo);
 480 }
 481
 482 static void write_to_buffer_32(u32 *dst, unsigned int idx, u32 value)
 483 {
 484         dst[idx] = value;
 485 }
 486
 487 static void write_to_buffer_64(u64 *dst, unsigned int idx, u64 value)
 488 {
 489         dst[idx] = value;
 490 }
 491
 492 static void
 493 write_to_buffer(void *dst, unsigned int idx, bool do_64bit, u64 value)
 494 {
 495         if (do_64bit)
 496                 write_to_buffer_64(dst, idx, value);
 497         else
 498                 write_to_buffer_32(dst, idx, value);
 499 }
 500
 501 static void
 502 v3d_copy_query_results(struct v3d_cpu_job *job)
 503 {
 504         struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query;
 505         struct v3d_timestamp_query *queries = timestamp_query->queries;
 506         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 507         struct v3d_bo *timestamp = to_v3d_bo(job->base.bo[1]);
 508         struct v3d_copy_query_results_info *copy = &job->copy;
 509         struct dma_fence *fence;
 510         u8 *query_addr;
 511         bool available, write_result;
 512         u8 *data;
 513         int i;
 514
 515         v3d_get_bo_vaddr(bo);
 516         v3d_get_bo_vaddr(timestamp);
 517
 518         data = ((u8 *)bo->vaddr) + copy->offset;
 519
 520         for (i = 0; i < timestamp_query->count; i++) {
 521                 fence = drm_syncobj_fence_get(queries[i].syncobj);
 522                 available = fence ? dma_fence_is_signaled(fence) : false;
 523
 524                 write_result = available || copy->do_partial;
 525                 if (write_result) {
 526                         query_addr = ((u8 *)timestamp->vaddr) + queries[i].offset;
 527                         write_to_buffer(data, 0, copy->do_64bit, *((u64 *)query_addr));
 528                 }
 529
 530                 if (copy->availability_bit)
 531                         write_to_buffer(data, 1, copy->do_64bit, available ? 1u : 0u);
 532
 533                 data += copy->stride;
 534
 535                 dma_fence_put(fence);
 536         }
 537
 538         v3d_put_bo_vaddr(timestamp);
 539         v3d_put_bo_vaddr(bo);
 540 }
 541
 542 static void
 543 v3d_reset_performance_queries(struct v3d_cpu_job *job)
 544 {
 545         struct v3d_performance_query_info *performance_query = &job->performance_query;
 546         struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;
 547         struct v3d_dev *v3d = job->base.v3d;
 548         struct v3d_perfmon *perfmon;
 549
 550         for (int i = 0; i < performance_query->count; i++) {
 551                 for (int j = 0; j < performance_query->nperfmons; j++) {
 552                         perfmon = v3d_perfmon_find(v3d_priv,
 553                                                    performance_query->queries[i].kperfmon_ids[j]);
 554                         if (!perfmon) {
 555                                 DRM_DEBUG("Failed to find perfmon.");
 556                                 continue;
 557                         }
 558
 559                         v3d_perfmon_stop(v3d, perfmon, false);
 560
 561                         memset(perfmon->values, 0, perfmon->ncounters * sizeof(u64));
 562
 563                         v3d_perfmon_put(perfmon);
 564                 }
 565
 566                 drm_syncobj_replace_fence(performance_query->queries[i].syncobj, NULL);
 567         }
 568 }
 569
 570 static void
 571 v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data,
 572                                    unsigned int query)
 573 {
 574         struct v3d_performance_query_info *performance_query =
 575                                                 &job->performance_query;
 576         struct v3d_file_priv *v3d_priv = job->base.file->driver_priv;
 577         struct v3d_performance_query *perf_query =
 578                         &performance_query->queries[query];
 579         struct v3d_dev *v3d = job->base.v3d;
 580         unsigned int i, j, offset;
 581
 582         for (i = 0, offset = 0;
 583              i < performance_query->nperfmons;
 584              i++, offset += DRM_V3D_MAX_PERF_COUNTERS) {
 585                 struct v3d_perfmon *perfmon;
 586
 587                 perfmon = v3d_perfmon_find(v3d_priv,
 588                                            perf_query->kperfmon_ids[i]);
 589                 if (!perfmon) {
 590                         DRM_DEBUG("Failed to find perfmon.");
 591                         continue;
 592                 }
 593
 594                 v3d_perfmon_stop(v3d, perfmon, true);
 595
 596                 if (job->copy.do_64bit) {
 597                         for (j = 0; j < perfmon->ncounters; j++)
 598                                 write_to_buffer_64(data, offset + j,
 599                                                    perfmon->values[j]);
 600                 } else {
 601                         for (j = 0; j < perfmon->ncounters; j++)
 602                                 write_to_buffer_32(data, offset + j,
 603                                                    perfmon->values[j]);
 604                 }
 605
 606                 v3d_perfmon_put(perfmon);
 607         }
 608 }
 609
 610 static void
 611 v3d_copy_performance_query(struct v3d_cpu_job *job)
 612 {
 613         struct v3d_performance_query_info *performance_query = &job->performance_query;
 614         struct v3d_copy_query_results_info *copy = &job->copy;
 615         struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]);
 616         struct dma_fence *fence;
 617         bool available, write_result;
 618         u8 *data;
 619
 620         v3d_get_bo_vaddr(bo);
 621
 622         data = ((u8 *)bo->vaddr) + copy->offset;
 623
 624         for (int i = 0; i < performance_query->count; i++) {
 625                 fence = drm_syncobj_fence_get(performance_query->queries[i].syncobj);
 626                 available = fence ? dma_fence_is_signaled(fence) : false;
 627
 628                 write_result = available || copy->do_partial;
 629                 if (write_result)
 630                         v3d_write_performance_query_result(job, data, i);
 631
 632                 if (copy->availability_bit)
 633                         write_to_buffer(data, performance_query->ncounters,
 634                                         copy->do_64bit, available ? 1u : 0u);
 635
 636                 data += copy->stride;
 637
 638                 dma_fence_put(fence);
 639         }
 640
 641         v3d_put_bo_vaddr(bo);
 642 }
 643
 644 static const v3d_cpu_job_fn cpu_job_function[] = {
 645         [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect,
 646         [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query,
 647         [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries,
 648         [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = v3d_copy_query_results,
 649         [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = v3d_reset_performance_queries,
 650         [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = v3d_copy_performance_query,
 651 };
 652
 653 static struct dma_fence *
 654 v3d_cpu_job_run(struct drm_sched_job *sched_job)
 655 {
 656         struct v3d_cpu_job *job = to_cpu_job(sched_job);
 657         struct v3d_dev *v3d = job->base.v3d;
 658
 659         v3d->cpu_job = job;
 660
 661         if (job->job_type >= ARRAY_SIZE(cpu_job_function)) {
 662                 DRM_DEBUG_DRIVER("Unknown CPU job: %d\n", job->job_type);
 663                 return NULL;
 664         }
 665
 666         v3d_job_start_stats(&job->base, V3D_CPU);
 667         trace_v3d_cpu_job_begin(&v3d->drm, job->job_type);
 668
 669         cpu_job_function[job->job_type](job);
 670
 671         trace_v3d_cpu_job_end(&v3d->drm, job->job_type);
 672         v3d_job_update_stats(&job->base, V3D_CPU);
 673
 674         return NULL;
 675 }
 676
 677 static struct dma_fence *
 678 v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 679 {
 680         struct v3d_job *job = to_v3d_job(sched_job);
 681         struct v3d_dev *v3d = job->v3d;
 682
 683         v3d_job_start_stats(job, V3D_CACHE_CLEAN);
 684
 685         v3d_clean_caches(v3d);
 686
 687         v3d_job_update_stats(job, V3D_CACHE_CLEAN);
 688
 689         return NULL;
 690 }
 691
 692 static enum drm_gpu_sched_stat
 693 v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)
 694 {
 695         enum v3d_queue q;
 696
 697         mutex_lock(&v3d->reset_lock);
 698
 699         /* block scheduler */
 700         for (q = 0; q < V3D_MAX_QUEUES; q++)
 701                 drm_sched_stop(&v3d->queue[q].sched, sched_job);
 702
 703         if (sched_job)
 704                 drm_sched_increase_karma(sched_job);
 705
 706         /* get the GPU back into the init state */
 707         v3d_reset(v3d);
 708
 709         for (q = 0; q < V3D_MAX_QUEUES; q++)
 710                 drm_sched_resubmit_jobs(&v3d->queue[q].sched);
 711
 712         /* Unblock schedulers and restart their jobs. */
 713         for (q = 0; q < V3D_MAX_QUEUES; q++) {
 714                 drm_sched_start(&v3d->queue[q].sched, 0);
 715         }
 716
 717         mutex_unlock(&v3d->reset_lock);
 718
 719         return DRM_GPU_SCHED_STAT_NOMINAL;
 720 }
 721
 722 /* If the current address or return address have changed, then the GPU
 723  * has probably made progress and we should delay the reset.  This
 724  * could fail if the GPU got in an infinite loop in the CL, but that
 725  * is pretty unlikely outside of an i-g-t testcase.
 726  */
 727 static enum drm_gpu_sched_stat
 728 v3d_cl_job_timedout(struct drm_sched_job *sched_job, enum v3d_queue q,
 729                     u32 *timedout_ctca, u32 *timedout_ctra)
 730 {
 731         struct v3d_job *job = to_v3d_job(sched_job);
 732         struct v3d_dev *v3d = job->v3d;
 733         u32 ctca = V3D_CORE_READ(0, V3D_CLE_CTNCA(q));
 734         u32 ctra = V3D_CORE_READ(0, V3D_CLE_CTNRA(q));
 735
 736         if (*timedout_ctca != ctca || *timedout_ctra != ctra) {
 737                 *timedout_ctca = ctca;
 738                 *timedout_ctra = ctra;
 739                 return DRM_GPU_SCHED_STAT_NOMINAL;
 740         }
 741
 742         return v3d_gpu_reset_for_timeout(v3d, sched_job);
 743 }
 744
 745 static enum drm_gpu_sched_stat
 746 v3d_bin_job_timedout(struct drm_sched_job *sched_job)
 747 {
 748         struct v3d_bin_job *job = to_bin_job(sched_job);
 749
 750         return v3d_cl_job_timedout(sched_job, V3D_BIN,
 751                                    &job->timedout_ctca, &job->timedout_ctra);
 752 }
 753
 754 static enum drm_gpu_sched_stat
 755 v3d_render_job_timedout(struct drm_sched_job *sched_job)
 756 {
 757         struct v3d_render_job *job = to_render_job(sched_job);
 758
 759         return v3d_cl_job_timedout(sched_job, V3D_RENDER,
 760                                    &job->timedout_ctca, &job->timedout_ctra);
 761 }
 762
 763 static enum drm_gpu_sched_stat
 764 v3d_generic_job_timedout(struct drm_sched_job *sched_job)
 765 {
 766         struct v3d_job *job = to_v3d_job(sched_job);
 767
 768         return v3d_gpu_reset_for_timeout(job->v3d, sched_job);
 769 }
 770
 771 static enum drm_gpu_sched_stat
 772 v3d_csd_job_timedout(struct drm_sched_job *sched_job)
 773 {
 774         struct v3d_csd_job *job = to_csd_job(sched_job);
 775         struct v3d_dev *v3d = job->base.v3d;
 776         u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4(v3d->ver));
 777
 778         /* If we've made progress, skip reset and let the timer get
 779          * rearmed.
 780          */
 781         if (job->timedout_batches != batches) {
 782                 job->timedout_batches = batches;
 783                 return DRM_GPU_SCHED_STAT_NOMINAL;
 784         }
 785
 786         return v3d_gpu_reset_for_timeout(v3d, sched_job);
 787 }
 788
 789 static const struct drm_sched_backend_ops v3d_bin_sched_ops = {
 790         .run_job = v3d_bin_job_run,
 791         .timedout_job = v3d_bin_job_timedout,
 792         .free_job = v3d_sched_job_free,
 793 };
 794
 795 static const struct drm_sched_backend_ops v3d_render_sched_ops = {
 796         .run_job = v3d_render_job_run,
 797         .timedout_job = v3d_render_job_timedout,
 798         .free_job = v3d_sched_job_free,
 799 };
 800
 801 static const struct drm_sched_backend_ops v3d_tfu_sched_ops = {
 802         .run_job = v3d_tfu_job_run,
 803         .timedout_job = v3d_generic_job_timedout,
 804         .free_job = v3d_sched_job_free,
 805 };
 806
 807 static const struct drm_sched_backend_ops v3d_csd_sched_ops = {
 808         .run_job = v3d_csd_job_run,
 809         .timedout_job = v3d_csd_job_timedout,
 810         .free_job = v3d_sched_job_free
 811 };
 812
 813 static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {
 814         .run_job = v3d_cache_clean_job_run,
 815         .timedout_job = v3d_generic_job_timedout,
 816         .free_job = v3d_sched_job_free
 817 };
 818
 819 static const struct drm_sched_backend_ops v3d_cpu_sched_ops = {
 820         .run_job = v3d_cpu_job_run,
 821         .timedout_job = v3d_generic_job_timedout,
 822         .free_job = v3d_cpu_job_free
 823 };
 824
 825 int
 826 v3d_sched_init(struct v3d_dev *v3d)
 827 {
 828         int hw_jobs_limit = 1;
 829         int job_hang_limit = 0;
 830         int hang_limit_ms = 500;
 831         int ret;
 832
 833         ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,
 834                              &v3d_bin_sched_ops, NULL,
 835                              DRM_SCHED_PRIORITY_COUNT,
 836                              hw_jobs_limit, job_hang_limit,
 837                              msecs_to_jiffies(hang_limit_ms), NULL,
 838                              NULL, "v3d_bin", v3d->drm.dev);
 839         if (ret)
 840                 return ret;
 841
 842         ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,
 843                              &v3d_render_sched_ops, NULL,
 844                              DRM_SCHED_PRIORITY_COUNT,
 845                              hw_jobs_limit, job_hang_limit,
 846                              msecs_to_jiffies(hang_limit_ms), NULL,
 847                              NULL, "v3d_render", v3d->drm.dev);
 848         if (ret)
 849                 goto fail;
 850
 851         ret = drm_sched_init(&v3d->queue[V3D_TFU].sched,
 852                              &v3d_tfu_sched_ops, NULL,
 853                              DRM_SCHED_PRIORITY_COUNT,
 854                              hw_jobs_limit, job_hang_limit,
 855                              msecs_to_jiffies(hang_limit_ms), NULL,
 856                              NULL, "v3d_tfu", v3d->drm.dev);
 857         if (ret)
 858                 goto fail;
 859
 860         if (v3d_has_csd(v3d)) {
 861                 ret = drm_sched_init(&v3d->queue[V3D_CSD].sched,
 862                                      &v3d_csd_sched_ops, NULL,
 863                                      DRM_SCHED_PRIORITY_COUNT,
 864                                      hw_jobs_limit, job_hang_limit,
 865                                      msecs_to_jiffies(hang_limit_ms), NULL,
 866                                      NULL, "v3d_csd", v3d->drm.dev);
 867                 if (ret)
 868                         goto fail;
 869
 870                 ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched,
 871                                      &v3d_cache_clean_sched_ops, NULL,
 872                                      DRM_SCHED_PRIORITY_COUNT,
 873                                      hw_jobs_limit, job_hang_limit,
 874                                      msecs_to_jiffies(hang_limit_ms), NULL,
 875                                      NULL, "v3d_cache_clean", v3d->drm.dev);
 876                 if (ret)
 877                         goto fail;
 878         }
 879
 880         ret = drm_sched_init(&v3d->queue[V3D_CPU].sched,
 881                              &v3d_cpu_sched_ops, NULL,
 882                              DRM_SCHED_PRIORITY_COUNT,
 883                              1, job_hang_limit,
 884                              msecs_to_jiffies(hang_limit_ms), NULL,
 885                              NULL, "v3d_cpu", v3d->drm.dev);
 886         if (ret)
 887                 goto fail;
 888
 889         return 0;
 890
 891 fail:
 892         v3d_sched_fini(v3d);
 893         return ret;
 894 }
 895
 896 void
 897 v3d_sched_fini(struct v3d_dev *v3d)
 898 {
 899         enum v3d_queue q;
 900
 901         for (q = 0; q < V3D_MAX_QUEUES; q++) {
 902                 if (v3d->queue[q].sched.ready)
 903                         drm_sched_fini(&v3d->queue[q].sched);
 904         }
 905 }