drivers/gpu/drm/panfrost/panfrost_job.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright 2019 Linaro, Ltd, Rob Herring <[email protected]> */
   3 /* Copyright 2019 Collabora ltd. */
   4 #include <linux/delay.h>
   5 #include <linux/interrupt.h>
   6 #include <linux/io.h>
   7 #include <linux/iopoll.h>
   8 #include <linux/platform_device.h>
   9 #include <linux/pm_runtime.h>
  10 #include <linux/dma-resv.h>
  11 #include <drm/gpu_scheduler.h>
  12 #include <drm/panfrost_drm.h>
  13
  14 #include "panfrost_device.h"
  15 #include "panfrost_devfreq.h"
  16 #include "panfrost_job.h"
  17 #include "panfrost_features.h"
  18 #include "panfrost_issues.h"
  19 #include "panfrost_gem.h"
  20 #include "panfrost_regs.h"
  21 #include "panfrost_gpu.h"
  22 #include "panfrost_mmu.h"
  23 #include "panfrost_dump.h"
  24
  25 #define JOB_TIMEOUT_MS 500
  26
  27 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
  28 #define job_read(dev, reg) readl(dev->iomem + (reg))
  29
  30 struct panfrost_queue_state {
  31         struct drm_gpu_scheduler sched;
  32         u64 fence_context;
  33         u64 emit_seqno;
  34 };
  35
  36 struct panfrost_job_slot {
  37         struct panfrost_queue_state queue[NUM_JOB_SLOTS];
  38         spinlock_t job_lock;
  39         int irq;
  40 };
  41
  42 static struct panfrost_job *
  43 to_panfrost_job(struct drm_sched_job *sched_job)
  44 {
  45         return container_of(sched_job, struct panfrost_job, base);
  46 }
  47
  48 struct panfrost_fence {
  49         struct dma_fence base;
  50         struct drm_device *dev;
  51         /* panfrost seqno for signaled() test */
  52         u64 seqno;
  53         int queue;
  54 };
  55
  56 static inline struct panfrost_fence *
  57 to_panfrost_fence(struct dma_fence *fence)
  58 {
  59         return (struct panfrost_fence *)fence;
  60 }
  61
  62 static const char *panfrost_fence_get_driver_name(struct dma_fence *fence)
  63 {
  64         return "panfrost";
  65 }
  66
  67 static const char *panfrost_fence_get_timeline_name(struct dma_fence *fence)
  68 {
  69         struct panfrost_fence *f = to_panfrost_fence(fence);
  70
  71         switch (f->queue) {
  72         case 0:
  73                 return "panfrost-js-0";
  74         case 1:
  75                 return "panfrost-js-1";
  76         case 2:
  77                 return "panfrost-js-2";
  78         default:
  79                 return NULL;
  80         }
  81 }
  82
  83 static const struct dma_fence_ops panfrost_fence_ops = {
  84         .get_driver_name = panfrost_fence_get_driver_name,
  85         .get_timeline_name = panfrost_fence_get_timeline_name,
  86 };
  87
  88 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)
  89 {
  90         struct panfrost_fence *fence;
  91         struct panfrost_job_slot *js = pfdev->js;
  92
  93         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
  94         if (!fence)
  95                 return ERR_PTR(-ENOMEM);
  96
  97         fence->dev = pfdev->ddev;
  98         fence->queue = js_num;
  99         fence->seqno = ++js->queue[js_num].emit_seqno;
 100         dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,
 101                        js->queue[js_num].fence_context, fence->seqno);
 102
 103         return &fence->base;
 104 }
 105
 106 int panfrost_job_get_slot(struct panfrost_job *job)
 107 {
 108         /* JS0: fragment jobs.
 109          * JS1: vertex/tiler jobs
 110          * JS2: compute jobs
 111          */
 112         if (job->requirements & PANFROST_JD_REQ_FS)
 113                 return 0;
 114
 115 /* Not exposed to userspace yet */
 116 #if 0
 117         if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {
 118                 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&
 119                     (job->pfdev->features.nr_core_groups == 2))
 120                         return 2;
 121                 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
 122                         return 2;
 123         }
 124 #endif
 125         return 1;
 126 }
 127
 128 static void panfrost_job_write_affinity(struct panfrost_device *pfdev,
 129                                         u32 requirements,
 130                                         int js)
 131 {
 132         u64 affinity;
 133
 134         /*
 135          * Use all cores for now.
 136          * Eventually we may need to support tiler only jobs and h/w with
 137          * multiple (2) coherent core groups
 138          */
 139         affinity = pfdev->features.shader_present;
 140
 141         job_write(pfdev, JS_AFFINITY_NEXT_LO(js), lower_32_bits(affinity));
 142         job_write(pfdev, JS_AFFINITY_NEXT_HI(js), upper_32_bits(affinity));
 143 }
 144
 145 static u32
 146 panfrost_get_job_chain_flag(const struct panfrost_job *job)
 147 {
 148         struct panfrost_fence *f = to_panfrost_fence(job->done_fence);
 149
 150         if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
 151                 return 0;
 152
 153         return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0;
 154 }
 155
 156 static struct panfrost_job *
 157 panfrost_dequeue_job(struct panfrost_device *pfdev, int slot)
 158 {
 159         struct panfrost_job *job = pfdev->jobs[slot][0];
 160
 161         WARN_ON(!job);
 162         pfdev->jobs[slot][0] = pfdev->jobs[slot][1];
 163         pfdev->jobs[slot][1] = NULL;
 164
 165         return job;
 166 }
 167
 168 static unsigned int
 169 panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,
 170                      struct panfrost_job *job)
 171 {
 172         if (WARN_ON(!job))
 173                 return 0;
 174
 175         if (!pfdev->jobs[slot][0]) {
 176                 pfdev->jobs[slot][0] = job;
 177                 return 0;
 178         }
 179
 180         WARN_ON(pfdev->jobs[slot][1]);
 181         pfdev->jobs[slot][1] = job;
 182         WARN_ON(panfrost_get_job_chain_flag(job) ==
 183                 panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));
 184         return 1;
 185 }
 186
 187 static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
 188 {
 189         struct panfrost_device *pfdev = job->pfdev;
 190         unsigned int subslot;
 191         u32 cfg;
 192         u64 jc_head = job->jc;
 193         int ret;
 194
 195         panfrost_devfreq_record_busy(&pfdev->pfdevfreq);
 196
 197         ret = pm_runtime_get_sync(pfdev->dev);
 198         if (ret < 0)
 199                 return;
 200
 201         if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) {
 202                 return;
 203         }
 204
 205         cfg = panfrost_mmu_as_get(pfdev, job->mmu);
 206
 207         job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head));
 208         job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head));
 209
 210         panfrost_job_write_affinity(pfdev, job->requirements, js);
 211
 212         /* start MMU, medium priority, cache clean/flush on end, clean/flush on
 213          * start */
 214         cfg |= JS_CONFIG_THREAD_PRI(8) |
 215                 JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |
 216                 JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE |
 217                 panfrost_get_job_chain_flag(job);
 218
 219         if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
 220                 cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
 221
 222         if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))
 223                 cfg |= JS_CONFIG_START_MMU;
 224
 225         job_write(pfdev, JS_CONFIG_NEXT(js), cfg);
 226
 227         if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
 228                 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
 229
 230         /* GO ! */
 231
 232         spin_lock(&pfdev->js->job_lock);
 233         subslot = panfrost_enqueue_job(pfdev, js, job);
 234         /* Don't queue the job if a reset is in progress */
 235         if (!atomic_read(&pfdev->reset.pending)) {
 236                 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
 237                 dev_dbg(pfdev->dev,
 238                         "JS: Submitting atom %p to js[%d][%d] with head=0x%llx AS %d",
 239                         job, js, subslot, jc_head, cfg & 0xf);
 240         }
 241         spin_unlock(&pfdev->js->job_lock);
 242 }
 243
 244 static int panfrost_acquire_object_fences(struct drm_gem_object **bos,
 245                                           int bo_count,
 246                                           struct drm_sched_job *job)
 247 {
 248         int i, ret;
 249
 250         for (i = 0; i < bo_count; i++) {
 251                 ret = dma_resv_reserve_fences(bos[i]->resv, 1);
 252                 if (ret)
 253                         return ret;
 254
 255                 /* panfrost always uses write mode in its current uapi */
 256                 ret = drm_sched_job_add_implicit_dependencies(job, bos[i],
 257                                                               true);
 258                 if (ret)
 259                         return ret;
 260         }
 261
 262         return 0;
 263 }
 264
 265 static void panfrost_attach_object_fences(struct drm_gem_object **bos,
 266                                           int bo_count,
 267                                           struct dma_fence *fence)
 268 {
 269         int i;
 270
 271         for (i = 0; i < bo_count; i++)
 272                 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE);
 273 }
 274
 275 int panfrost_job_push(struct panfrost_job *job)
 276 {
 277         struct panfrost_device *pfdev = job->pfdev;
 278         struct ww_acquire_ctx acquire_ctx;
 279         int ret = 0;
 280
 281         ret = drm_gem_lock_reservations(job->bos, job->bo_count,
 282                                             &acquire_ctx);
 283         if (ret)
 284                 return ret;
 285
 286         mutex_lock(&pfdev->sched_lock);
 287         drm_sched_job_arm(&job->base);
 288
 289         job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);
 290
 291         ret = panfrost_acquire_object_fences(job->bos, job->bo_count,
 292                                              &job->base);
 293         if (ret) {
 294                 mutex_unlock(&pfdev->sched_lock);
 295                 goto unlock;
 296         }
 297
 298         kref_get(&job->refcount); /* put by scheduler job completion */
 299
 300         drm_sched_entity_push_job(&job->base);
 301
 302         mutex_unlock(&pfdev->sched_lock);
 303
 304         panfrost_attach_object_fences(job->bos, job->bo_count,
 305                                       job->render_done_fence);
 306
 307 unlock:
 308         drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);
 309
 310         return ret;
 311 }
 312
 313 static void panfrost_job_cleanup(struct kref *ref)
 314 {
 315         struct panfrost_job *job = container_of(ref, struct panfrost_job,
 316                                                 refcount);
 317         unsigned int i;
 318
 319         dma_fence_put(job->done_fence);
 320         dma_fence_put(job->render_done_fence);
 321
 322         if (job->mappings) {
 323                 for (i = 0; i < job->bo_count; i++) {
 324                         if (!job->mappings[i])
 325                                 break;
 326
 327                         atomic_dec(&job->mappings[i]->obj->gpu_usecount);
 328                         panfrost_gem_mapping_put(job->mappings[i]);
 329                 }
 330                 kvfree(job->mappings);
 331         }
 332
 333         if (job->bos) {
 334                 for (i = 0; i < job->bo_count; i++)
 335                         drm_gem_object_put(job->bos[i]);
 336
 337                 kvfree(job->bos);
 338         }
 339
 340         kfree(job);
 341 }
 342
 343 void panfrost_job_put(struct panfrost_job *job)
 344 {
 345         kref_put(&job->refcount, panfrost_job_cleanup);
 346 }
 347
 348 static void panfrost_job_free(struct drm_sched_job *sched_job)
 349 {
 350         struct panfrost_job *job = to_panfrost_job(sched_job);
 351
 352         drm_sched_job_cleanup(sched_job);
 353
 354         panfrost_job_put(job);
 355 }
 356
 357 static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job)
 358 {
 359         struct panfrost_job *job = to_panfrost_job(sched_job);
 360         struct panfrost_device *pfdev = job->pfdev;
 361         int slot = panfrost_job_get_slot(job);
 362         struct dma_fence *fence = NULL;
 363
 364         if (unlikely(job->base.s_fence->finished.error))
 365                 return NULL;
 366
 367         /* Nothing to execute: can happen if the job has finished while
 368          * we were resetting the GPU.
 369          */
 370         if (!job->jc)
 371                 return NULL;
 372
 373         fence = panfrost_fence_create(pfdev, slot);
 374         if (IS_ERR(fence))
 375                 return fence;
 376
 377         if (job->done_fence)
 378                 dma_fence_put(job->done_fence);
 379         job->done_fence = dma_fence_get(fence);
 380
 381         panfrost_job_hw_submit(job, slot);
 382
 383         return fence;
 384 }
 385
 386 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
 387 {
 388         int j;
 389         u32 irq_mask = 0;
 390
 391         for (j = 0; j < NUM_JOB_SLOTS; j++) {
 392                 irq_mask |= MK_JS_MASK(j);
 393         }
 394
 395         job_write(pfdev, JOB_INT_CLEAR, irq_mask);
 396         job_write(pfdev, JOB_INT_MASK, irq_mask);
 397 }
 398
 399 static void panfrost_job_handle_err(struct panfrost_device *pfdev,
 400                                     struct panfrost_job *job,
 401                                     unsigned int js)
 402 {
 403         u32 js_status = job_read(pfdev, JS_STATUS(js));
 404         const char *exception_name = panfrost_exception_name(js_status);
 405         bool signal_fence = true;
 406
 407         if (!panfrost_exception_is_fault(js_status)) {
 408                 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",
 409                         js, exception_name,
 410                         job_read(pfdev, JS_HEAD_LO(js)),
 411                         job_read(pfdev, JS_TAIL_LO(js)));
 412         } else {
 413                 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
 414                         js, exception_name,
 415                         job_read(pfdev, JS_HEAD_LO(js)),
 416                         job_read(pfdev, JS_TAIL_LO(js)));
 417         }
 418
 419         if (js_status == DRM_PANFROST_EXCEPTION_STOPPED) {
 420                 /* Update the job head so we can resume */
 421                 job->jc = job_read(pfdev, JS_TAIL_LO(js)) |
 422                           ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);
 423
 424                 /* The job will be resumed, don't signal the fence */
 425                 signal_fence = false;
 426         } else if (js_status == DRM_PANFROST_EXCEPTION_TERMINATED) {
 427                 /* Job has been hard-stopped, flag it as canceled */
 428                 dma_fence_set_error(job->done_fence, -ECANCELED);
 429                 job->jc = 0;
 430         } else if (panfrost_exception_is_fault(js_status)) {
 431                 /* We might want to provide finer-grained error code based on
 432                  * the exception type, but unconditionally setting to EINVAL
 433                  * is good enough for now.
 434                  */
 435                 dma_fence_set_error(job->done_fence, -EINVAL);
 436                 job->jc = 0;
 437         }
 438
 439         panfrost_mmu_as_put(pfdev, job->mmu);
 440         panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
 441
 442         if (signal_fence)
 443                 dma_fence_signal_locked(job->done_fence);
 444
 445         pm_runtime_put_autosuspend(pfdev->dev);
 446
 447         if (panfrost_exception_needs_reset(pfdev, js_status)) {
 448                 atomic_set(&pfdev->reset.pending, 1);
 449                 drm_sched_fault(&pfdev->js->queue[js].sched);
 450         }
 451 }
 452
 453 static void panfrost_job_handle_done(struct panfrost_device *pfdev,
 454                                      struct panfrost_job *job)
 455 {
 456         /* Set ->jc to 0 to avoid re-submitting an already finished job (can
 457          * happen when we receive the DONE interrupt while doing a GPU reset).
 458          */
 459         job->jc = 0;
 460         panfrost_mmu_as_put(pfdev, job->mmu);
 461         panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
 462
 463         dma_fence_signal_locked(job->done_fence);
 464         pm_runtime_put_autosuspend(pfdev->dev);
 465 }
 466
 467 static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status)
 468 {
 469         struct panfrost_job *done[NUM_JOB_SLOTS][2] = {};
 470         struct panfrost_job *failed[NUM_JOB_SLOTS] = {};
 471         u32 js_state = 0, js_events = 0;
 472         unsigned int i, j;
 473
 474         /* First we collect all failed/done jobs. */
 475         while (status) {
 476                 u32 js_state_mask = 0;
 477
 478                 for (j = 0; j < NUM_JOB_SLOTS; j++) {
 479                         if (status & MK_JS_MASK(j))
 480                                 js_state_mask |= MK_JS_MASK(j);
 481
 482                         if (status & JOB_INT_MASK_DONE(j)) {
 483                                 if (done[j][0])
 484                                         done[j][1] = panfrost_dequeue_job(pfdev, j);
 485                                 else
 486                                         done[j][0] = panfrost_dequeue_job(pfdev, j);
 487                         }
 488
 489                         if (status & JOB_INT_MASK_ERR(j)) {
 490                                 /* Cancel the next submission. Will be submitted
 491                                  * after we're done handling this failure if
 492                                  * there's no reset pending.
 493                                  */
 494                                 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
 495                                 failed[j] = panfrost_dequeue_job(pfdev, j);
 496                         }
 497                 }
 498
 499                 /* JS_STATE is sampled when JOB_INT_CLEAR is written.
 500                  * For each BIT(slot) or BIT(slot + 16) bit written to
 501                  * JOB_INT_CLEAR, the corresponding bits in JS_STATE
 502                  * (BIT(slot) and BIT(slot + 16)) are updated, but this
 503                  * is racy. If we only have one job done at the time we
 504                  * read JOB_INT_RAWSTAT but the second job fails before we
 505                  * clear the status, we end up with a status containing
 506                  * only the DONE bit and consider both jobs as DONE since
 507                  * JS_STATE reports both NEXT and CURRENT as inactive.
 508                  * To prevent that, let's repeat this clear+read steps
 509                  * until status is 0.
 510                  */
 511                 job_write(pfdev, JOB_INT_CLEAR, status);
 512                 js_state &= ~js_state_mask;
 513                 js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;
 514                 js_events |= status;
 515                 status = job_read(pfdev, JOB_INT_RAWSTAT);
 516         }
 517
 518         /* Then we handle the dequeued jobs. */
 519         for (j = 0; j < NUM_JOB_SLOTS; j++) {
 520                 if (!(js_events & MK_JS_MASK(j)))
 521                         continue;
 522
 523                 if (failed[j]) {
 524                         panfrost_job_handle_err(pfdev, failed[j], j);
 525                 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {
 526                         /* When the current job doesn't fail, the JM dequeues
 527                          * the next job without waiting for an ACK, this means
 528                          * we can have 2 jobs dequeued and only catch the
 529                          * interrupt when the second one is done. If both slots
 530                          * are inactive, but one job remains in pfdev->jobs[j],
 531                          * consider it done. Of course that doesn't apply if a
 532                          * failure happened since we cancelled execution of the
 533                          * job in _NEXT (see above).
 534                          */
 535                         if (WARN_ON(!done[j][0]))
 536                                 done[j][0] = panfrost_dequeue_job(pfdev, j);
 537                         else
 538                                 done[j][1] = panfrost_dequeue_job(pfdev, j);
 539                 }
 540
 541                 for (i = 0; i < ARRAY_SIZE(done[0]) && done[j][i]; i++)
 542                         panfrost_job_handle_done(pfdev, done[j][i]);
 543         }
 544
 545         /* And finally we requeue jobs that were waiting in the second slot
 546          * and have been stopped if we detected a failure on the first slot.
 547          */
 548         for (j = 0; j < NUM_JOB_SLOTS; j++) {
 549                 if (!(js_events & MK_JS_MASK(j)))
 550                         continue;
 551
 552                 if (!failed[j] || !pfdev->jobs[j][0])
 553                         continue;
 554
 555                 if (pfdev->jobs[j][0]->jc == 0) {
 556                         /* The job was cancelled, signal the fence now */
 557                         struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);
 558
 559                         dma_fence_set_error(canceled->done_fence, -ECANCELED);
 560                         panfrost_job_handle_done(pfdev, canceled);
 561                 } else if (!atomic_read(&pfdev->reset.pending)) {
 562                         /* Requeue the job we removed if no reset is pending */
 563                         job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);
 564                 }
 565         }
 566 }
 567
 568 static void panfrost_job_handle_irqs(struct panfrost_device *pfdev)
 569 {
 570         u32 status = job_read(pfdev, JOB_INT_RAWSTAT);
 571
 572         while (status) {
 573                 pm_runtime_mark_last_busy(pfdev->dev);
 574
 575                 spin_lock(&pfdev->js->job_lock);
 576                 panfrost_job_handle_irq(pfdev, status);
 577                 spin_unlock(&pfdev->js->job_lock);
 578                 status = job_read(pfdev, JOB_INT_RAWSTAT);
 579         }
 580 }
 581
 582 static u32 panfrost_active_slots(struct panfrost_device *pfdev,
 583                                  u32 *js_state_mask, u32 js_state)
 584 {
 585         u32 rawstat;
 586
 587         if (!(js_state & *js_state_mask))
 588                 return 0;
 589
 590         rawstat = job_read(pfdev, JOB_INT_RAWSTAT);
 591         if (rawstat) {
 592                 unsigned int i;
 593
 594                 for (i = 0; i < NUM_JOB_SLOTS; i++) {
 595                         if (rawstat & MK_JS_MASK(i))
 596                                 *js_state_mask &= ~MK_JS_MASK(i);
 597                 }
 598         }
 599
 600         return js_state & *js_state_mask;
 601 }
 602
 603 static void
 604 panfrost_reset(struct panfrost_device *pfdev,
 605                struct drm_sched_job *bad)
 606 {
 607         u32 js_state, js_state_mask = 0xffffffff;
 608         unsigned int i, j;
 609         bool cookie;
 610         int ret;
 611
 612         if (!atomic_read(&pfdev->reset.pending))
 613                 return;
 614
 615         /* Stop the schedulers.
 616          *
 617          * FIXME: We temporarily get out of the dma_fence_signalling section
 618          * because the cleanup path generate lockdep splats when taking locks
 619          * to release job resources. We should rework the code to follow this
 620          * pattern:
 621          *
 622          *      try_lock
 623          *      if (locked)
 624          *              release
 625          *      else
 626          *              schedule_work_to_release_later
 627          */
 628         for (i = 0; i < NUM_JOB_SLOTS; i++)
 629                 drm_sched_stop(&pfdev->js->queue[i].sched, bad);
 630
 631         cookie = dma_fence_begin_signalling();
 632
 633         if (bad)
 634                 drm_sched_increase_karma(bad);
 635
 636         /* Mask job interrupts and synchronize to make sure we won't be
 637          * interrupted during our reset.
 638          */
 639         job_write(pfdev, JOB_INT_MASK, 0);
 640         synchronize_irq(pfdev->js->irq);
 641
 642         for (i = 0; i < NUM_JOB_SLOTS; i++) {
 643                 /* Cancel the next job and soft-stop the running job. */
 644                 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
 645                 job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);
 646         }
 647
 648         /* Wait at most 10ms for soft-stops to complete */
 649         ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,
 650                                  !panfrost_active_slots(pfdev, &js_state_mask, js_state),
 651                                  10, 10000);
 652
 653         if (ret)
 654                 dev_err(pfdev->dev, "Soft-stop failed\n");
 655
 656         /* Handle the remaining interrupts before we reset. */
 657         panfrost_job_handle_irqs(pfdev);
 658
 659         /* Remaining interrupts have been handled, but we might still have
 660          * stuck jobs. Let's make sure the PM counters stay balanced by
 661          * manually calling pm_runtime_put_noidle() and
 662          * panfrost_devfreq_record_idle() for each stuck job.
 663          */
 664         spin_lock(&pfdev->js->job_lock);
 665         for (i = 0; i < NUM_JOB_SLOTS; i++) {
 666                 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {
 667                         pm_runtime_put_noidle(pfdev->dev);
 668                         panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
 669                 }
 670         }
 671         memset(pfdev->jobs, 0, sizeof(pfdev->jobs));
 672         spin_unlock(&pfdev->js->job_lock);
 673
 674         /* Proceed with reset now. */
 675         panfrost_device_reset(pfdev);
 676
 677         /* panfrost_device_reset() unmasks job interrupts, but we want to
 678          * keep them masked a bit longer.
 679          */
 680         job_write(pfdev, JOB_INT_MASK, 0);
 681
 682         /* GPU has been reset, we can clear the reset pending bit. */
 683         atomic_set(&pfdev->reset.pending, 0);
 684
 685         /* Now resubmit jobs that were previously queued but didn't have a
 686          * chance to finish.
 687          * FIXME: We temporarily get out of the DMA fence signalling section
 688          * while resubmitting jobs because the job submission logic will
 689          * allocate memory with the GFP_KERNEL flag which can trigger memory
 690          * reclaim and exposes a lock ordering issue.
 691          */
 692         dma_fence_end_signalling(cookie);
 693         for (i = 0; i < NUM_JOB_SLOTS; i++)
 694                 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
 695         cookie = dma_fence_begin_signalling();
 696
 697         /* Restart the schedulers */
 698         for (i = 0; i < NUM_JOB_SLOTS; i++)
 699                 drm_sched_start(&pfdev->js->queue[i].sched, true);
 700
 701         /* Re-enable job interrupts now that everything has been restarted. */
 702         job_write(pfdev, JOB_INT_MASK,
 703                   GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
 704                   GENMASK(NUM_JOB_SLOTS - 1, 0));
 705
 706         dma_fence_end_signalling(cookie);
 707 }
 708
 709 static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job
 710                                                      *sched_job)
 711 {
 712         struct panfrost_job *job = to_panfrost_job(sched_job);
 713         struct panfrost_device *pfdev = job->pfdev;
 714         int js = panfrost_job_get_slot(job);
 715
 716         /*
 717          * If the GPU managed to complete this jobs fence, the timeout is
 718          * spurious. Bail out.
 719          */
 720         if (dma_fence_is_signaled(job->done_fence))
 721                 return DRM_GPU_SCHED_STAT_NOMINAL;
 722
 723         dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
 724                 js,
 725                 job_read(pfdev, JS_CONFIG(js)),
 726                 job_read(pfdev, JS_STATUS(js)),
 727                 job_read(pfdev, JS_HEAD_LO(js)),
 728                 job_read(pfdev, JS_TAIL_LO(js)),
 729                 sched_job);
 730
 731         panfrost_core_dump(job);
 732
 733         atomic_set(&pfdev->reset.pending, 1);
 734         panfrost_reset(pfdev, sched_job);
 735
 736         return DRM_GPU_SCHED_STAT_NOMINAL;
 737 }
 738
 739 static void panfrost_reset_work(struct work_struct *work)
 740 {
 741         struct panfrost_device *pfdev;
 742
 743         pfdev = container_of(work, struct panfrost_device, reset.work);
 744         panfrost_reset(pfdev, NULL);
 745 }
 746
 747 static const struct drm_sched_backend_ops panfrost_sched_ops = {
 748         .run_job = panfrost_job_run,
 749         .timedout_job = panfrost_job_timedout,
 750         .free_job = panfrost_job_free
 751 };
 752
 753 static irqreturn_t panfrost_job_irq_handler_thread(int irq, void *data)
 754 {
 755         struct panfrost_device *pfdev = data;
 756
 757         panfrost_job_handle_irqs(pfdev);
 758         job_write(pfdev, JOB_INT_MASK,
 759                   GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
 760                   GENMASK(NUM_JOB_SLOTS - 1, 0));
 761         return IRQ_HANDLED;
 762 }
 763
 764 static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
 765 {
 766         struct panfrost_device *pfdev = data;
 767         u32 status = job_read(pfdev, JOB_INT_STAT);
 768
 769         if (!status)
 770                 return IRQ_NONE;
 771
 772         job_write(pfdev, JOB_INT_MASK, 0);
 773         return IRQ_WAKE_THREAD;
 774 }
 775
 776 int panfrost_job_init(struct panfrost_device *pfdev)
 777 {
 778         struct panfrost_job_slot *js;
 779         unsigned int nentries = 2;
 780         int ret, j;
 781
 782         /* All GPUs have two entries per queue, but without jobchain
 783          * disambiguation stopping the right job in the close path is tricky,
 784          * so let's just advertise one entry in that case.
 785          */
 786         if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
 787                 nentries = 1;
 788
 789         pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
 790         if (!js)
 791                 return -ENOMEM;
 792
 793         INIT_WORK(&pfdev->reset.work, panfrost_reset_work);
 794         spin_lock_init(&js->job_lock);
 795
 796         js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
 797         if (js->irq <= 0)
 798                 return -ENODEV;
 799
 800         ret = devm_request_threaded_irq(pfdev->dev, js->irq,
 801                                         panfrost_job_irq_handler,
 802                                         panfrost_job_irq_handler_thread,
 803                                         IRQF_SHARED, KBUILD_MODNAME "-job",
 804                                         pfdev);
 805         if (ret) {
 806                 dev_err(pfdev->dev, "failed to request job irq");
 807                 return ret;
 808         }
 809
 810         pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);
 811         if (!pfdev->reset.wq)
 812                 return -ENOMEM;
 813
 814         for (j = 0; j < NUM_JOB_SLOTS; j++) {
 815                 js->queue[j].fence_context = dma_fence_context_alloc(1);
 816
 817                 ret = drm_sched_init(&js->queue[j].sched,
 818                                      &panfrost_sched_ops,
 819                                      nentries, 0,
 820                                      msecs_to_jiffies(JOB_TIMEOUT_MS),
 821                                      pfdev->reset.wq,
 822                                      NULL, "pan_js", pfdev->dev);
 823                 if (ret) {
 824                         dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
 825                         goto err_sched;
 826                 }
 827         }
 828
 829         panfrost_job_enable_interrupts(pfdev);
 830
 831         return 0;
 832
 833 err_sched:
 834         for (j--; j >= 0; j--)
 835                 drm_sched_fini(&js->queue[j].sched);
 836
 837         destroy_workqueue(pfdev->reset.wq);
 838         return ret;
 839 }
 840
 841 void panfrost_job_fini(struct panfrost_device *pfdev)
 842 {
 843         struct panfrost_job_slot *js = pfdev->js;
 844         int j;
 845
 846         job_write(pfdev, JOB_INT_MASK, 0);
 847
 848         for (j = 0; j < NUM_JOB_SLOTS; j++) {
 849                 drm_sched_fini(&js->queue[j].sched);
 850         }
 851
 852         cancel_work_sync(&pfdev->reset.work);
 853         destroy_workqueue(pfdev->reset.wq);
 854 }
 855
 856 int panfrost_job_open(struct panfrost_file_priv *panfrost_priv)
 857 {
 858         struct panfrost_device *pfdev = panfrost_priv->pfdev;
 859         struct panfrost_job_slot *js = pfdev->js;
 860         struct drm_gpu_scheduler *sched;
 861         int ret, i;
 862
 863         for (i = 0; i < NUM_JOB_SLOTS; i++) {
 864                 sched = &js->queue[i].sched;
 865                 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i],
 866                                             DRM_SCHED_PRIORITY_NORMAL, &sched,
 867                                             1, NULL);
 868                 if (WARN_ON(ret))
 869                         return ret;
 870         }
 871         return 0;
 872 }
 873
 874 void panfrost_job_close(struct panfrost_file_priv *panfrost_priv)
 875 {
 876         struct panfrost_device *pfdev = panfrost_priv->pfdev;
 877         int i;
 878
 879         for (i = 0; i < NUM_JOB_SLOTS; i++)
 880                 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);
 881
 882         /* Kill in-flight jobs */
 883         spin_lock(&pfdev->js->job_lock);
 884         for (i = 0; i < NUM_JOB_SLOTS; i++) {
 885                 struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i];
 886                 int j;
 887
 888                 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {
 889                         struct panfrost_job *job = pfdev->jobs[i][j];
 890                         u32 cmd;
 891
 892                         if (!job || job->base.entity != entity)
 893                                 continue;
 894
 895                         if (j == 1) {
 896                                 /* Try to cancel the job before it starts */
 897                                 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);
 898                                 /* Reset the job head so it doesn't get restarted if
 899                                  * the job in the first slot failed.
 900                                  */
 901                                 job->jc = 0;
 902                         }
 903
 904                         if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
 905                                 cmd = panfrost_get_job_chain_flag(job) ?
 906                                       JS_COMMAND_HARD_STOP_1 :
 907                                       JS_COMMAND_HARD_STOP_0;
 908                         } else {
 909                                 cmd = JS_COMMAND_HARD_STOP;
 910                         }
 911
 912                         job_write(pfdev, JS_COMMAND(i), cmd);
 913                 }
 914         }
 915         spin_unlock(&pfdev->js->job_lock);
 916 }
 917
 918 int panfrost_job_is_idle(struct panfrost_device *pfdev)
 919 {
 920         struct panfrost_job_slot *js = pfdev->js;
 921         int i;
 922
 923         for (i = 0; i < NUM_JOB_SLOTS; i++) {
 924                 /* If there are any jobs in the HW queue, we're not idle */
 925                 if (atomic_read(&js->queue[i].sched.hw_rq_count))
 926                         return false;
 927         }
 928
 929         return true;
 930 }