drivers/gpu/drm/v3d/v3d_irq.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2014-2018 Broadcom */
   3
   4 /**
   5  * DOC: Interrupt management for the V3D engine
   6  *
   7  * When we take a bin, render, TFU done, or CSD done interrupt, we
   8  * need to signal the fence for that job so that the scheduler can
   9  * queue up the next one and unblock any waiters.
  10  *
  11  * When we take the binner out of memory interrupt, we need to
  12  * allocate some new memory and pass it to the binner so that the
  13  * current job can make progress.
  14  */
  15
  16 #include <linux/platform_device.h>
  17 #include <linux/sched/clock.h>
  18
  19 #include "v3d_drv.h"
  20 #include "v3d_regs.h"
  21 #include "v3d_trace.h"
  22
  23 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM |     \
  24                                   V3D_INT_FLDONE |      \
  25                                   V3D_INT_FRDONE |      \
  26                                   V3D_INT_CSDDONE(ver) |        \
  27                                   (ver < 71 ? V3D_INT_GMPV : 0)))
  28
  29 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV |  \
  30                                  V3D_HUB_INT_MMU_PTI |  \
  31                                  V3D_HUB_INT_MMU_CAP |  \
  32                                  V3D_HUB_INT_TFUC |             \
  33                                  (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0)))
  34
  35 static irqreturn_t
  36 v3d_hub_irq(int irq, void *arg);
  37
  38 static void
  39 v3d_overflow_mem_work(struct work_struct *work)
  40 {
  41         struct v3d_dev *v3d =
  42                 container_of(work, struct v3d_dev, overflow_mem_work);
  43         struct drm_device *dev = &v3d->drm;
  44         struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
  45         struct drm_gem_object *obj;
  46         unsigned long irqflags;
  47
  48         if (IS_ERR(bo)) {
  49                 DRM_ERROR("Couldn't allocate binner overflow mem\n");
  50                 return;
  51         }
  52         obj = &bo->base.base;
  53
  54         /* We lost a race, and our work task came in after the bin job
  55          * completed and exited.  This can happen because the HW
  56          * signals OOM before it's fully OOM, so the binner might just
  57          * barely complete.
  58          *
  59          * If we lose the race and our work task comes in after a new
  60          * bin job got scheduled, that's fine.  We'll just give them
  61          * some binner pool anyway.
  62          */
  63         spin_lock_irqsave(&v3d->job_lock, irqflags);
  64         if (!v3d->bin_job) {
  65                 spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  66                 goto out;
  67         }
  68
  69         drm_gem_object_get(obj);
  70         list_add_tail(&bo->unref_head, &v3d->bin_job->render->unref_list);
  71         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  72
  73         V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT);
  74         V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
  75
  76 out:
  77         drm_gem_object_put(obj);
  78 }
  79
  80 static irqreturn_t
  81 v3d_irq(int irq, void *arg)
  82 {
  83         struct v3d_dev *v3d = arg;
  84         u32 intsts;
  85         irqreturn_t status = IRQ_NONE;
  86
  87         intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
  88
  89         /* Acknowledge the interrupts we're handling here. */
  90         V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
  91
  92         if (intsts & V3D_INT_OUTOMEM) {
  93                 /* Note that the OOM status is edge signaled, so the
  94                  * interrupt won't happen again until the we actually
  95                  * add more memory.  Also, as of V3D 4.1, FLDONE won't
  96                  * be reported until any OOM state has been cleared.
  97                  */
  98                 schedule_work(&v3d->overflow_mem_work);
  99                 status = IRQ_HANDLED;
 100         }
 101
 102         if (intsts & V3D_INT_FLDONE) {
 103                 struct v3d_fence *fence =
 104                         to_v3d_fence(v3d->bin_job->base.irq_fence);
 105                 struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
 106                 u64 runtime = local_clock() - file->start_ns[V3D_BIN];
 107
 108                 file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
 109                 file->jobs_sent[V3D_BIN]++;
 110                 v3d->queue[V3D_BIN].jobs_sent++;
 111
 112                 file->start_ns[V3D_BIN] = 0;
 113                 v3d->queue[V3D_BIN].start_ns = 0;
 114
 115                 file->enabled_ns[V3D_BIN] += runtime;
 116                 v3d->queue[V3D_BIN].enabled_ns += runtime;
 117
 118                 trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 119                 dma_fence_signal(&fence->base);
 120                 status = IRQ_HANDLED;
 121         }
 122
 123         if (intsts & V3D_INT_FRDONE) {
 124                 struct v3d_fence *fence =
 125                         to_v3d_fence(v3d->render_job->base.irq_fence);
 126                 struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
 127                 u64 runtime = local_clock() - file->start_ns[V3D_RENDER];
 128
 129                 file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
 130                 file->jobs_sent[V3D_RENDER]++;
 131                 v3d->queue[V3D_RENDER].jobs_sent++;
 132
 133                 file->start_ns[V3D_RENDER] = 0;
 134                 v3d->queue[V3D_RENDER].start_ns = 0;
 135
 136                 file->enabled_ns[V3D_RENDER] += runtime;
 137                 v3d->queue[V3D_RENDER].enabled_ns += runtime;
 138
 139                 trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 140                 dma_fence_signal(&fence->base);
 141                 status = IRQ_HANDLED;
 142         }
 143
 144         if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 145                 struct v3d_fence *fence =
 146                         to_v3d_fence(v3d->csd_job->base.irq_fence);
 147                 struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
 148                 u64 runtime = local_clock() - file->start_ns[V3D_CSD];
 149
 150                 file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
 151                 file->jobs_sent[V3D_CSD]++;
 152                 v3d->queue[V3D_CSD].jobs_sent++;
 153
 154                 file->start_ns[V3D_CSD] = 0;
 155                 v3d->queue[V3D_CSD].start_ns = 0;
 156
 157                 file->enabled_ns[V3D_CSD] += runtime;
 158                 v3d->queue[V3D_CSD].enabled_ns += runtime;
 159
 160                 trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 161                 dma_fence_signal(&fence->base);
 162                 status = IRQ_HANDLED;
 163         }
 164
 165         /* We shouldn't be triggering these if we have GMP in
 166          * always-allowed mode.
 167          */
 168         if (v3d->ver < 71 && (intsts & V3D_INT_GMPV))
 169                 dev_err(v3d->drm.dev, "GMP violation\n");
 170
 171         /* V3D 4.2 wires the hub and core IRQs together, so if we &
 172          * didn't see the common one then check hub for MMU IRQs.
 173          */
 174         if (v3d->single_irq_line && status == IRQ_NONE)
 175                 return v3d_hub_irq(irq, arg);
 176
 177         return status;
 178 }
 179
 180 static irqreturn_t
 181 v3d_hub_irq(int irq, void *arg)
 182 {
 183         struct v3d_dev *v3d = arg;
 184         u32 intsts;
 185         irqreturn_t status = IRQ_NONE;
 186
 187         intsts = V3D_READ(V3D_HUB_INT_STS);
 188
 189         /* Acknowledge the interrupts we're handling here. */
 190         V3D_WRITE(V3D_HUB_INT_CLR, intsts);
 191
 192         if (intsts & V3D_HUB_INT_TFUC) {
 193                 struct v3d_fence *fence =
 194                         to_v3d_fence(v3d->tfu_job->base.irq_fence);
 195                 struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
 196                 u64 runtime = local_clock() - file->start_ns[V3D_TFU];
 197
 198                 file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
 199                 file->jobs_sent[V3D_TFU]++;
 200                 v3d->queue[V3D_TFU].jobs_sent++;
 201
 202                 file->start_ns[V3D_TFU] = 0;
 203                 v3d->queue[V3D_TFU].start_ns = 0;
 204
 205                 file->enabled_ns[V3D_TFU] += runtime;
 206                 v3d->queue[V3D_TFU].enabled_ns += runtime;
 207
 208                 trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 209                 dma_fence_signal(&fence->base);
 210                 status = IRQ_HANDLED;
 211         }
 212
 213         if (intsts & (V3D_HUB_INT_MMU_WRV |
 214                       V3D_HUB_INT_MMU_PTI |
 215                       V3D_HUB_INT_MMU_CAP)) {
 216                 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
 217                 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) <<
 218                                 (v3d->va_width - 32));
 219                 static const char *const v3d41_axi_ids[] = {
 220                         "L2T",
 221                         "PTB",
 222                         "PSE",
 223                         "TLB",
 224                         "CLE",
 225                         "TFU",
 226                         "MMU",
 227                         "GMP",
 228                 };
 229                 const char *client = "?";
 230
 231                 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL));
 232
 233                 if (v3d->ver >= 41) {
 234                         axi_id = axi_id >> 5;
 235                         if (axi_id < ARRAY_SIZE(v3d41_axi_ids))
 236                                 client = v3d41_axi_ids[axi_id];
 237                 }
 238
 239                 dev_err(v3d->drm.dev, "MMU error from client %s (%d) at 0x%llx%s%s%s\n",
 240                         client, axi_id, (long long)vio_addr,
 241                         ((intsts & V3D_HUB_INT_MMU_WRV) ?
 242                          ", write violation" : ""),
 243                         ((intsts & V3D_HUB_INT_MMU_PTI) ?
 244                          ", pte invalid" : ""),
 245                         ((intsts & V3D_HUB_INT_MMU_CAP) ?
 246                          ", cap exceeded" : ""));
 247                 status = IRQ_HANDLED;
 248         }
 249
 250         if (v3d->ver >= 71 && (intsts & V3D_V7_HUB_INT_GMPV)) {
 251                 dev_err(v3d->drm.dev, "GMP Violation\n");
 252                 status = IRQ_HANDLED;
 253         }
 254
 255         return status;
 256 }
 257
 258 int
 259 v3d_irq_init(struct v3d_dev *v3d)
 260 {
 261         int irq1, ret, core;
 262
 263         INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 264
 265         /* Clear any pending interrupts someone might have left around
 266          * for us.
 267          */
 268         for (core = 0; core < v3d->cores; core++)
 269                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 270         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 271
 272         irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 273         if (irq1 == -EPROBE_DEFER)
 274                 return irq1;
 275         if (irq1 > 0) {
 276                 ret = devm_request_irq(v3d->drm.dev, irq1,
 277                                        v3d_irq, IRQF_SHARED,
 278                                        "v3d_core0", v3d);
 279                 if (ret)
 280                         goto fail;
 281                 ret = devm_request_irq(v3d->drm.dev,
 282                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 283                                        v3d_hub_irq, IRQF_SHARED,
 284                                        "v3d_hub", v3d);
 285                 if (ret)
 286                         goto fail;
 287         } else {
 288                 v3d->single_irq_line = true;
 289
 290                 ret = devm_request_irq(v3d->drm.dev,
 291                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 292                                        v3d_irq, IRQF_SHARED,
 293                                        "v3d", v3d);
 294                 if (ret)
 295                         goto fail;
 296         }
 297
 298         v3d_irq_enable(v3d);
 299         return 0;
 300
 301 fail:
 302         if (ret != -EPROBE_DEFER)
 303                 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret);
 304         return ret;
 305 }
 306
 307 void
 308 v3d_irq_enable(struct v3d_dev *v3d)
 309 {
 310         int core;
 311
 312         /* Enable our set of interrupts, masking out any others. */
 313         for (core = 0; core < v3d->cores; core++) {
 314                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
 315                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver));
 316         }
 317
 318         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver));
 319         V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver));
 320 }
 321
 322 void
 323 v3d_irq_disable(struct v3d_dev *v3d)
 324 {
 325         int core;
 326
 327         /* Disable all interrupts. */
 328         for (core = 0; core < v3d->cores; core++)
 329                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
 330         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
 331
 332         /* Clear any pending interrupts we might have left. */
 333         for (core = 0; core < v3d->cores; core++)
 334                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 335         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 336
 337         cancel_work_sync(&v3d->overflow_mem_work);
 338 }
 339
 340 /** Reinitializes interrupt registers when a GPU reset is performed. */
 341 void v3d_irq_reset(struct v3d_dev *v3d)
 342 {
 343         v3d_irq_enable(v3d);
 344 }