drivers/gpu/drm/v3d/v3d_irq.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2014-2018 Broadcom */
   3
   4 /**
   5  * DOC: Interrupt management for the V3D engine
   6  *
   7  * When we take a bin, render, TFU done, or CSD done interrupt, we
   8  * need to signal the fence for that job so that the scheduler can
   9  * queue up the next one and unblock any waiters.
  10  *
  11  * When we take the binner out of memory interrupt, we need to
  12  * allocate some new memory and pass it to the binner so that the
  13  * current job can make progress.
  14  */
  15
  16 #include <linux/platform_device.h>
  17 #include <linux/sched/clock.h>
  18
  19 #include "v3d_drv.h"
  20 #include "v3d_regs.h"
  21 #include "v3d_trace.h"
  22
  23 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM |     \
  24                                   V3D_INT_FLDONE |      \
  25                                   V3D_INT_FRDONE |      \
  26                                   V3D_INT_CSDDONE(ver) |        \
  27                                   (ver < 71 ? V3D_INT_GMPV : 0)))
  28
  29 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV |  \
  30                                  V3D_HUB_INT_MMU_PTI |  \
  31                                  V3D_HUB_INT_MMU_CAP |  \
  32                                  V3D_HUB_INT_TFUC |             \
  33                                  (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0)))
  34
  35 static irqreturn_t
  36 v3d_hub_irq(int irq, void *arg);
  37
  38 static void
  39 v3d_overflow_mem_work(struct work_struct *work)
  40 {
  41         struct v3d_dev *v3d =
  42                 container_of(work, struct v3d_dev, overflow_mem_work);
  43         struct drm_device *dev = &v3d->drm;
  44         struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
  45         struct drm_gem_object *obj;
  46         unsigned long irqflags;
  47
  48         if (IS_ERR(bo)) {
  49                 DRM_ERROR("Couldn't allocate binner overflow mem\n");
  50                 return;
  51         }
  52         obj = &bo->base.base;
  53
  54         /* We lost a race, and our work task came in after the bin job
  55          * completed and exited.  This can happen because the HW
  56          * signals OOM before it's fully OOM, so the binner might just
  57          * barely complete.
  58          *
  59          * If we lose the race and our work task comes in after a new
  60          * bin job got scheduled, that's fine.  We'll just give them
  61          * some binner pool anyway.
  62          */
  63         spin_lock_irqsave(&v3d->job_lock, irqflags);
  64         if (!v3d->bin_job) {
  65                 spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  66                 goto out;
  67         }
  68
  69         drm_gem_object_get(obj);
  70         list_add_tail(&bo->unref_head, &v3d->bin_job->render->unref_list);
  71         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  72
  73         v3d_mmu_flush_all(v3d);
  74
  75         V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT);
  76         V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
  77
  78 out:
  79         drm_gem_object_put(obj);
  80 }
  81
  82 static irqreturn_t
  83 v3d_irq(int irq, void *arg)
  84 {
  85         struct v3d_dev *v3d = arg;
  86         u32 intsts;
  87         irqreturn_t status = IRQ_NONE;
  88
  89         intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
  90
  91         /* Acknowledge the interrupts we're handling here. */
  92         V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
  93
  94         if (intsts & V3D_INT_OUTOMEM) {
  95                 /* Note that the OOM status is edge signaled, so the
  96                  * interrupt won't happen again until the we actually
  97                  * add more memory.  Also, as of V3D 4.1, FLDONE won't
  98                  * be reported until any OOM state has been cleared.
  99                  */
 100                 schedule_work(&v3d->overflow_mem_work);
 101                 status = IRQ_HANDLED;
 102         }
 103
 104         if (intsts & V3D_INT_FLDONE) {
 105                 struct v3d_fence *fence =
 106                         to_v3d_fence(v3d->bin_job->base.irq_fence);
 107
 108                 v3d_job_update_stats(&v3d->bin_job->base, V3D_BIN);
 109                 trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 110                 dma_fence_signal(&fence->base);
 111                 status = IRQ_HANDLED;
 112         }
 113
 114         if (intsts & V3D_INT_FRDONE) {
 115                 struct v3d_fence *fence =
 116                         to_v3d_fence(v3d->render_job->base.irq_fence);
 117
 118                 v3d_job_update_stats(&v3d->render_job->base, V3D_RENDER);
 119                 trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 120                 dma_fence_signal(&fence->base);
 121                 status = IRQ_HANDLED;
 122         }
 123
 124         if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 125                 struct v3d_fence *fence =
 126                         to_v3d_fence(v3d->csd_job->base.irq_fence);
 127
 128                 v3d_job_update_stats(&v3d->csd_job->base, V3D_CSD);
 129                 trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 130                 dma_fence_signal(&fence->base);
 131                 status = IRQ_HANDLED;
 132         }
 133
 134         /* We shouldn't be triggering these if we have GMP in
 135          * always-allowed mode.
 136          */
 137         if (v3d->ver < 71 && (intsts & V3D_INT_GMPV))
 138                 dev_err(v3d->drm.dev, "GMP violation\n");
 139
 140         /* V3D 4.2 wires the hub and core IRQs together, so if we &
 141          * didn't see the common one then check hub for MMU IRQs.
 142          */
 143         if (v3d->single_irq_line && status == IRQ_NONE)
 144                 return v3d_hub_irq(irq, arg);
 145
 146         return status;
 147 }
 148
 149 static irqreturn_t
 150 v3d_hub_irq(int irq, void *arg)
 151 {
 152         struct v3d_dev *v3d = arg;
 153         u32 intsts;
 154         irqreturn_t status = IRQ_NONE;
 155
 156         intsts = V3D_READ(V3D_HUB_INT_STS);
 157
 158         /* Acknowledge the interrupts we're handling here. */
 159         V3D_WRITE(V3D_HUB_INT_CLR, intsts);
 160
 161         if (intsts & V3D_HUB_INT_TFUC) {
 162                 struct v3d_fence *fence =
 163                         to_v3d_fence(v3d->tfu_job->base.irq_fence);
 164
 165                 v3d_job_update_stats(&v3d->tfu_job->base, V3D_TFU);
 166                 trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 167                 dma_fence_signal(&fence->base);
 168                 status = IRQ_HANDLED;
 169         }
 170
 171         if (intsts & (V3D_HUB_INT_MMU_WRV |
 172                       V3D_HUB_INT_MMU_PTI |
 173                       V3D_HUB_INT_MMU_CAP)) {
 174                 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
 175                 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) <<
 176                                 (v3d->va_width - 32));
 177                 static const char *const v3d41_axi_ids[] = {
 178                         "L2T",
 179                         "PTB",
 180                         "PSE",
 181                         "TLB",
 182                         "CLE",
 183                         "TFU",
 184                         "MMU",
 185                         "GMP",
 186                 };
 187                 const char *client = "?";
 188
 189                 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL));
 190
 191                 if (v3d->ver >= 41) {
 192                         axi_id = axi_id >> 5;
 193                         if (axi_id < ARRAY_SIZE(v3d41_axi_ids))
 194                                 client = v3d41_axi_ids[axi_id];
 195                 }
 196
 197                 dev_err(v3d->drm.dev, "MMU error from client %s (%d) at 0x%llx%s%s%s\n",
 198                         client, axi_id, (long long)vio_addr,
 199                         ((intsts & V3D_HUB_INT_MMU_WRV) ?
 200                          ", write violation" : ""),
 201                         ((intsts & V3D_HUB_INT_MMU_PTI) ?
 202                          ", pte invalid" : ""),
 203                         ((intsts & V3D_HUB_INT_MMU_CAP) ?
 204                          ", cap exceeded" : ""));
 205                 status = IRQ_HANDLED;
 206         }
 207
 208         if (v3d->ver >= 71 && (intsts & V3D_V7_HUB_INT_GMPV)) {
 209                 dev_err(v3d->drm.dev, "GMP Violation\n");
 210                 status = IRQ_HANDLED;
 211         }
 212
 213         return status;
 214 }
 215
 216 int
 217 v3d_irq_init(struct v3d_dev *v3d)
 218 {
 219         int irq1, ret, core;
 220
 221         INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 222
 223         /* Clear any pending interrupts someone might have left around
 224          * for us.
 225          */
 226         for (core = 0; core < v3d->cores; core++)
 227                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 228         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 229
 230         irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 231         if (irq1 == -EPROBE_DEFER)
 232                 return irq1;
 233         if (irq1 > 0) {
 234                 ret = devm_request_irq(v3d->drm.dev, irq1,
 235                                        v3d_irq, IRQF_SHARED,
 236                                        "v3d_core0", v3d);
 237                 if (ret)
 238                         goto fail;
 239                 ret = devm_request_irq(v3d->drm.dev,
 240                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 241                                        v3d_hub_irq, IRQF_SHARED,
 242                                        "v3d_hub", v3d);
 243                 if (ret)
 244                         goto fail;
 245         } else {
 246                 v3d->single_irq_line = true;
 247
 248                 ret = devm_request_irq(v3d->drm.dev,
 249                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 250                                        v3d_irq, IRQF_SHARED,
 251                                        "v3d", v3d);
 252                 if (ret)
 253                         goto fail;
 254         }
 255
 256         v3d_irq_enable(v3d);
 257         return 0;
 258
 259 fail:
 260         if (ret != -EPROBE_DEFER)
 261                 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret);
 262         return ret;
 263 }
 264
 265 void
 266 v3d_irq_enable(struct v3d_dev *v3d)
 267 {
 268         int core;
 269
 270         /* Enable our set of interrupts, masking out any others. */
 271         for (core = 0; core < v3d->cores; core++) {
 272                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
 273                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver));
 274         }
 275
 276         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver));
 277         V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver));
 278 }
 279
 280 void
 281 v3d_irq_disable(struct v3d_dev *v3d)
 282 {
 283         int core;
 284
 285         /* Disable all interrupts. */
 286         for (core = 0; core < v3d->cores; core++)
 287                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
 288         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
 289
 290         /* Clear any pending interrupts we might have left. */
 291         for (core = 0; core < v3d->cores; core++)
 292                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 293         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 294
 295         cancel_work_sync(&v3d->overflow_mem_work);
 296 }
 297
 298 /** Reinitializes interrupt registers when a GPU reset is performed. */
 299 void v3d_irq_reset(struct v3d_dev *v3d)
 300 {
 301         v3d_irq_enable(v3d);
 302 }