drivers/gpu/drm/v3d/v3d_irq.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2014-2018 Broadcom */
   3
   4 /**
   5  * DOC: Interrupt management for the V3D engine
   6  *
   7  * When we take a bin, render, TFU done, or CSD done interrupt, we
   8  * need to signal the fence for that job so that the scheduler can
   9  * queue up the next one and unblock any waiters.
  10  *
  11  * When we take the binner out of memory interrupt, we need to
  12  * allocate some new memory and pass it to the binner so that the
  13  * current job can make progress.
  14  */
  15
  16 #include <linux/platform_device.h>
  17 #include <linux/sched/clock.h>
  18
  19 #include "v3d_drv.h"
  20 #include "v3d_regs.h"
  21 #include "v3d_trace.h"
  22
  23 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM |     \
  24                                   V3D_INT_FLDONE |      \
  25                                   V3D_INT_FRDONE |      \
  26                                   V3D_INT_CSDDONE(ver) |        \
  27                                   (ver < 71 ? V3D_INT_GMPV : 0)))
  28
  29 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV |  \
  30                                  V3D_HUB_INT_MMU_PTI |  \
  31                                  V3D_HUB_INT_MMU_CAP |  \
  32                                  V3D_HUB_INT_TFUC |             \
  33                                  (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0)))
  34
  35 static irqreturn_t
  36 v3d_hub_irq(int irq, void *arg);
  37
  38 static void
  39 v3d_overflow_mem_work(struct work_struct *work)
  40 {
  41         struct v3d_dev *v3d =
  42                 container_of(work, struct v3d_dev, overflow_mem_work);
  43         struct drm_device *dev = &v3d->drm;
  44         struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
  45         struct drm_gem_object *obj;
  46         unsigned long irqflags;
  47
  48         if (IS_ERR(bo)) {
  49                 DRM_ERROR("Couldn't allocate binner overflow mem\n");
  50                 return;
  51         }
  52         obj = &bo->base.base;
  53
  54         /* We lost a race, and our work task came in after the bin job
  55          * completed and exited.  This can happen because the HW
  56          * signals OOM before it's fully OOM, so the binner might just
  57          * barely complete.
  58          *
  59          * If we lose the race and our work task comes in after a new
  60          * bin job got scheduled, that's fine.  We'll just give them
  61          * some binner pool anyway.
  62          */
  63         spin_lock_irqsave(&v3d->job_lock, irqflags);
  64         if (!v3d->bin_job) {
  65                 spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  66                 goto out;
  67         }
  68
  69         drm_gem_object_get(obj);
  70         list_add_tail(&bo->unref_head, &v3d->bin_job->render->unref_list);
  71         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  72
  73         v3d_mmu_flush_all(v3d);
  74
  75         V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT);
  76         V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
  77
  78 out:
  79         drm_gem_object_put(obj);
  80 }
  81
  82 static irqreturn_t
  83 v3d_irq(int irq, void *arg)
  84 {
  85         struct v3d_dev *v3d = arg;
  86         u32 intsts;
  87         irqreturn_t status = IRQ_NONE;
  88
  89         intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
  90
  91         /* Acknowledge the interrupts we're handling here. */
  92         V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
  93
  94         if (intsts & V3D_INT_OUTOMEM) {
  95                 /* Note that the OOM status is edge signaled, so the
  96                  * interrupt won't happen again until the we actually
  97                  * add more memory.  Also, as of V3D 4.1, FLDONE won't
  98                  * be reported until any OOM state has been cleared.
  99                  */
 100                 schedule_work(&v3d->overflow_mem_work);
 101                 status = IRQ_HANDLED;
 102         }
 103
 104         if (intsts & V3D_INT_FLDONE) {
 105                 struct v3d_fence *fence =
 106                         to_v3d_fence(v3d->bin_job->base.irq_fence);
 107
 108                 v3d_job_update_stats(&v3d->bin_job->base, V3D_BIN);
 109                 trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 110
 111                 v3d->bin_job = NULL;
 112                 dma_fence_signal(&fence->base);
 113
 114                 status = IRQ_HANDLED;
 115         }
 116
 117         if (intsts & V3D_INT_FRDONE) {
 118                 struct v3d_fence *fence =
 119                         to_v3d_fence(v3d->render_job->base.irq_fence);
 120
 121                 v3d_job_update_stats(&v3d->render_job->base, V3D_RENDER);
 122                 trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 123
 124                 v3d->render_job = NULL;
 125                 dma_fence_signal(&fence->base);
 126
 127                 status = IRQ_HANDLED;
 128         }
 129
 130         if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 131                 struct v3d_fence *fence =
 132                         to_v3d_fence(v3d->csd_job->base.irq_fence);
 133
 134                 v3d_job_update_stats(&v3d->csd_job->base, V3D_CSD);
 135                 trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 136
 137                 v3d->csd_job = NULL;
 138                 dma_fence_signal(&fence->base);
 139
 140                 status = IRQ_HANDLED;
 141         }
 142
 143         /* We shouldn't be triggering these if we have GMP in
 144          * always-allowed mode.
 145          */
 146         if (v3d->ver < 71 && (intsts & V3D_INT_GMPV))
 147                 dev_err(v3d->drm.dev, "GMP violation\n");
 148
 149         /* V3D 4.2 wires the hub and core IRQs together, so if we &
 150          * didn't see the common one then check hub for MMU IRQs.
 151          */
 152         if (v3d->single_irq_line && status == IRQ_NONE)
 153                 return v3d_hub_irq(irq, arg);
 154
 155         return status;
 156 }
 157
 158 static irqreturn_t
 159 v3d_hub_irq(int irq, void *arg)
 160 {
 161         struct v3d_dev *v3d = arg;
 162         u32 intsts;
 163         irqreturn_t status = IRQ_NONE;
 164
 165         intsts = V3D_READ(V3D_HUB_INT_STS);
 166
 167         /* Acknowledge the interrupts we're handling here. */
 168         V3D_WRITE(V3D_HUB_INT_CLR, intsts);
 169
 170         if (intsts & V3D_HUB_INT_TFUC) {
 171                 struct v3d_fence *fence =
 172                         to_v3d_fence(v3d->tfu_job->base.irq_fence);
 173
 174                 v3d_job_update_stats(&v3d->tfu_job->base, V3D_TFU);
 175                 trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 176
 177                 v3d->tfu_job = NULL;
 178                 dma_fence_signal(&fence->base);
 179
 180                 status = IRQ_HANDLED;
 181         }
 182
 183         if (intsts & (V3D_HUB_INT_MMU_WRV |
 184                       V3D_HUB_INT_MMU_PTI |
 185                       V3D_HUB_INT_MMU_CAP)) {
 186                 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
 187                 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) <<
 188                                 (v3d->va_width - 32));
 189                 static const char *const v3d41_axi_ids[] = {
 190                         "L2T",
 191                         "PTB",
 192                         "PSE",
 193                         "TLB",
 194                         "CLE",
 195                         "TFU",
 196                         "MMU",
 197                         "GMP",
 198                 };
 199                 const char *client = "?";
 200
 201                 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL));
 202
 203                 if (v3d->ver >= 41) {
 204                         axi_id = axi_id >> 5;
 205                         if (axi_id < ARRAY_SIZE(v3d41_axi_ids))
 206                                 client = v3d41_axi_ids[axi_id];
 207                 }
 208
 209                 dev_err(v3d->drm.dev, "MMU error from client %s (%d) at 0x%llx%s%s%s\n",
 210                         client, axi_id, (long long)vio_addr,
 211                         ((intsts & V3D_HUB_INT_MMU_WRV) ?
 212                          ", write violation" : ""),
 213                         ((intsts & V3D_HUB_INT_MMU_PTI) ?
 214                          ", pte invalid" : ""),
 215                         ((intsts & V3D_HUB_INT_MMU_CAP) ?
 216                          ", cap exceeded" : ""));
 217                 status = IRQ_HANDLED;
 218         }
 219
 220         if (v3d->ver >= 71 && (intsts & V3D_V7_HUB_INT_GMPV)) {
 221                 dev_err(v3d->drm.dev, "GMP Violation\n");
 222                 status = IRQ_HANDLED;
 223         }
 224
 225         return status;
 226 }
 227
 228 int
 229 v3d_irq_init(struct v3d_dev *v3d)
 230 {
 231         int irq1, ret, core;
 232
 233         INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 234
 235         /* Clear any pending interrupts someone might have left around
 236          * for us.
 237          */
 238         for (core = 0; core < v3d->cores; core++)
 239                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 240         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 241
 242         irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 243         if (irq1 == -EPROBE_DEFER)
 244                 return irq1;
 245         if (irq1 > 0) {
 246                 ret = devm_request_irq(v3d->drm.dev, irq1,
 247                                        v3d_irq, IRQF_SHARED,
 248                                        "v3d_core0", v3d);
 249                 if (ret)
 250                         goto fail;
 251                 ret = devm_request_irq(v3d->drm.dev,
 252                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 253                                        v3d_hub_irq, IRQF_SHARED,
 254                                        "v3d_hub", v3d);
 255                 if (ret)
 256                         goto fail;
 257         } else {
 258                 v3d->single_irq_line = true;
 259
 260                 ret = devm_request_irq(v3d->drm.dev,
 261                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 262                                        v3d_irq, IRQF_SHARED,
 263                                        "v3d", v3d);
 264                 if (ret)
 265                         goto fail;
 266         }
 267
 268         v3d_irq_enable(v3d);
 269         return 0;
 270
 271 fail:
 272         if (ret != -EPROBE_DEFER)
 273                 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret);
 274         return ret;
 275 }
 276
 277 void
 278 v3d_irq_enable(struct v3d_dev *v3d)
 279 {
 280         int core;
 281
 282         /* Enable our set of interrupts, masking out any others. */
 283         for (core = 0; core < v3d->cores; core++) {
 284                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
 285                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver));
 286         }
 287
 288         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver));
 289         V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver));
 290 }
 291
 292 void
 293 v3d_irq_disable(struct v3d_dev *v3d)
 294 {
 295         int core;
 296
 297         /* Disable all interrupts. */
 298         for (core = 0; core < v3d->cores; core++)
 299                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
 300         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
 301
 302         /* Clear any pending interrupts we might have left. */
 303         for (core = 0; core < v3d->cores; core++)
 304                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 305         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 306
 307         cancel_work_sync(&v3d->overflow_mem_work);
 308 }
 309
 310 /** Reinitializes interrupt registers when a GPU reset is performed. */
 311 void v3d_irq_reset(struct v3d_dev *v3d)
 312 {
 313         v3d_irq_enable(v3d);
 314 }