drivers/gpu/drm/v3d/v3d_irq.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2014-2018 Broadcom */
   3
   4 /**
   5  * DOC: Interrupt management for the V3D engine
   6  *
   7  * When we take a bin, render, TFU done, or CSD done interrupt, we
   8  * need to signal the fence for that job so that the scheduler can
   9  * queue up the next one and unblock any waiters.
  10  *
  11  * When we take the binner out of memory interrupt, we need to
  12  * allocate some new memory and pass it to the binner so that the
  13  * current job can make progress.
  14  */
  15
  16 #include <linux/platform_device.h>
  17 #include <linux/sched/clock.h>
  18
  19 #include "v3d_drv.h"
  20 #include "v3d_regs.h"
  21 #include "v3d_trace.h"
  22
  23 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM |     \
  24                                   V3D_INT_FLDONE |      \
  25                                   V3D_INT_FRDONE |      \
  26                                   V3D_INT_CSDDONE(ver) |        \
  27                                   (ver < 71 ? V3D_INT_GMPV : 0)))
  28
  29 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV |  \
  30                                  V3D_HUB_INT_MMU_PTI |  \
  31                                  V3D_HUB_INT_MMU_CAP |  \
  32                                  V3D_HUB_INT_TFUC |             \
  33                                  (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0)))
  34
  35 static irqreturn_t
  36 v3d_hub_irq(int irq, void *arg);
  37
  38 static void
  39 v3d_overflow_mem_work(struct work_struct *work)
  40 {
  41         struct v3d_dev *v3d =
  42                 container_of(work, struct v3d_dev, overflow_mem_work);
  43         struct drm_device *dev = &v3d->drm;
  44         struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
  45         struct drm_gem_object *obj;
  46         unsigned long irqflags;
  47
  48         if (IS_ERR(bo)) {
  49                 DRM_ERROR("Couldn't allocate binner overflow mem\n");
  50                 return;
  51         }
  52         obj = &bo->base.base;
  53
  54         /* We lost a race, and our work task came in after the bin job
  55          * completed and exited.  This can happen because the HW
  56          * signals OOM before it's fully OOM, so the binner might just
  57          * barely complete.
  58          *
  59          * If we lose the race and our work task comes in after a new
  60          * bin job got scheduled, that's fine.  We'll just give them
  61          * some binner pool anyway.
  62          */
  63         spin_lock_irqsave(&v3d->job_lock, irqflags);
  64         if (!v3d->bin_job) {
  65                 spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  66                 goto out;
  67         }
  68
  69         drm_gem_object_get(obj);
  70         list_add_tail(&bo->unref_head, &v3d->bin_job->render->unref_list);
  71         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  72
  73         v3d_mmu_flush_all(v3d);
  74
  75         V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT);
  76         V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
  77
  78 out:
  79         drm_gem_object_put(obj);
  80 }
  81
  82 static irqreturn_t
  83 v3d_irq(int irq, void *arg)
  84 {
  85         struct v3d_dev *v3d = arg;
  86         u32 intsts;
  87         irqreturn_t status = IRQ_NONE;
  88
  89         intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
  90
  91         /* Acknowledge the interrupts we're handling here. */
  92         V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
  93
  94         if (intsts & V3D_INT_OUTOMEM) {
  95                 /* Note that the OOM status is edge signaled, so the
  96                  * interrupt won't happen again until the we actually
  97                  * add more memory.  Also, as of V3D 4.1, FLDONE won't
  98                  * be reported until any OOM state has been cleared.
  99                  */
 100                 schedule_work(&v3d->overflow_mem_work);
 101                 status = IRQ_HANDLED;
 102         }
 103
 104         if (intsts & V3D_INT_FLDONE) {
 105                 struct v3d_fence *fence =
 106                         to_v3d_fence(v3d->bin_job->base.irq_fence);
 107
 108                 v3d_job_update_stats(&v3d->bin_job->base, V3D_BIN);
 109                 trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 110                 dma_fence_signal(&fence->base);
 111                 v3d->bin_job = NULL;
 112                 status = IRQ_HANDLED;
 113         }
 114
 115         if (intsts & V3D_INT_FRDONE) {
 116                 struct v3d_fence *fence =
 117                         to_v3d_fence(v3d->render_job->base.irq_fence);
 118
 119                 v3d_job_update_stats(&v3d->render_job->base, V3D_RENDER);
 120                 trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 121                 dma_fence_signal(&fence->base);
 122                 v3d->render_job = NULL;
 123                 status = IRQ_HANDLED;
 124         }
 125
 126         if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 127                 struct v3d_fence *fence =
 128                         to_v3d_fence(v3d->csd_job->base.irq_fence);
 129
 130                 v3d_job_update_stats(&v3d->csd_job->base, V3D_CSD);
 131                 trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 132                 dma_fence_signal(&fence->base);
 133                 v3d->csd_job = NULL;
 134                 status = IRQ_HANDLED;
 135         }
 136
 137         /* We shouldn't be triggering these if we have GMP in
 138          * always-allowed mode.
 139          */
 140         if (v3d->ver < 71 && (intsts & V3D_INT_GMPV))
 141                 dev_err(v3d->drm.dev, "GMP violation\n");
 142
 143         /* V3D 4.2 wires the hub and core IRQs together, so if we &
 144          * didn't see the common one then check hub for MMU IRQs.
 145          */
 146         if (v3d->single_irq_line && status == IRQ_NONE)
 147                 return v3d_hub_irq(irq, arg);
 148
 149         return status;
 150 }
 151
 152 static irqreturn_t
 153 v3d_hub_irq(int irq, void *arg)
 154 {
 155         struct v3d_dev *v3d = arg;
 156         u32 intsts;
 157         irqreturn_t status = IRQ_NONE;
 158
 159         intsts = V3D_READ(V3D_HUB_INT_STS);
 160
 161         /* Acknowledge the interrupts we're handling here. */
 162         V3D_WRITE(V3D_HUB_INT_CLR, intsts);
 163
 164         if (intsts & V3D_HUB_INT_TFUC) {
 165                 struct v3d_fence *fence =
 166                         to_v3d_fence(v3d->tfu_job->base.irq_fence);
 167
 168                 v3d_job_update_stats(&v3d->tfu_job->base, V3D_TFU);
 169                 trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 170                 dma_fence_signal(&fence->base);
 171                 v3d->tfu_job = NULL;
 172                 status = IRQ_HANDLED;
 173         }
 174
 175         if (intsts & (V3D_HUB_INT_MMU_WRV |
 176                       V3D_HUB_INT_MMU_PTI |
 177                       V3D_HUB_INT_MMU_CAP)) {
 178                 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
 179                 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) <<
 180                                 (v3d->va_width - 32));
 181                 static const char *const v3d41_axi_ids[] = {
 182                         "L2T",
 183                         "PTB",
 184                         "PSE",
 185                         "TLB",
 186                         "CLE",
 187                         "TFU",
 188                         "MMU",
 189                         "GMP",
 190                 };
 191                 const char *client = "?";
 192
 193                 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL));
 194
 195                 if (v3d->ver >= 41) {
 196                         axi_id = axi_id >> 5;
 197                         if (axi_id < ARRAY_SIZE(v3d41_axi_ids))
 198                                 client = v3d41_axi_ids[axi_id];
 199                 }
 200
 201                 dev_err(v3d->drm.dev, "MMU error from client %s (%d) at 0x%llx%s%s%s\n",
 202                         client, axi_id, (long long)vio_addr,
 203                         ((intsts & V3D_HUB_INT_MMU_WRV) ?
 204                          ", write violation" : ""),
 205                         ((intsts & V3D_HUB_INT_MMU_PTI) ?
 206                          ", pte invalid" : ""),
 207                         ((intsts & V3D_HUB_INT_MMU_CAP) ?
 208                          ", cap exceeded" : ""));
 209                 status = IRQ_HANDLED;
 210         }
 211
 212         if (v3d->ver >= 71 && (intsts & V3D_V7_HUB_INT_GMPV)) {
 213                 dev_err(v3d->drm.dev, "GMP Violation\n");
 214                 status = IRQ_HANDLED;
 215         }
 216
 217         return status;
 218 }
 219
 220 int
 221 v3d_irq_init(struct v3d_dev *v3d)
 222 {
 223         int irq1, ret, core;
 224
 225         INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 226
 227         /* Clear any pending interrupts someone might have left around
 228          * for us.
 229          */
 230         for (core = 0; core < v3d->cores; core++)
 231                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 232         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 233
 234         irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 235         if (irq1 == -EPROBE_DEFER)
 236                 return irq1;
 237         if (irq1 > 0) {
 238                 ret = devm_request_irq(v3d->drm.dev, irq1,
 239                                        v3d_irq, IRQF_SHARED,
 240                                        "v3d_core0", v3d);
 241                 if (ret)
 242                         goto fail;
 243                 ret = devm_request_irq(v3d->drm.dev,
 244                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 245                                        v3d_hub_irq, IRQF_SHARED,
 246                                        "v3d_hub", v3d);
 247                 if (ret)
 248                         goto fail;
 249         } else {
 250                 v3d->single_irq_line = true;
 251
 252                 ret = devm_request_irq(v3d->drm.dev,
 253                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 254                                        v3d_irq, IRQF_SHARED,
 255                                        "v3d", v3d);
 256                 if (ret)
 257                         goto fail;
 258         }
 259
 260         v3d_irq_enable(v3d);
 261         return 0;
 262
 263 fail:
 264         if (ret != -EPROBE_DEFER)
 265                 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret);
 266         return ret;
 267 }
 268
 269 void
 270 v3d_irq_enable(struct v3d_dev *v3d)
 271 {
 272         int core;
 273
 274         /* Enable our set of interrupts, masking out any others. */
 275         for (core = 0; core < v3d->cores; core++) {
 276                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
 277                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver));
 278         }
 279
 280         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver));
 281         V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver));
 282 }
 283
 284 void
 285 v3d_irq_disable(struct v3d_dev *v3d)
 286 {
 287         int core;
 288
 289         /* Disable all interrupts. */
 290         for (core = 0; core < v3d->cores; core++)
 291                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
 292         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
 293
 294         /* Clear any pending interrupts we might have left. */
 295         for (core = 0; core < v3d->cores; core++)
 296                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 297         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 298
 299         cancel_work_sync(&v3d->overflow_mem_work);
 300 }
 301
 302 /** Reinitializes interrupt registers when a GPU reset is performed. */
 303 void v3d_irq_reset(struct v3d_dev *v3d)
 304 {
 305         v3d_irq_enable(v3d);
 306 }