drivers/gpu/drm/v3d/v3d_irq.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Copyright (C) 2014-2018 Broadcom */
   3
   4 /**
   5  * DOC: Interrupt management for the V3D engine
   6  *
   7  * When we take a bin, render, TFU done, or CSD done interrupt, we
   8  * need to signal the fence for that job so that the scheduler can
   9  * queue up the next one and unblock any waiters.
  10  *
  11  * When we take the binner out of memory interrupt, we need to
  12  * allocate some new memory and pass it to the binner so that the
  13  * current job can make progress.
  14  */
  15
  16 #include <linux/platform_device.h>
  17 #include <linux/sched/clock.h>
  18
  19 #include "v3d_drv.h"
  20 #include "v3d_regs.h"
  21 #include "v3d_trace.h"
  22
  23 #define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM |     \
  24                                   V3D_INT_FLDONE |      \
  25                                   V3D_INT_FRDONE |      \
  26                                   V3D_INT_CSDDONE(ver) |        \
  27                                   (ver < 71 ? V3D_INT_GMPV : 0)))
  28
  29 #define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV |  \
  30                                  V3D_HUB_INT_MMU_PTI |  \
  31                                  V3D_HUB_INT_MMU_CAP |  \
  32                                  V3D_HUB_INT_TFUC |             \
  33                                  (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0)))
  34
  35 static irqreturn_t
  36 v3d_hub_irq(int irq, void *arg);
  37
  38 static void
  39 v3d_overflow_mem_work(struct work_struct *work)
  40 {
  41         struct v3d_dev *v3d =
  42                 container_of(work, struct v3d_dev, overflow_mem_work);
  43         struct drm_device *dev = &v3d->drm;
  44         struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
  45         struct drm_gem_object *obj;
  46         unsigned long irqflags;
  47
  48         if (IS_ERR(bo)) {
  49                 DRM_ERROR("Couldn't allocate binner overflow mem\n");
  50                 return;
  51         }
  52         obj = &bo->base.base;
  53
  54         /* We lost a race, and our work task came in after the bin job
  55          * completed and exited.  This can happen because the HW
  56          * signals OOM before it's fully OOM, so the binner might just
  57          * barely complete.
  58          *
  59          * If we lose the race and our work task comes in after a new
  60          * bin job got scheduled, that's fine.  We'll just give them
  61          * some binner pool anyway.
  62          */
  63         spin_lock_irqsave(&v3d->job_lock, irqflags);
  64         if (!v3d->bin_job) {
  65                 spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  66                 goto out;
  67         }
  68
  69         drm_gem_object_get(obj);
  70         list_add_tail(&bo->unref_head, &v3d->bin_job->render->unref_list);
  71         spin_unlock_irqrestore(&v3d->job_lock, irqflags);
  72
  73         V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << V3D_MMU_PAGE_SHIFT);
  74         V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
  75
  76 out:
  77         drm_gem_object_put(obj);
  78 }
  79
  80 static irqreturn_t
  81 v3d_irq(int irq, void *arg)
  82 {
  83         struct v3d_dev *v3d = arg;
  84         u32 intsts;
  85         irqreturn_t status = IRQ_NONE;
  86
  87         intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
  88
  89         /* Acknowledge the interrupts we're handling here. */
  90         V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
  91
  92         if (intsts & V3D_INT_OUTOMEM) {
  93                 /* Note that the OOM status is edge signaled, so the
  94                  * interrupt won't happen again until the we actually
  95                  * add more memory.  Also, as of V3D 4.1, FLDONE won't
  96                  * be reported until any OOM state has been cleared.
  97                  */
  98                 schedule_work(&v3d->overflow_mem_work);
  99                 status = IRQ_HANDLED;
 100         }
 101
 102         if (intsts & V3D_INT_FLDONE) {
 103                 struct v3d_fence *fence =
 104                         to_v3d_fence(v3d->bin_job->base.irq_fence);
 105
 106                 v3d_job_update_stats(&v3d->bin_job->base, V3D_BIN);
 107                 trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 108                 dma_fence_signal(&fence->base);
 109                 status = IRQ_HANDLED;
 110         }
 111
 112         if (intsts & V3D_INT_FRDONE) {
 113                 struct v3d_fence *fence =
 114                         to_v3d_fence(v3d->render_job->base.irq_fence);
 115
 116                 v3d_job_update_stats(&v3d->render_job->base, V3D_RENDER);
 117                 trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 118                 dma_fence_signal(&fence->base);
 119                 status = IRQ_HANDLED;
 120         }
 121
 122         if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 123                 struct v3d_fence *fence =
 124                         to_v3d_fence(v3d->csd_job->base.irq_fence);
 125
 126                 v3d_job_update_stats(&v3d->csd_job->base, V3D_CSD);
 127                 trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 128                 dma_fence_signal(&fence->base);
 129                 status = IRQ_HANDLED;
 130         }
 131
 132         /* We shouldn't be triggering these if we have GMP in
 133          * always-allowed mode.
 134          */
 135         if (v3d->ver < 71 && (intsts & V3D_INT_GMPV))
 136                 dev_err(v3d->drm.dev, "GMP violation\n");
 137
 138         /* V3D 4.2 wires the hub and core IRQs together, so if we &
 139          * didn't see the common one then check hub for MMU IRQs.
 140          */
 141         if (v3d->single_irq_line && status == IRQ_NONE)
 142                 return v3d_hub_irq(irq, arg);
 143
 144         return status;
 145 }
 146
 147 static irqreturn_t
 148 v3d_hub_irq(int irq, void *arg)
 149 {
 150         struct v3d_dev *v3d = arg;
 151         u32 intsts;
 152         irqreturn_t status = IRQ_NONE;
 153
 154         intsts = V3D_READ(V3D_HUB_INT_STS);
 155
 156         /* Acknowledge the interrupts we're handling here. */
 157         V3D_WRITE(V3D_HUB_INT_CLR, intsts);
 158
 159         if (intsts & V3D_HUB_INT_TFUC) {
 160                 struct v3d_fence *fence =
 161                         to_v3d_fence(v3d->tfu_job->base.irq_fence);
 162
 163                 v3d_job_update_stats(&v3d->tfu_job->base, V3D_TFU);
 164                 trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 165                 dma_fence_signal(&fence->base);
 166                 status = IRQ_HANDLED;
 167         }
 168
 169         if (intsts & (V3D_HUB_INT_MMU_WRV |
 170                       V3D_HUB_INT_MMU_PTI |
 171                       V3D_HUB_INT_MMU_CAP)) {
 172                 u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
 173                 u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) <<
 174                                 (v3d->va_width - 32));
 175                 static const char *const v3d41_axi_ids[] = {
 176                         "L2T",
 177                         "PTB",
 178                         "PSE",
 179                         "TLB",
 180                         "CLE",
 181                         "TFU",
 182                         "MMU",
 183                         "GMP",
 184                 };
 185                 const char *client = "?";
 186
 187                 V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL));
 188
 189                 if (v3d->ver >= 41) {
 190                         axi_id = axi_id >> 5;
 191                         if (axi_id < ARRAY_SIZE(v3d41_axi_ids))
 192                                 client = v3d41_axi_ids[axi_id];
 193                 }
 194
 195                 dev_err(v3d->drm.dev, "MMU error from client %s (%d) at 0x%llx%s%s%s\n",
 196                         client, axi_id, (long long)vio_addr,
 197                         ((intsts & V3D_HUB_INT_MMU_WRV) ?
 198                          ", write violation" : ""),
 199                         ((intsts & V3D_HUB_INT_MMU_PTI) ?
 200                          ", pte invalid" : ""),
 201                         ((intsts & V3D_HUB_INT_MMU_CAP) ?
 202                          ", cap exceeded" : ""));
 203                 status = IRQ_HANDLED;
 204         }
 205
 206         if (v3d->ver >= 71 && (intsts & V3D_V7_HUB_INT_GMPV)) {
 207                 dev_err(v3d->drm.dev, "GMP Violation\n");
 208                 status = IRQ_HANDLED;
 209         }
 210
 211         return status;
 212 }
 213
 214 int
 215 v3d_irq_init(struct v3d_dev *v3d)
 216 {
 217         int irq1, ret, core;
 218
 219         INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 220
 221         /* Clear any pending interrupts someone might have left around
 222          * for us.
 223          */
 224         for (core = 0; core < v3d->cores; core++)
 225                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 226         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 227
 228         irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 229         if (irq1 == -EPROBE_DEFER)
 230                 return irq1;
 231         if (irq1 > 0) {
 232                 ret = devm_request_irq(v3d->drm.dev, irq1,
 233                                        v3d_irq, IRQF_SHARED,
 234                                        "v3d_core0", v3d);
 235                 if (ret)
 236                         goto fail;
 237                 ret = devm_request_irq(v3d->drm.dev,
 238                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 239                                        v3d_hub_irq, IRQF_SHARED,
 240                                        "v3d_hub", v3d);
 241                 if (ret)
 242                         goto fail;
 243         } else {
 244                 v3d->single_irq_line = true;
 245
 246                 ret = devm_request_irq(v3d->drm.dev,
 247                                        platform_get_irq(v3d_to_pdev(v3d), 0),
 248                                        v3d_irq, IRQF_SHARED,
 249                                        "v3d", v3d);
 250                 if (ret)
 251                         goto fail;
 252         }
 253
 254         v3d_irq_enable(v3d);
 255         return 0;
 256
 257 fail:
 258         if (ret != -EPROBE_DEFER)
 259                 dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret);
 260         return ret;
 261 }
 262
 263 void
 264 v3d_irq_enable(struct v3d_dev *v3d)
 265 {
 266         int core;
 267
 268         /* Enable our set of interrupts, masking out any others. */
 269         for (core = 0; core < v3d->cores; core++) {
 270                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
 271                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver));
 272         }
 273
 274         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver));
 275         V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver));
 276 }
 277
 278 void
 279 v3d_irq_disable(struct v3d_dev *v3d)
 280 {
 281         int core;
 282
 283         /* Disable all interrupts. */
 284         for (core = 0; core < v3d->cores; core++)
 285                 V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
 286         V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
 287
 288         /* Clear any pending interrupts we might have left. */
 289         for (core = 0; core < v3d->cores; core++)
 290                 V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
 291         V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 292
 293         cancel_work_sync(&v3d->overflow_mem_work);
 294 }
 295
 296 /** Reinitializes interrupt registers when a GPU reset is performed. */
 297 void v3d_irq_reset(struct v3d_dev *v3d)
 298 {
 299         v3d_irq_enable(v3d);
 300 }