drivers/gpu/drm/panfrost/panfrost_perfcnt.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright 2019 Collabora Ltd */
   3
   4 #include <drm/drm_file.h>
   5 #include <drm/drm_gem_shmem_helper.h>
   6 #include <drm/panfrost_drm.h>
   7 #include <linux/completion.h>
   8 #include <linux/dma-buf-map.h>
   9 #include <linux/iopoll.h>
  10 #include <linux/pm_runtime.h>
  11 #include <linux/slab.h>
  12 #include <linux/uaccess.h>
  13
  14 #include "panfrost_device.h"
  15 #include "panfrost_features.h"
  16 #include "panfrost_gem.h"
  17 #include "panfrost_issues.h"
  18 #include "panfrost_job.h"
  19 #include "panfrost_mmu.h"
  20 #include "panfrost_perfcnt.h"
  21 #include "panfrost_regs.h"
  22
  23 #define COUNTERS_PER_BLOCK              64
  24 #define BYTES_PER_COUNTER               4
  25 #define BLOCKS_PER_COREGROUP            8
  26 #define V4_SHADERS_PER_COREGROUP        4
  27
  28 struct panfrost_perfcnt {
  29         struct panfrost_gem_mapping *mapping;
  30         size_t bosize;
  31         void *buf;
  32         struct panfrost_file_priv *user;
  33         struct mutex lock;
  34         struct completion dump_comp;
  35 };
  36
  37 void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
  38 {
  39         complete(&pfdev->perfcnt->dump_comp);
  40 }
  41
  42 void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
  43 {
  44         gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
  45 }
  46
  47 static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
  48 {
  49         u64 gpuva;
  50         int ret;
  51
  52         reinit_completion(&pfdev->perfcnt->dump_comp);
  53         gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
  54         gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
  55         gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
  56         gpu_write(pfdev, GPU_INT_CLEAR,
  57                   GPU_IRQ_CLEAN_CACHES_COMPLETED |
  58                   GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
  59         gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
  60         ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
  61                                                         msecs_to_jiffies(1000));
  62         if (!ret)
  63                 ret = -ETIMEDOUT;
  64         else if (ret > 0)
  65                 ret = 0;
  66
  67         return ret;
  68 }
  69
  70 static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
  71                                           struct drm_file *file_priv,
  72                                           unsigned int counterset)
  73 {
  74         struct panfrost_file_priv *user = file_priv->driver_priv;
  75         struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
  76         struct dma_buf_map map;
  77         struct drm_gem_shmem_object *bo;
  78         u32 cfg, as;
  79         int ret;
  80
  81         if (user == perfcnt->user)
  82                 return 0;
  83         else if (perfcnt->user)
  84                 return -EBUSY;
  85
  86         ret = pm_runtime_get_sync(pfdev->dev);
  87         if (ret < 0)
  88                 goto err_put_pm;
  89
  90         bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
  91         if (IS_ERR(bo)) {
  92                 ret = PTR_ERR(bo);
  93                 goto err_put_pm;
  94         }
  95
  96         /* Map the perfcnt buf in the address space attached to file_priv. */
  97         ret = panfrost_gem_open(&bo->base, file_priv);
  98         if (ret)
  99                 goto err_put_bo;
 100
 101         perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
 102                                                     user);
 103         if (!perfcnt->mapping) {
 104                 ret = -EINVAL;
 105                 goto err_close_bo;
 106         }
 107
 108         ret = drm_gem_shmem_vmap(&bo->base, &map);
 109         if (ret)
 110                 goto err_put_mapping;
 111         perfcnt->buf = map.vaddr;
 112
 113         /*
 114          * Invalidate the cache and clear the counters to start from a fresh
 115          * state.
 116          */
 117         reinit_completion(&pfdev->perfcnt->dump_comp);
 118         gpu_write(pfdev, GPU_INT_CLEAR,
 119                   GPU_IRQ_CLEAN_CACHES_COMPLETED |
 120                   GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
 121         gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
 122         gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
 123         ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
 124                                           msecs_to_jiffies(1000));
 125         if (!ret) {
 126                 ret = -ETIMEDOUT;
 127                 goto err_vunmap;
 128         }
 129
 130         perfcnt->user = user;
 131
 132         as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
 133         cfg = GPU_PERFCNT_CFG_AS(as) |
 134               GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
 135
 136         /*
 137          * Bifrost GPUs have 2 set of counters, but we're only interested by
 138          * the first one for now.
 139          */
 140         if (panfrost_model_is_bifrost(pfdev))
 141                 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
 142
 143         gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
 144         gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
 145         gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
 146
 147         /*
 148          * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
 149          * counters.
 150          */
 151         if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
 152                 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 153         else
 154                 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
 155
 156         gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
 157
 158         if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
 159                 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
 160
 161         /* The BO ref is retained by the mapping. */
 162         drm_gem_object_put(&bo->base);
 163
 164         return 0;
 165
 166 err_vunmap:
 167         drm_gem_shmem_vunmap(&bo->base, &map);
 168 err_put_mapping:
 169         panfrost_gem_mapping_put(perfcnt->mapping);
 170 err_close_bo:
 171         panfrost_gem_close(&bo->base, file_priv);
 172 err_put_bo:
 173         drm_gem_object_put(&bo->base);
 174 err_put_pm:
 175         pm_runtime_put(pfdev->dev);
 176         return ret;
 177 }
 178
 179 static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
 180                                            struct drm_file *file_priv)
 181 {
 182         struct panfrost_file_priv *user = file_priv->driver_priv;
 183         struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 184         struct dma_buf_map map = DMA_BUF_MAP_INIT_VADDR(perfcnt->buf);
 185
 186         if (user != perfcnt->user)
 187                 return -EINVAL;
 188
 189         gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
 190         gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
 191         gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
 192         gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 193         gpu_write(pfdev, GPU_PERFCNT_CFG,
 194                   GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
 195
 196         perfcnt->user = NULL;
 197         drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, &map);
 198         perfcnt->buf = NULL;
 199         panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
 200         panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
 201         panfrost_gem_mapping_put(perfcnt->mapping);
 202         perfcnt->mapping = NULL;
 203         pm_runtime_mark_last_busy(pfdev->dev);
 204         pm_runtime_put_autosuspend(pfdev->dev);
 205
 206         return 0;
 207 }
 208
 209 int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
 210                                   struct drm_file *file_priv)
 211 {
 212         struct panfrost_device *pfdev = dev->dev_private;
 213         struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 214         struct drm_panfrost_perfcnt_enable *req = data;
 215         int ret;
 216
 217         ret = panfrost_unstable_ioctl_check();
 218         if (ret)
 219                 return ret;
 220
 221         /* Only Bifrost GPUs have 2 set of counters. */
 222         if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
 223                 return -EINVAL;
 224
 225         mutex_lock(&perfcnt->lock);
 226         if (req->enable)
 227                 ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
 228                                                      req->counterset);
 229         else
 230                 ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
 231         mutex_unlock(&perfcnt->lock);
 232
 233         return ret;
 234 }
 235
 236 int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
 237                                 struct drm_file *file_priv)
 238 {
 239         struct panfrost_device *pfdev = dev->dev_private;
 240         struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 241         struct drm_panfrost_perfcnt_dump *req = data;
 242         void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
 243         int ret;
 244
 245         ret = panfrost_unstable_ioctl_check();
 246         if (ret)
 247                 return ret;
 248
 249         mutex_lock(&perfcnt->lock);
 250         if (perfcnt->user != file_priv->driver_priv) {
 251                 ret = -EINVAL;
 252                 goto out;
 253         }
 254
 255         ret = panfrost_perfcnt_dump_locked(pfdev);
 256         if (ret)
 257                 goto out;
 258
 259         if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
 260                 ret = -EFAULT;
 261
 262 out:
 263         mutex_unlock(&perfcnt->lock);
 264
 265         return ret;
 266 }
 267
 268 void panfrost_perfcnt_close(struct drm_file *file_priv)
 269 {
 270         struct panfrost_file_priv *pfile = file_priv->driver_priv;
 271         struct panfrost_device *pfdev = pfile->pfdev;
 272         struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 273
 274         pm_runtime_get_sync(pfdev->dev);
 275         mutex_lock(&perfcnt->lock);
 276         if (perfcnt->user == pfile)
 277                 panfrost_perfcnt_disable_locked(pfdev, file_priv);
 278         mutex_unlock(&perfcnt->lock);
 279         pm_runtime_mark_last_busy(pfdev->dev);
 280         pm_runtime_put_autosuspend(pfdev->dev);
 281 }
 282
 283 int panfrost_perfcnt_init(struct panfrost_device *pfdev)
 284 {
 285         struct panfrost_perfcnt *perfcnt;
 286         size_t size;
 287
 288         if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
 289                 unsigned int ncoregroups;
 290
 291                 ncoregroups = hweight64(pfdev->features.l2_present);
 292                 size = ncoregroups * BLOCKS_PER_COREGROUP *
 293                        COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
 294         } else {
 295                 unsigned int nl2c, ncores;
 296
 297                 /*
 298                  * TODO: define a macro to extract the number of l2 caches from
 299                  * mem_features.
 300                  */
 301                 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
 302
 303                 /*
 304                  * shader_present might be sparse, but the counters layout
 305                  * forces to dump unused regions too, hence the fls64() call
 306                  * instead of hweight64().
 307                  */
 308                 ncores = fls64(pfdev->features.shader_present);
 309
 310                 /*
 311                  * There's always one JM and one Tiler block, hence the '+ 2'
 312                  * here.
 313                  */
 314                 size = (nl2c + ncores + 2) *
 315                        COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
 316         }
 317
 318         perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
 319         if (!perfcnt)
 320                 return -ENOMEM;
 321
 322         perfcnt->bosize = size;
 323
 324         /* Start with everything disabled. */
 325         gpu_write(pfdev, GPU_PERFCNT_CFG,
 326                   GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
 327         gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
 328         gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
 329         gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
 330         gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 331
 332         init_completion(&perfcnt->dump_comp);
 333         mutex_init(&perfcnt->lock);
 334         pfdev->perfcnt = perfcnt;
 335
 336         return 0;
 337 }
 338
 339 void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
 340 {
 341         /* Disable everything before leaving. */
 342         gpu_write(pfdev, GPU_PERFCNT_CFG,
 343                   GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
 344         gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
 345         gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
 346         gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
 347         gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 348 }