drivers/gpu/drm/panfrost/panfrost_drv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright 2018 Marty E. Plummer <[email protected]> */
   3 /* Copyright 2019 Linaro, Ltd., Rob Herring <[email protected]> */
   4 /* Copyright 2019 Collabora ltd. */
   5
   6 #ifdef CONFIG_ARM_ARCH_TIMER
   7 #include <asm/arch_timer.h>
   8 #endif
   9
  10 #include <linux/module.h>
  11 #include <linux/of.h>
  12 #include <linux/pagemap.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/pm_runtime.h>
  15 #include <drm/panfrost_drm.h>
  16 #include <drm/drm_drv.h>
  17 #include <drm/drm_ioctl.h>
  18 #include <drm/drm_syncobj.h>
  19 #include <drm/drm_utils.h>
  20
  21 #include "panfrost_device.h"
  22 #include "panfrost_gem.h"
  23 #include "panfrost_mmu.h"
  24 #include "panfrost_job.h"
  25 #include "panfrost_gpu.h"
  26 #include "panfrost_perfcnt.h"
  27
  28 #define JOB_REQUIREMENTS (PANFROST_JD_REQ_FS | PANFROST_JD_REQ_CYCLE_COUNT)
  29
  30 static bool unstable_ioctls;
  31 module_param_unsafe(unstable_ioctls, bool, 0600);
  32
  33 static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev,
  34                                           u64 *arg)
  35 {
  36         int ret;
  37
  38         ret = pm_runtime_resume_and_get(pfdev->dev);
  39         if (ret)
  40                 return ret;
  41
  42         panfrost_cycle_counter_get(pfdev);
  43         *arg = panfrost_timestamp_read(pfdev);
  44         panfrost_cycle_counter_put(pfdev);
  45
  46         pm_runtime_put(pfdev->dev);
  47         return 0;
  48 }
  49
  50 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file)
  51 {
  52         struct drm_panfrost_get_param *param = data;
  53         struct panfrost_device *pfdev = ddev->dev_private;
  54         int ret;
  55
  56         if (param->pad != 0)
  57                 return -EINVAL;
  58
  59 #define PANFROST_FEATURE(name, member)                  \
  60         case DRM_PANFROST_PARAM_ ## name:               \
  61                 param->value = pfdev->features.member;  \
  62                 break
  63 #define PANFROST_FEATURE_ARRAY(name, member, max)                       \
  64         case DRM_PANFROST_PARAM_ ## name ## 0 ...                       \
  65                 DRM_PANFROST_PARAM_ ## name ## max:                     \
  66                 param->value = pfdev->features.member[param->param -    \
  67                         DRM_PANFROST_PARAM_ ## name ## 0];              \
  68                 break
  69
  70         switch (param->param) {
  71                 PANFROST_FEATURE(GPU_PROD_ID, id);
  72                 PANFROST_FEATURE(GPU_REVISION, revision);
  73                 PANFROST_FEATURE(SHADER_PRESENT, shader_present);
  74                 PANFROST_FEATURE(TILER_PRESENT, tiler_present);
  75                 PANFROST_FEATURE(L2_PRESENT, l2_present);
  76                 PANFROST_FEATURE(STACK_PRESENT, stack_present);
  77                 PANFROST_FEATURE(AS_PRESENT, as_present);
  78                 PANFROST_FEATURE(JS_PRESENT, js_present);
  79                 PANFROST_FEATURE(L2_FEATURES, l2_features);
  80                 PANFROST_FEATURE(CORE_FEATURES, core_features);
  81                 PANFROST_FEATURE(TILER_FEATURES, tiler_features);
  82                 PANFROST_FEATURE(MEM_FEATURES, mem_features);
  83                 PANFROST_FEATURE(MMU_FEATURES, mmu_features);
  84                 PANFROST_FEATURE(THREAD_FEATURES, thread_features);
  85                 PANFROST_FEATURE(MAX_THREADS, max_threads);
  86                 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ,
  87                                 thread_max_workgroup_sz);
  88                 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ,
  89                                 thread_max_barrier_sz);
  90                 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features);
  91                 PANFROST_FEATURE(AFBC_FEATURES, afbc_features);
  92                 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3);
  93                 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15);
  94                 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups);
  95                 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc);
  96
  97         case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP:
  98                 ret = panfrost_ioctl_query_timestamp(pfdev, &param->value);
  99                 if (ret)
 100                         return ret;
 101                 break;
 102
 103         case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY:
 104 #ifdef CONFIG_ARM_ARCH_TIMER
 105                 param->value = arch_timer_get_cntfrq();
 106 #else
 107                 param->value = 0;
 108 #endif
 109                 break;
 110
 111         default:
 112                 return -EINVAL;
 113         }
 114
 115         return 0;
 116 }
 117
 118 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
 119                 struct drm_file *file)
 120 {
 121         struct panfrost_file_priv *priv = file->driver_priv;
 122         struct panfrost_gem_object *bo;
 123         struct drm_panfrost_create_bo *args = data;
 124         struct panfrost_gem_mapping *mapping;
 125         int ret;
 126
 127         if (!args->size || args->pad ||
 128             (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP)))
 129                 return -EINVAL;
 130
 131         /* Heaps should never be executable */
 132         if ((args->flags & PANFROST_BO_HEAP) &&
 133             !(args->flags & PANFROST_BO_NOEXEC))
 134                 return -EINVAL;
 135
 136         bo = panfrost_gem_create(dev, args->size, args->flags);
 137         if (IS_ERR(bo))
 138                 return PTR_ERR(bo);
 139
 140         ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
 141         if (ret)
 142                 goto out;
 143
 144         mapping = panfrost_gem_mapping_get(bo, priv);
 145         if (mapping) {
 146                 args->offset = mapping->mmnode.start << PAGE_SHIFT;
 147                 panfrost_gem_mapping_put(mapping);
 148         } else {
 149                 /* This can only happen if the handle from
 150                  * drm_gem_handle_create() has already been guessed and freed
 151                  * by user space
 152                  */
 153                 ret = -EINVAL;
 154         }
 155
 156 out:
 157         drm_gem_object_put(&bo->base.base);
 158         return ret;
 159 }
 160
 161 /**
 162  * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects
 163  * referenced by the job.
 164  * @dev: DRM device
 165  * @file_priv: DRM file for this fd
 166  * @args: IOCTL args
 167  * @job: job being set up
 168  *
 169  * Resolve handles from userspace to BOs and attach them to job.
 170  *
 171  * Note that this function doesn't need to unreference the BOs on
 172  * failure, because that will happen at panfrost_job_cleanup() time.
 173  */
 174 static int
 175 panfrost_lookup_bos(struct drm_device *dev,
 176                   struct drm_file *file_priv,
 177                   struct drm_panfrost_submit *args,
 178                   struct panfrost_job *job)
 179 {
 180         struct panfrost_file_priv *priv = file_priv->driver_priv;
 181         struct panfrost_gem_object *bo;
 182         unsigned int i;
 183         int ret;
 184
 185         job->bo_count = args->bo_handle_count;
 186
 187         if (!job->bo_count)
 188                 return 0;
 189
 190         ret = drm_gem_objects_lookup(file_priv,
 191                                      (void __user *)(uintptr_t)args->bo_handles,
 192                                      job->bo_count, &job->bos);
 193         if (ret)
 194                 return ret;
 195
 196         job->mappings = kvmalloc_array(job->bo_count,
 197                                        sizeof(struct panfrost_gem_mapping *),
 198                                        GFP_KERNEL | __GFP_ZERO);
 199         if (!job->mappings)
 200                 return -ENOMEM;
 201
 202         for (i = 0; i < job->bo_count; i++) {
 203                 struct panfrost_gem_mapping *mapping;
 204
 205                 bo = to_panfrost_bo(job->bos[i]);
 206                 mapping = panfrost_gem_mapping_get(bo, priv);
 207                 if (!mapping) {
 208                         ret = -EINVAL;
 209                         break;
 210                 }
 211
 212                 atomic_inc(&bo->gpu_usecount);
 213                 job->mappings[i] = mapping;
 214         }
 215
 216         return ret;
 217 }
 218
 219 /**
 220  * panfrost_copy_in_sync() - Sets up job->deps with the sync objects
 221  * referenced by the job.
 222  * @dev: DRM device
 223  * @file_priv: DRM file for this fd
 224  * @args: IOCTL args
 225  * @job: job being set up
 226  *
 227  * Resolve syncobjs from userspace to fences and attach them to job.
 228  *
 229  * Note that this function doesn't need to unreference the fences on
 230  * failure, because that will happen at panfrost_job_cleanup() time.
 231  */
 232 static int
 233 panfrost_copy_in_sync(struct drm_device *dev,
 234                   struct drm_file *file_priv,
 235                   struct drm_panfrost_submit *args,
 236                   struct panfrost_job *job)
 237 {
 238         u32 *handles;
 239         int ret = 0;
 240         int i, in_fence_count;
 241
 242         in_fence_count = args->in_sync_count;
 243
 244         if (!in_fence_count)
 245                 return 0;
 246
 247         handles = kvmalloc_array(in_fence_count, sizeof(u32), GFP_KERNEL);
 248         if (!handles) {
 249                 ret = -ENOMEM;
 250                 DRM_DEBUG("Failed to allocate incoming syncobj handles\n");
 251                 goto fail;
 252         }
 253
 254         if (copy_from_user(handles,
 255                            (void __user *)(uintptr_t)args->in_syncs,
 256                            in_fence_count * sizeof(u32))) {
 257                 ret = -EFAULT;
 258                 DRM_DEBUG("Failed to copy in syncobj handles\n");
 259                 goto fail;
 260         }
 261
 262         for (i = 0; i < in_fence_count; i++) {
 263                 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv,
 264                                                            handles[i], 0);
 265                 if (ret)
 266                         goto fail;
 267         }
 268
 269 fail:
 270         kvfree(handles);
 271         return ret;
 272 }
 273
 274 static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
 275                 struct drm_file *file)
 276 {
 277         struct panfrost_device *pfdev = dev->dev_private;
 278         struct panfrost_file_priv *file_priv = file->driver_priv;
 279         struct drm_panfrost_submit *args = data;
 280         struct drm_syncobj *sync_out = NULL;
 281         struct panfrost_job *job;
 282         int ret = 0, slot;
 283
 284         if (!args->jc)
 285                 return -EINVAL;
 286
 287         if (args->requirements & ~JOB_REQUIREMENTS)
 288                 return -EINVAL;
 289
 290         if (args->out_sync > 0) {
 291                 sync_out = drm_syncobj_find(file, args->out_sync);
 292                 if (!sync_out)
 293                         return -ENODEV;
 294         }
 295
 296         job = kzalloc(sizeof(*job), GFP_KERNEL);
 297         if (!job) {
 298                 ret = -ENOMEM;
 299                 goto out_put_syncout;
 300         }
 301
 302         kref_init(&job->refcount);
 303
 304         job->pfdev = pfdev;
 305         job->jc = args->jc;
 306         job->requirements = args->requirements;
 307         job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev);
 308         job->mmu = file_priv->mmu;
 309         job->engine_usage = &file_priv->engine_usage;
 310
 311         slot = panfrost_job_get_slot(job);
 312
 313         ret = drm_sched_job_init(&job->base,
 314                                  &file_priv->sched_entity[slot],
 315                                  1, NULL);
 316         if (ret)
 317                 goto out_put_job;
 318
 319         ret = panfrost_copy_in_sync(dev, file, args, job);
 320         if (ret)
 321                 goto out_cleanup_job;
 322
 323         ret = panfrost_lookup_bos(dev, file, args, job);
 324         if (ret)
 325                 goto out_cleanup_job;
 326
 327         ret = panfrost_job_push(job);
 328         if (ret)
 329                 goto out_cleanup_job;
 330
 331         /* Update the return sync object for the job */
 332         if (sync_out)
 333                 drm_syncobj_replace_fence(sync_out, job->render_done_fence);
 334
 335 out_cleanup_job:
 336         if (ret)
 337                 drm_sched_job_cleanup(&job->base);
 338 out_put_job:
 339         panfrost_job_put(job);
 340 out_put_syncout:
 341         if (sync_out)
 342                 drm_syncobj_put(sync_out);
 343
 344         return ret;
 345 }
 346
 347 static int
 348 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data,
 349                        struct drm_file *file_priv)
 350 {
 351         long ret;
 352         struct drm_panfrost_wait_bo *args = data;
 353         struct drm_gem_object *gem_obj;
 354         unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
 355
 356         if (args->pad)
 357                 return -EINVAL;
 358
 359         gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 360         if (!gem_obj)
 361                 return -ENOENT;
 362
 363         ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_READ,
 364                                     true, timeout);
 365         if (!ret)
 366                 ret = timeout ? -ETIMEDOUT : -EBUSY;
 367
 368         drm_gem_object_put(gem_obj);
 369
 370         return ret;
 371 }
 372
 373 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data,
 374                       struct drm_file *file_priv)
 375 {
 376         struct drm_panfrost_mmap_bo *args = data;
 377         struct drm_gem_object *gem_obj;
 378         int ret;
 379
 380         if (args->flags != 0) {
 381                 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
 382                 return -EINVAL;
 383         }
 384
 385         gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 386         if (!gem_obj) {
 387                 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 388                 return -ENOENT;
 389         }
 390
 391         /* Don't allow mmapping of heap objects as pages are not pinned. */
 392         if (to_panfrost_bo(gem_obj)->is_heap) {
 393                 ret = -EINVAL;
 394                 goto out;
 395         }
 396
 397         ret = drm_gem_create_mmap_offset(gem_obj);
 398         if (ret == 0)
 399                 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
 400
 401 out:
 402         drm_gem_object_put(gem_obj);
 403         return ret;
 404 }
 405
 406 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data,
 407                             struct drm_file *file_priv)
 408 {
 409         struct panfrost_file_priv *priv = file_priv->driver_priv;
 410         struct drm_panfrost_get_bo_offset *args = data;
 411         struct panfrost_gem_mapping *mapping;
 412         struct drm_gem_object *gem_obj;
 413         struct panfrost_gem_object *bo;
 414
 415         gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 416         if (!gem_obj) {
 417                 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 418                 return -ENOENT;
 419         }
 420         bo = to_panfrost_bo(gem_obj);
 421
 422         mapping = panfrost_gem_mapping_get(bo, priv);
 423         drm_gem_object_put(gem_obj);
 424
 425         if (!mapping)
 426                 return -EINVAL;
 427
 428         args->offset = mapping->mmnode.start << PAGE_SHIFT;
 429         panfrost_gem_mapping_put(mapping);
 430         return 0;
 431 }
 432
 433 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data,
 434                                   struct drm_file *file_priv)
 435 {
 436         struct panfrost_file_priv *priv = file_priv->driver_priv;
 437         struct drm_panfrost_madvise *args = data;
 438         struct panfrost_device *pfdev = dev->dev_private;
 439         struct drm_gem_object *gem_obj;
 440         struct panfrost_gem_object *bo;
 441         int ret = 0;
 442
 443         gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 444         if (!gem_obj) {
 445                 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 446                 return -ENOENT;
 447         }
 448
 449         bo = to_panfrost_bo(gem_obj);
 450
 451         ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL);
 452         if (ret)
 453                 goto out_put_object;
 454
 455         mutex_lock(&pfdev->shrinker_lock);
 456         mutex_lock(&bo->mappings.lock);
 457         if (args->madv == PANFROST_MADV_DONTNEED) {
 458                 struct panfrost_gem_mapping *first;
 459
 460                 first = list_first_entry(&bo->mappings.list,
 461                                          struct panfrost_gem_mapping,
 462                                          node);
 463
 464                 /*
 465                  * If we want to mark the BO purgeable, there must be only one
 466                  * user: the caller FD.
 467                  * We could do something smarter and mark the BO purgeable only
 468                  * when all its users have marked it purgeable, but globally
 469                  * visible/shared BOs are likely to never be marked purgeable
 470                  * anyway, so let's not bother.
 471                  */
 472                 if (!list_is_singular(&bo->mappings.list) ||
 473                     WARN_ON_ONCE(first->mmu != priv->mmu)) {
 474                         ret = -EINVAL;
 475                         goto out_unlock_mappings;
 476                 }
 477         }
 478
 479         args->retained = drm_gem_shmem_madvise(&bo->base, args->madv);
 480
 481         if (args->retained) {
 482                 if (args->madv == PANFROST_MADV_DONTNEED)
 483                         list_move_tail(&bo->base.madv_list,
 484                                        &pfdev->shrinker_list);
 485                 else if (args->madv == PANFROST_MADV_WILLNEED)
 486                         list_del_init(&bo->base.madv_list);
 487         }
 488
 489 out_unlock_mappings:
 490         mutex_unlock(&bo->mappings.lock);
 491         mutex_unlock(&pfdev->shrinker_lock);
 492         dma_resv_unlock(bo->base.base.resv);
 493 out_put_object:
 494         drm_gem_object_put(gem_obj);
 495         return ret;
 496 }
 497
 498 int panfrost_unstable_ioctl_check(void)
 499 {
 500         if (!unstable_ioctls)
 501                 return -ENOSYS;
 502
 503         return 0;
 504 }
 505
 506 static int
 507 panfrost_open(struct drm_device *dev, struct drm_file *file)
 508 {
 509         int ret;
 510         struct panfrost_device *pfdev = dev->dev_private;
 511         struct panfrost_file_priv *panfrost_priv;
 512
 513         panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL);
 514         if (!panfrost_priv)
 515                 return -ENOMEM;
 516
 517         panfrost_priv->pfdev = pfdev;
 518         file->driver_priv = panfrost_priv;
 519
 520         panfrost_priv->mmu = panfrost_mmu_ctx_create(pfdev);
 521         if (IS_ERR(panfrost_priv->mmu)) {
 522                 ret = PTR_ERR(panfrost_priv->mmu);
 523                 goto err_free;
 524         }
 525
 526         ret = panfrost_job_open(panfrost_priv);
 527         if (ret)
 528                 goto err_job;
 529
 530         return 0;
 531
 532 err_job:
 533         panfrost_mmu_ctx_put(panfrost_priv->mmu);
 534 err_free:
 535         kfree(panfrost_priv);
 536         return ret;
 537 }
 538
 539 static void
 540 panfrost_postclose(struct drm_device *dev, struct drm_file *file)
 541 {
 542         struct panfrost_file_priv *panfrost_priv = file->driver_priv;
 543
 544         panfrost_perfcnt_close(file);
 545         panfrost_job_close(panfrost_priv);
 546
 547         panfrost_mmu_ctx_put(panfrost_priv->mmu);
 548         kfree(panfrost_priv);
 549 }
 550
 551 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = {
 552 #define PANFROST_IOCTL(n, func, flags) \
 553         DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags)
 554
 555         PANFROST_IOCTL(SUBMIT,          submit,         DRM_RENDER_ALLOW),
 556         PANFROST_IOCTL(WAIT_BO,         wait_bo,        DRM_RENDER_ALLOW),
 557         PANFROST_IOCTL(CREATE_BO,       create_bo,      DRM_RENDER_ALLOW),
 558         PANFROST_IOCTL(MMAP_BO,         mmap_bo,        DRM_RENDER_ALLOW),
 559         PANFROST_IOCTL(GET_PARAM,       get_param,      DRM_RENDER_ALLOW),
 560         PANFROST_IOCTL(GET_BO_OFFSET,   get_bo_offset,  DRM_RENDER_ALLOW),
 561         PANFROST_IOCTL(PERFCNT_ENABLE,  perfcnt_enable, DRM_RENDER_ALLOW),
 562         PANFROST_IOCTL(PERFCNT_DUMP,    perfcnt_dump,   DRM_RENDER_ALLOW),
 563         PANFROST_IOCTL(MADVISE,         madvise,        DRM_RENDER_ALLOW),
 564 };
 565
 566 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev,
 567                                      struct panfrost_file_priv *panfrost_priv,
 568                                      struct drm_printer *p)
 569 {
 570         int i;
 571
 572         /*
 573          * IMPORTANT NOTE: drm-cycles and drm-engine measurements are not
 574          * accurate, as they only provide a rough estimation of the number of
 575          * GPU cycles and CPU time spent in a given context. This is due to two
 576          * different factors:
 577          * - Firstly, we must consider the time the CPU and then the kernel
 578          *   takes to process the GPU interrupt, which means additional time and
 579          *   GPU cycles will be added in excess to the real figure.
 580          * - Secondly, the pipelining done by the Job Manager (2 job slots per
 581          *   engine) implies there is no way to know exactly how much time each
 582          *   job spent on the GPU.
 583          */
 584
 585         static const char * const engine_names[] = {
 586                 "fragment", "vertex-tiler", "compute-only"
 587         };
 588
 589         BUILD_BUG_ON(ARRAY_SIZE(engine_names) != NUM_JOB_SLOTS);
 590
 591         for (i = 0; i < NUM_JOB_SLOTS - 1; i++) {
 592                 if (pfdev->profile_mode) {
 593                         drm_printf(p, "drm-engine-%s:\t%llu ns\n",
 594                                    engine_names[i], panfrost_priv->engine_usage.elapsed_ns[i]);
 595                         drm_printf(p, "drm-cycles-%s:\t%llu\n",
 596                                    engine_names[i], panfrost_priv->engine_usage.cycles[i]);
 597                 }
 598                 drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n",
 599                            engine_names[i], pfdev->pfdevfreq.fast_rate);
 600                 drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n",
 601                            engine_names[i], pfdev->pfdevfreq.current_frequency);
 602         }
 603 }
 604
 605 static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file)
 606 {
 607         struct drm_device *dev = file->minor->dev;
 608         struct panfrost_device *pfdev = dev->dev_private;
 609
 610         panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p);
 611
 612         drm_show_memory_stats(p, file);
 613 }
 614
 615 static const struct file_operations panfrost_drm_driver_fops = {
 616         .owner = THIS_MODULE,
 617         DRM_GEM_FOPS,
 618         .show_fdinfo = drm_show_fdinfo,
 619 };
 620
 621 /*
 622  * Panfrost driver version:
 623  * - 1.0 - initial interface
 624  * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO
 625  * - 1.2 - adds AFBC_FEATURES query
 626  * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT
 627  *       - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries
 628  */
 629 static const struct drm_driver panfrost_drm_driver = {
 630         .driver_features        = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ,
 631         .open                   = panfrost_open,
 632         .postclose              = panfrost_postclose,
 633         .show_fdinfo            = panfrost_show_fdinfo,
 634         .ioctls                 = panfrost_drm_driver_ioctls,
 635         .num_ioctls             = ARRAY_SIZE(panfrost_drm_driver_ioctls),
 636         .fops                   = &panfrost_drm_driver_fops,
 637         .name                   = "panfrost",
 638         .desc                   = "panfrost DRM",
 639         .date                   = "20180908",
 640         .major                  = 1,
 641         .minor                  = 3,
 642
 643         .gem_create_object      = panfrost_gem_create_object,
 644         .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
 645 };
 646
 647 static int panfrost_probe(struct platform_device *pdev)
 648 {
 649         struct panfrost_device *pfdev;
 650         struct drm_device *ddev;
 651         int err;
 652
 653         pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL);
 654         if (!pfdev)
 655                 return -ENOMEM;
 656
 657         pfdev->pdev = pdev;
 658         pfdev->dev = &pdev->dev;
 659
 660         platform_set_drvdata(pdev, pfdev);
 661
 662         pfdev->comp = of_device_get_match_data(&pdev->dev);
 663         if (!pfdev->comp)
 664                 return -ENODEV;
 665
 666         pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT;
 667
 668         /* Allocate and initialize the DRM device. */
 669         ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev);
 670         if (IS_ERR(ddev))
 671                 return PTR_ERR(ddev);
 672
 673         ddev->dev_private = pfdev;
 674         pfdev->ddev = ddev;
 675
 676         mutex_init(&pfdev->shrinker_lock);
 677         INIT_LIST_HEAD(&pfdev->shrinker_list);
 678
 679         err = panfrost_device_init(pfdev);
 680         if (err) {
 681                 if (err != -EPROBE_DEFER)
 682                         dev_err(&pdev->dev, "Fatal error during GPU init\n");
 683                 goto err_out0;
 684         }
 685
 686         pm_runtime_set_active(pfdev->dev);
 687         pm_runtime_mark_last_busy(pfdev->dev);
 688         pm_runtime_enable(pfdev->dev);
 689         pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */
 690         pm_runtime_use_autosuspend(pfdev->dev);
 691
 692         /*
 693          * Register the DRM device with the core and the connectors with
 694          * sysfs
 695          */
 696         err = drm_dev_register(ddev, 0);
 697         if (err < 0)
 698                 goto err_out1;
 699
 700         err = panfrost_gem_shrinker_init(ddev);
 701         if (err)
 702                 goto err_out2;
 703
 704         return 0;
 705
 706 err_out2:
 707         drm_dev_unregister(ddev);
 708 err_out1:
 709         pm_runtime_disable(pfdev->dev);
 710         panfrost_device_fini(pfdev);
 711         pm_runtime_set_suspended(pfdev->dev);
 712 err_out0:
 713         drm_dev_put(ddev);
 714         return err;
 715 }
 716
 717 static void panfrost_remove(struct platform_device *pdev)
 718 {
 719         struct panfrost_device *pfdev = platform_get_drvdata(pdev);
 720         struct drm_device *ddev = pfdev->ddev;
 721
 722         drm_dev_unregister(ddev);
 723         panfrost_gem_shrinker_cleanup(ddev);
 724
 725         pm_runtime_get_sync(pfdev->dev);
 726         pm_runtime_disable(pfdev->dev);
 727         panfrost_device_fini(pfdev);
 728         pm_runtime_set_suspended(pfdev->dev);
 729
 730         drm_dev_put(ddev);
 731 }
 732
 733 static ssize_t profiling_show(struct device *dev,
 734                               struct device_attribute *attr, char *buf)
 735 {
 736         struct panfrost_device *pfdev = dev_get_drvdata(dev);
 737
 738         return sysfs_emit(buf, "%d\n", pfdev->profile_mode);
 739 }
 740
 741 static ssize_t profiling_store(struct device *dev,
 742                                struct device_attribute *attr,
 743                                const char *buf, size_t len)
 744 {
 745         struct panfrost_device *pfdev = dev_get_drvdata(dev);
 746         bool value;
 747         int err;
 748
 749         err = kstrtobool(buf, &value);
 750         if (err)
 751                 return err;
 752
 753         pfdev->profile_mode = value;
 754
 755         return len;
 756 }
 757
 758 static DEVICE_ATTR_RW(profiling);
 759
 760 static struct attribute *panfrost_attrs[] = {
 761         &dev_attr_profiling.attr,
 762         NULL,
 763 };
 764
 765 ATTRIBUTE_GROUPS(panfrost);
 766
 767 /*
 768  * The OPP core wants the supply names to be NULL terminated, but we need the
 769  * correct num_supplies value for regulator core. Hence, we NULL terminate here
 770  * and then initialize num_supplies with ARRAY_SIZE - 1.
 771  */
 772 static const char * const default_supplies[] = { "mali", NULL };
 773 static const struct panfrost_compatible default_data = {
 774         .num_supplies = ARRAY_SIZE(default_supplies) - 1,
 775         .supply_names = default_supplies,
 776         .num_pm_domains = 1, /* optional */
 777         .pm_domain_names = NULL,
 778 };
 779
 780 static const struct panfrost_compatible amlogic_data = {
 781         .num_supplies = ARRAY_SIZE(default_supplies) - 1,
 782         .supply_names = default_supplies,
 783         .vendor_quirk = panfrost_gpu_amlogic_quirk,
 784 };
 785
 786 /*
 787  * The old data with two power supplies for MT8183 is here only to
 788  * keep retro-compatibility with older devicetrees, as DVFS will
 789  * not work with this one.
 790  *
 791  * On new devicetrees please use the _b variant with a single and
 792  * coupled regulators instead.
 793  */
 794 static const char * const mediatek_mt8183_supplies[] = { "mali", "sram", NULL };
 795 static const char * const mediatek_mt8183_pm_domains[] = { "core0", "core1", "core2" };
 796 static const struct panfrost_compatible mediatek_mt8183_data = {
 797         .num_supplies = ARRAY_SIZE(mediatek_mt8183_supplies) - 1,
 798         .supply_names = mediatek_mt8183_supplies,
 799         .num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains),
 800         .pm_domain_names = mediatek_mt8183_pm_domains,
 801 };
 802
 803 static const char * const mediatek_mt8183_b_supplies[] = { "mali", NULL };
 804 static const struct panfrost_compatible mediatek_mt8183_b_data = {
 805         .num_supplies = ARRAY_SIZE(mediatek_mt8183_b_supplies) - 1,
 806         .supply_names = mediatek_mt8183_b_supplies,
 807         .num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains),
 808         .pm_domain_names = mediatek_mt8183_pm_domains,
 809         .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 810 };
 811
 812 static const char * const mediatek_mt8186_pm_domains[] = { "core0", "core1" };
 813 static const struct panfrost_compatible mediatek_mt8186_data = {
 814         .num_supplies = ARRAY_SIZE(mediatek_mt8183_b_supplies) - 1,
 815         .supply_names = mediatek_mt8183_b_supplies,
 816         .num_pm_domains = ARRAY_SIZE(mediatek_mt8186_pm_domains),
 817         .pm_domain_names = mediatek_mt8186_pm_domains,
 818         .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 819 };
 820
 821 /* MT8188 uses the same power domains and power supplies as MT8183 */
 822 static const struct panfrost_compatible mediatek_mt8188_data = {
 823         .num_supplies = ARRAY_SIZE(mediatek_mt8183_b_supplies) - 1,
 824         .supply_names = mediatek_mt8183_b_supplies,
 825         .num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains),
 826         .pm_domain_names = mediatek_mt8183_pm_domains,
 827         .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 828 };
 829
 830 static const char * const mediatek_mt8192_supplies[] = { "mali", NULL };
 831 static const char * const mediatek_mt8192_pm_domains[] = { "core0", "core1", "core2",
 832                                                            "core3", "core4" };
 833 static const struct panfrost_compatible mediatek_mt8192_data = {
 834         .num_supplies = ARRAY_SIZE(mediatek_mt8192_supplies) - 1,
 835         .supply_names = mediatek_mt8192_supplies,
 836         .num_pm_domains = ARRAY_SIZE(mediatek_mt8192_pm_domains),
 837         .pm_domain_names = mediatek_mt8192_pm_domains,
 838         .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 839 };
 840
 841 static const struct of_device_id dt_match[] = {
 842         /* Set first to probe before the generic compatibles */
 843         { .compatible = "amlogic,meson-gxm-mali",
 844           .data = &amlogic_data, },
 845         { .compatible = "amlogic,meson-g12a-mali",
 846           .data = &amlogic_data, },
 847         { .compatible = "arm,mali-t604", .data = &default_data, },
 848         { .compatible = "arm,mali-t624", .data = &default_data, },
 849         { .compatible = "arm,mali-t628", .data = &default_data, },
 850         { .compatible = "arm,mali-t720", .data = &default_data, },
 851         { .compatible = "arm,mali-t760", .data = &default_data, },
 852         { .compatible = "arm,mali-t820", .data = &default_data, },
 853         { .compatible = "arm,mali-t830", .data = &default_data, },
 854         { .compatible = "arm,mali-t860", .data = &default_data, },
 855         { .compatible = "arm,mali-t880", .data = &default_data, },
 856         { .compatible = "arm,mali-bifrost", .data = &default_data, },
 857         { .compatible = "arm,mali-valhall-jm", .data = &default_data, },
 858         { .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data },
 859         { .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data },
 860         { .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data },
 861         { .compatible = "mediatek,mt8188-mali", .data = &mediatek_mt8188_data },
 862         { .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data },
 863         {}
 864 };
 865 MODULE_DEVICE_TABLE(of, dt_match);
 866
 867 static struct platform_driver panfrost_driver = {
 868         .probe          = panfrost_probe,
 869         .remove         = panfrost_remove,
 870         .driver         = {
 871                 .name   = "panfrost",
 872                 .pm     = pm_ptr(&panfrost_pm_ops),
 873                 .of_match_table = dt_match,
 874                 .dev_groups = panfrost_groups,
 875         },
 876 };
 877 module_platform_driver(panfrost_driver);
 878
 879 MODULE_AUTHOR("Panfrost Project Developers");
 880 MODULE_DESCRIPTION("Panfrost DRM Driver");
 881 MODULE_LICENSE("GPL v2");
 882 MODULE_SOFTDEP("pre: governor_simpleondemand");