drivers/gpu/drm/tegra/drm.c

   1 /*
   2  * Copyright (C) 2012 Avionic Design GmbH
   3  * Copyright (C) 2012-2016 NVIDIA CORPORATION.  All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  */
   9
  10 #include <linux/bitops.h>
  11 #include <linux/host1x.h>
  12 #include <linux/idr.h>
  13 #include <linux/iommu.h>
  14
  15 #include <drm/drm_atomic.h>
  16 #include <drm/drm_atomic_helper.h>
  17
  18 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
  19 #include <asm/dma-iommu.h>
  20 #endif
  21
  22 #include "drm.h"
  23 #include "gem.h"
  24
  25 #define DRIVER_NAME "tegra"
  26 #define DRIVER_DESC "NVIDIA Tegra graphics"
  27 #define DRIVER_DATE "20120330"
  28 #define DRIVER_MAJOR 0
  29 #define DRIVER_MINOR 0
  30 #define DRIVER_PATCHLEVEL 0
  31
  32 #define CARVEOUT_SZ SZ_64M
  33 #define CDMA_GATHER_FETCHES_MAX_NB 16383
  34
  35 struct tegra_drm_file {
  36         struct idr contexts;
  37         struct mutex lock;
  38 };
  39
  40 static int tegra_atomic_check(struct drm_device *drm,
  41                               struct drm_atomic_state *state)
  42 {
  43         int err;
  44
  45         err = drm_atomic_helper_check(drm, state);
  46         if (err < 0)
  47                 return err;
  48
  49         return tegra_display_hub_atomic_check(drm, state);
  50 }
  51
  52 static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {
  53         .fb_create = tegra_fb_create,
  54 #ifdef CONFIG_DRM_FBDEV_EMULATION
  55         .output_poll_changed = drm_fb_helper_output_poll_changed,
  56 #endif
  57         .atomic_check = tegra_atomic_check,
  58         .atomic_commit = drm_atomic_helper_commit,
  59 };
  60
  61 static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state)
  62 {
  63         struct drm_device *drm = old_state->dev;
  64         struct tegra_drm *tegra = drm->dev_private;
  65
  66         if (tegra->hub) {
  67                 drm_atomic_helper_commit_modeset_disables(drm, old_state);
  68                 tegra_display_hub_atomic_commit(drm, old_state);
  69                 drm_atomic_helper_commit_planes(drm, old_state, 0);
  70                 drm_atomic_helper_commit_modeset_enables(drm, old_state);
  71                 drm_atomic_helper_commit_hw_done(old_state);
  72                 drm_atomic_helper_wait_for_vblanks(drm, old_state);
  73                 drm_atomic_helper_cleanup_planes(drm, old_state);
  74         } else {
  75                 drm_atomic_helper_commit_tail_rpm(old_state);
  76         }
  77 }
  78
  79 static const struct drm_mode_config_helper_funcs
  80 tegra_drm_mode_config_helpers = {
  81         .atomic_commit_tail = tegra_atomic_commit_tail,
  82 };
  83
  84 static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
  85 {
  86         struct host1x_device *device = to_host1x_device(drm->dev);
  87         struct tegra_drm *tegra;
  88         int err;
  89
  90         tegra = kzalloc(sizeof(*tegra), GFP_KERNEL);
  91         if (!tegra)
  92                 return -ENOMEM;
  93
  94         if (iommu_present(&platform_bus_type)) {
  95                 tegra->domain = iommu_domain_alloc(&platform_bus_type);
  96                 if (!tegra->domain) {
  97                         err = -ENOMEM;
  98                         goto free;
  99                 }
 100
 101                 err = iova_cache_get();
 102                 if (err < 0)
 103                         goto domain;
 104         }
 105
 106         mutex_init(&tegra->clients_lock);
 107         INIT_LIST_HEAD(&tegra->clients);
 108
 109         drm->dev_private = tegra;
 110         tegra->drm = drm;
 111
 112         drm_mode_config_init(drm);
 113
 114         drm->mode_config.min_width = 0;
 115         drm->mode_config.min_height = 0;
 116
 117         drm->mode_config.max_width = 4096;
 118         drm->mode_config.max_height = 4096;
 119
 120         drm->mode_config.allow_fb_modifiers = true;
 121
 122         drm->mode_config.normalize_zpos = true;
 123
 124         drm->mode_config.funcs = &tegra_drm_mode_config_funcs;
 125         drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;
 126
 127         err = tegra_drm_fb_prepare(drm);
 128         if (err < 0)
 129                 goto config;
 130
 131         drm_kms_helper_poll_init(drm);
 132
 133         err = host1x_device_init(device);
 134         if (err < 0)
 135                 goto fbdev;
 136
 137         if (tegra->domain) {
 138                 u64 carveout_start, carveout_end, gem_start, gem_end;
 139                 u64 dma_mask = dma_get_mask(&device->dev);
 140                 dma_addr_t start, end;
 141                 unsigned long order;
 142
 143                 start = tegra->domain->geometry.aperture_start & dma_mask;
 144                 end = tegra->domain->geometry.aperture_end & dma_mask;
 145
 146                 gem_start = start;
 147                 gem_end = end - CARVEOUT_SZ;
 148                 carveout_start = gem_end + 1;
 149                 carveout_end = end;
 150
 151                 order = __ffs(tegra->domain->pgsize_bitmap);
 152                 init_iova_domain(&tegra->carveout.domain, 1UL << order,
 153                                  carveout_start >> order);
 154
 155                 tegra->carveout.shift = iova_shift(&tegra->carveout.domain);
 156                 tegra->carveout.limit = carveout_end >> tegra->carveout.shift;
 157
 158                 drm_mm_init(&tegra->mm, gem_start, gem_end - gem_start + 1);
 159                 mutex_init(&tegra->mm_lock);
 160
 161                 DRM_DEBUG("IOMMU apertures:\n");
 162                 DRM_DEBUG("  GEM: %#llx-%#llx\n", gem_start, gem_end);
 163                 DRM_DEBUG("  Carveout: %#llx-%#llx\n", carveout_start,
 164                           carveout_end);
 165         }
 166
 167         if (tegra->hub) {
 168                 err = tegra_display_hub_prepare(tegra->hub);
 169                 if (err < 0)
 170                         goto device;
 171         }
 172
 173         /*
 174          * We don't use the drm_irq_install() helpers provided by the DRM
 175          * core, so we need to set this manually in order to allow the
 176          * DRM_IOCTL_WAIT_VBLANK to operate correctly.
 177          */
 178         drm->irq_enabled = true;
 179
 180         /* syncpoints are used for full 32-bit hardware VBLANK counters */
 181         drm->max_vblank_count = 0xffffffff;
 182
 183         err = drm_vblank_init(drm, drm->mode_config.num_crtc);
 184         if (err < 0)
 185                 goto hub;
 186
 187         drm_mode_config_reset(drm);
 188
 189         err = tegra_drm_fb_init(drm);
 190         if (err < 0)
 191                 goto hub;
 192
 193         return 0;
 194
 195 hub:
 196         if (tegra->hub)
 197                 tegra_display_hub_cleanup(tegra->hub);
 198 device:
 199         host1x_device_exit(device);
 200 fbdev:
 201         drm_kms_helper_poll_fini(drm);
 202         tegra_drm_fb_free(drm);
 203 config:
 204         drm_mode_config_cleanup(drm);
 205
 206         if (tegra->domain) {
 207                 mutex_destroy(&tegra->mm_lock);
 208                 drm_mm_takedown(&tegra->mm);
 209                 put_iova_domain(&tegra->carveout.domain);
 210                 iova_cache_put();
 211         }
 212 domain:
 213         if (tegra->domain)
 214                 iommu_domain_free(tegra->domain);
 215 free:
 216         kfree(tegra);
 217         return err;
 218 }
 219
 220 static void tegra_drm_unload(struct drm_device *drm)
 221 {
 222         struct host1x_device *device = to_host1x_device(drm->dev);
 223         struct tegra_drm *tegra = drm->dev_private;
 224         int err;
 225
 226         drm_kms_helper_poll_fini(drm);
 227         tegra_drm_fb_exit(drm);
 228         drm_atomic_helper_shutdown(drm);
 229         drm_mode_config_cleanup(drm);
 230
 231         err = host1x_device_exit(device);
 232         if (err < 0)
 233                 return;
 234
 235         if (tegra->domain) {
 236                 mutex_destroy(&tegra->mm_lock);
 237                 drm_mm_takedown(&tegra->mm);
 238                 put_iova_domain(&tegra->carveout.domain);
 239                 iova_cache_put();
 240                 iommu_domain_free(tegra->domain);
 241         }
 242
 243         kfree(tegra);
 244 }
 245
 246 static int tegra_drm_open(struct drm_device *drm, struct drm_file *filp)
 247 {
 248         struct tegra_drm_file *fpriv;
 249
 250         fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
 251         if (!fpriv)
 252                 return -ENOMEM;
 253
 254         idr_init(&fpriv->contexts);
 255         mutex_init(&fpriv->lock);
 256         filp->driver_priv = fpriv;
 257
 258         return 0;
 259 }
 260
 261 static void tegra_drm_context_free(struct tegra_drm_context *context)
 262 {
 263         context->client->ops->close_channel(context);
 264         kfree(context);
 265 }
 266
 267 static struct host1x_bo *
 268 host1x_bo_lookup(struct drm_file *file, u32 handle)
 269 {
 270         struct drm_gem_object *gem;
 271         struct tegra_bo *bo;
 272
 273         gem = drm_gem_object_lookup(file, handle);
 274         if (!gem)
 275                 return NULL;
 276
 277         bo = to_tegra_bo(gem);
 278         return &bo->base;
 279 }
 280
 281 static int host1x_reloc_copy_from_user(struct host1x_reloc *dest,
 282                                        struct drm_tegra_reloc __user *src,
 283                                        struct drm_device *drm,
 284                                        struct drm_file *file)
 285 {
 286         u32 cmdbuf, target;
 287         int err;
 288
 289         err = get_user(cmdbuf, &src->cmdbuf.handle);
 290         if (err < 0)
 291                 return err;
 292
 293         err = get_user(dest->cmdbuf.offset, &src->cmdbuf.offset);
 294         if (err < 0)
 295                 return err;
 296
 297         err = get_user(target, &src->target.handle);
 298         if (err < 0)
 299                 return err;
 300
 301         err = get_user(dest->target.offset, &src->target.offset);
 302         if (err < 0)
 303                 return err;
 304
 305         err = get_user(dest->shift, &src->shift);
 306         if (err < 0)
 307                 return err;
 308
 309         dest->cmdbuf.bo = host1x_bo_lookup(file, cmdbuf);
 310         if (!dest->cmdbuf.bo)
 311                 return -ENOENT;
 312
 313         dest->target.bo = host1x_bo_lookup(file, target);
 314         if (!dest->target.bo)
 315                 return -ENOENT;
 316
 317         return 0;
 318 }
 319
 320 int tegra_drm_submit(struct tegra_drm_context *context,
 321                      struct drm_tegra_submit *args, struct drm_device *drm,
 322                      struct drm_file *file)
 323 {
 324         struct host1x_client *client = &context->client->base;
 325         unsigned int num_cmdbufs = args->num_cmdbufs;
 326         unsigned int num_relocs = args->num_relocs;
 327         struct drm_tegra_cmdbuf __user *user_cmdbufs;
 328         struct drm_tegra_reloc __user *user_relocs;
 329         struct drm_tegra_syncpt __user *user_syncpt;
 330         struct drm_tegra_syncpt syncpt;
 331         struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
 332         struct drm_gem_object **refs;
 333         struct host1x_syncpt *sp;
 334         struct host1x_job *job;
 335         unsigned int num_refs;
 336         int err;
 337
 338         user_cmdbufs = u64_to_user_ptr(args->cmdbufs);
 339         user_relocs = u64_to_user_ptr(args->relocs);
 340         user_syncpt = u64_to_user_ptr(args->syncpts);
 341
 342         /* We don't yet support other than one syncpt_incr struct per submit */
 343         if (args->num_syncpts != 1)
 344                 return -EINVAL;
 345
 346         /* We don't yet support waitchks */
 347         if (args->num_waitchks != 0)
 348                 return -EINVAL;
 349
 350         job = host1x_job_alloc(context->channel, args->num_cmdbufs,
 351                                args->num_relocs);
 352         if (!job)
 353                 return -ENOMEM;
 354
 355         job->num_relocs = args->num_relocs;
 356         job->client = client;
 357         job->class = client->class;
 358         job->serialize = true;
 359
 360         /*
 361          * Track referenced BOs so that they can be unreferenced after the
 362          * submission is complete.
 363          */
 364         num_refs = num_cmdbufs + num_relocs * 2;
 365
 366         refs = kmalloc_array(num_refs, sizeof(*refs), GFP_KERNEL);
 367         if (!refs) {
 368                 err = -ENOMEM;
 369                 goto put;
 370         }
 371
 372         /* reuse as an iterator later */
 373         num_refs = 0;
 374
 375         while (num_cmdbufs) {
 376                 struct drm_tegra_cmdbuf cmdbuf;
 377                 struct host1x_bo *bo;
 378                 struct tegra_bo *obj;
 379                 u64 offset;
 380
 381                 if (copy_from_user(&cmdbuf, user_cmdbufs, sizeof(cmdbuf))) {
 382                         err = -EFAULT;
 383                         goto fail;
 384                 }
 385
 386                 /*
 387                  * The maximum number of CDMA gather fetches is 16383, a higher
 388                  * value means the words count is malformed.
 389                  */
 390                 if (cmdbuf.words > CDMA_GATHER_FETCHES_MAX_NB) {
 391                         err = -EINVAL;
 392                         goto fail;
 393                 }
 394
 395                 bo = host1x_bo_lookup(file, cmdbuf.handle);
 396                 if (!bo) {
 397                         err = -ENOENT;
 398                         goto fail;
 399                 }
 400
 401                 offset = (u64)cmdbuf.offset + (u64)cmdbuf.words * sizeof(u32);
 402                 obj = host1x_to_tegra_bo(bo);
 403                 refs[num_refs++] = &obj->gem;
 404
 405                 /*
 406                  * Gather buffer base address must be 4-bytes aligned,
 407                  * unaligned offset is malformed and cause commands stream
 408                  * corruption on the buffer address relocation.
 409                  */
 410                 if (offset & 3 || offset > obj->gem.size) {
 411                         err = -EINVAL;
 412                         goto fail;
 413                 }
 414
 415                 host1x_job_add_gather(job, bo, cmdbuf.words, cmdbuf.offset);
 416                 num_cmdbufs--;
 417                 user_cmdbufs++;
 418         }
 419
 420         /* copy and resolve relocations from submit */
 421         while (num_relocs--) {
 422                 struct host1x_reloc *reloc;
 423                 struct tegra_bo *obj;
 424
 425                 err = host1x_reloc_copy_from_user(&job->relocs[num_relocs],
 426                                                   &user_relocs[num_relocs], drm,
 427                                                   file);
 428                 if (err < 0)
 429                         goto fail;
 430
 431                 reloc = &job->relocs[num_relocs];
 432                 obj = host1x_to_tegra_bo(reloc->cmdbuf.bo);
 433                 refs[num_refs++] = &obj->gem;
 434
 435                 /*
 436                  * The unaligned cmdbuf offset will cause an unaligned write
 437                  * during of the relocations patching, corrupting the commands
 438                  * stream.
 439                  */
 440                 if (reloc->cmdbuf.offset & 3 ||
 441                     reloc->cmdbuf.offset >= obj->gem.size) {
 442                         err = -EINVAL;
 443                         goto fail;
 444                 }
 445
 446                 obj = host1x_to_tegra_bo(reloc->target.bo);
 447                 refs[num_refs++] = &obj->gem;
 448
 449                 if (reloc->target.offset >= obj->gem.size) {
 450                         err = -EINVAL;
 451                         goto fail;
 452                 }
 453         }
 454
 455         if (copy_from_user(&syncpt, user_syncpt, sizeof(syncpt))) {
 456                 err = -EFAULT;
 457                 goto fail;
 458         }
 459
 460         /* check whether syncpoint ID is valid */
 461         sp = host1x_syncpt_get(host1x, syncpt.id);
 462         if (!sp) {
 463                 err = -ENOENT;
 464                 goto fail;
 465         }
 466
 467         job->is_addr_reg = context->client->ops->is_addr_reg;
 468         job->is_valid_class = context->client->ops->is_valid_class;
 469         job->syncpt_incrs = syncpt.incrs;
 470         job->syncpt_id = syncpt.id;
 471         job->timeout = 10000;
 472
 473         if (args->timeout && args->timeout < 10000)
 474                 job->timeout = args->timeout;
 475
 476         err = host1x_job_pin(job, context->client->base.dev);
 477         if (err)
 478                 goto fail;
 479
 480         err = host1x_job_submit(job);
 481         if (err) {
 482                 host1x_job_unpin(job);
 483                 goto fail;
 484         }
 485
 486         args->fence = job->syncpt_end;
 487
 488 fail:
 489         while (num_refs--)
 490                 drm_gem_object_put_unlocked(refs[num_refs]);
 491
 492         kfree(refs);
 493
 494 put:
 495         host1x_job_put(job);
 496         return err;
 497 }
 498
 499
 500 #ifdef CONFIG_DRM_TEGRA_STAGING
 501 static int tegra_gem_create(struct drm_device *drm, void *data,
 502                             struct drm_file *file)
 503 {
 504         struct drm_tegra_gem_create *args = data;
 505         struct tegra_bo *bo;
 506
 507         bo = tegra_bo_create_with_handle(file, drm, args->size, args->flags,
 508                                          &args->handle);
 509         if (IS_ERR(bo))
 510                 return PTR_ERR(bo);
 511
 512         return 0;
 513 }
 514
 515 static int tegra_gem_mmap(struct drm_device *drm, void *data,
 516                           struct drm_file *file)
 517 {
 518         struct drm_tegra_gem_mmap *args = data;
 519         struct drm_gem_object *gem;
 520         struct tegra_bo *bo;
 521
 522         gem = drm_gem_object_lookup(file, args->handle);
 523         if (!gem)
 524                 return -EINVAL;
 525
 526         bo = to_tegra_bo(gem);
 527
 528         args->offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
 529
 530         drm_gem_object_put_unlocked(gem);
 531
 532         return 0;
 533 }
 534
 535 static int tegra_syncpt_read(struct drm_device *drm, void *data,
 536                              struct drm_file *file)
 537 {
 538         struct host1x *host = dev_get_drvdata(drm->dev->parent);
 539         struct drm_tegra_syncpt_read *args = data;
 540         struct host1x_syncpt *sp;
 541
 542         sp = host1x_syncpt_get(host, args->id);
 543         if (!sp)
 544                 return -EINVAL;
 545
 546         args->value = host1x_syncpt_read_min(sp);
 547         return 0;
 548 }
 549
 550 static int tegra_syncpt_incr(struct drm_device *drm, void *data,
 551                              struct drm_file *file)
 552 {
 553         struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
 554         struct drm_tegra_syncpt_incr *args = data;
 555         struct host1x_syncpt *sp;
 556
 557         sp = host1x_syncpt_get(host1x, args->id);
 558         if (!sp)
 559                 return -EINVAL;
 560
 561         return host1x_syncpt_incr(sp);
 562 }
 563
 564 static int tegra_syncpt_wait(struct drm_device *drm, void *data,
 565                              struct drm_file *file)
 566 {
 567         struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
 568         struct drm_tegra_syncpt_wait *args = data;
 569         struct host1x_syncpt *sp;
 570
 571         sp = host1x_syncpt_get(host1x, args->id);
 572         if (!sp)
 573                 return -EINVAL;
 574
 575         return host1x_syncpt_wait(sp, args->thresh,
 576                                   msecs_to_jiffies(args->timeout),
 577                                   &args->value);
 578 }
 579
 580 static int tegra_client_open(struct tegra_drm_file *fpriv,
 581                              struct tegra_drm_client *client,
 582                              struct tegra_drm_context *context)
 583 {
 584         int err;
 585
 586         err = client->ops->open_channel(client, context);
 587         if (err < 0)
 588                 return err;
 589
 590         err = idr_alloc(&fpriv->contexts, context, 1, 0, GFP_KERNEL);
 591         if (err < 0) {
 592                 client->ops->close_channel(context);
 593                 return err;
 594         }
 595
 596         context->client = client;
 597         context->id = err;
 598
 599         return 0;
 600 }
 601
 602 static int tegra_open_channel(struct drm_device *drm, void *data,
 603                               struct drm_file *file)
 604 {
 605         struct tegra_drm_file *fpriv = file->driver_priv;
 606         struct tegra_drm *tegra = drm->dev_private;
 607         struct drm_tegra_open_channel *args = data;
 608         struct tegra_drm_context *context;
 609         struct tegra_drm_client *client;
 610         int err = -ENODEV;
 611
 612         context = kzalloc(sizeof(*context), GFP_KERNEL);
 613         if (!context)
 614                 return -ENOMEM;
 615
 616         mutex_lock(&fpriv->lock);
 617
 618         list_for_each_entry(client, &tegra->clients, list)
 619                 if (client->base.class == args->client) {
 620                         err = tegra_client_open(fpriv, client, context);
 621                         if (err < 0)
 622                                 break;
 623
 624                         args->context = context->id;
 625                         break;
 626                 }
 627
 628         if (err < 0)
 629                 kfree(context);
 630
 631         mutex_unlock(&fpriv->lock);
 632         return err;
 633 }
 634
 635 static int tegra_close_channel(struct drm_device *drm, void *data,
 636                                struct drm_file *file)
 637 {
 638         struct tegra_drm_file *fpriv = file->driver_priv;
 639         struct drm_tegra_close_channel *args = data;
 640         struct tegra_drm_context *context;
 641         int err = 0;
 642
 643         mutex_lock(&fpriv->lock);
 644
 645         context = idr_find(&fpriv->contexts, args->context);
 646         if (!context) {
 647                 err = -EINVAL;
 648                 goto unlock;
 649         }
 650
 651         idr_remove(&fpriv->contexts, context->id);
 652         tegra_drm_context_free(context);
 653
 654 unlock:
 655         mutex_unlock(&fpriv->lock);
 656         return err;
 657 }
 658
 659 static int tegra_get_syncpt(struct drm_device *drm, void *data,
 660                             struct drm_file *file)
 661 {
 662         struct tegra_drm_file *fpriv = file->driver_priv;
 663         struct drm_tegra_get_syncpt *args = data;
 664         struct tegra_drm_context *context;
 665         struct host1x_syncpt *syncpt;
 666         int err = 0;
 667
 668         mutex_lock(&fpriv->lock);
 669
 670         context = idr_find(&fpriv->contexts, args->context);
 671         if (!context) {
 672                 err = -ENODEV;
 673                 goto unlock;
 674         }
 675
 676         if (args->index >= context->client->base.num_syncpts) {
 677                 err = -EINVAL;
 678                 goto unlock;
 679         }
 680
 681         syncpt = context->client->base.syncpts[args->index];
 682         args->id = host1x_syncpt_id(syncpt);
 683
 684 unlock:
 685         mutex_unlock(&fpriv->lock);
 686         return err;
 687 }
 688
 689 static int tegra_submit(struct drm_device *drm, void *data,
 690                         struct drm_file *file)
 691 {
 692         struct tegra_drm_file *fpriv = file->driver_priv;
 693         struct drm_tegra_submit *args = data;
 694         struct tegra_drm_context *context;
 695         int err;
 696
 697         mutex_lock(&fpriv->lock);
 698
 699         context = idr_find(&fpriv->contexts, args->context);
 700         if (!context) {
 701                 err = -ENODEV;
 702                 goto unlock;
 703         }
 704
 705         err = context->client->ops->submit(context, args, drm, file);
 706
 707 unlock:
 708         mutex_unlock(&fpriv->lock);
 709         return err;
 710 }
 711
 712 static int tegra_get_syncpt_base(struct drm_device *drm, void *data,
 713                                  struct drm_file *file)
 714 {
 715         struct tegra_drm_file *fpriv = file->driver_priv;
 716         struct drm_tegra_get_syncpt_base *args = data;
 717         struct tegra_drm_context *context;
 718         struct host1x_syncpt_base *base;
 719         struct host1x_syncpt *syncpt;
 720         int err = 0;
 721
 722         mutex_lock(&fpriv->lock);
 723
 724         context = idr_find(&fpriv->contexts, args->context);
 725         if (!context) {
 726                 err = -ENODEV;
 727                 goto unlock;
 728         }
 729
 730         if (args->syncpt >= context->client->base.num_syncpts) {
 731                 err = -EINVAL;
 732                 goto unlock;
 733         }
 734
 735         syncpt = context->client->base.syncpts[args->syncpt];
 736
 737         base = host1x_syncpt_get_base(syncpt);
 738         if (!base) {
 739                 err = -ENXIO;
 740                 goto unlock;
 741         }
 742
 743         args->id = host1x_syncpt_base_id(base);
 744
 745 unlock:
 746         mutex_unlock(&fpriv->lock);
 747         return err;
 748 }
 749
 750 static int tegra_gem_set_tiling(struct drm_device *drm, void *data,
 751                                 struct drm_file *file)
 752 {
 753         struct drm_tegra_gem_set_tiling *args = data;
 754         enum tegra_bo_tiling_mode mode;
 755         struct drm_gem_object *gem;
 756         unsigned long value = 0;
 757         struct tegra_bo *bo;
 758
 759         switch (args->mode) {
 760         case DRM_TEGRA_GEM_TILING_MODE_PITCH:
 761                 mode = TEGRA_BO_TILING_MODE_PITCH;
 762
 763                 if (args->value != 0)
 764                         return -EINVAL;
 765
 766                 break;
 767
 768         case DRM_TEGRA_GEM_TILING_MODE_TILED:
 769                 mode = TEGRA_BO_TILING_MODE_TILED;
 770
 771                 if (args->value != 0)
 772                         return -EINVAL;
 773
 774                 break;
 775
 776         case DRM_TEGRA_GEM_TILING_MODE_BLOCK:
 777                 mode = TEGRA_BO_TILING_MODE_BLOCK;
 778
 779                 if (args->value > 5)
 780                         return -EINVAL;
 781
 782                 value = args->value;
 783                 break;
 784
 785         default:
 786                 return -EINVAL;
 787         }
 788
 789         gem = drm_gem_object_lookup(file, args->handle);
 790         if (!gem)
 791                 return -ENOENT;
 792
 793         bo = to_tegra_bo(gem);
 794
 795         bo->tiling.mode = mode;
 796         bo->tiling.value = value;
 797
 798         drm_gem_object_put_unlocked(gem);
 799
 800         return 0;
 801 }
 802
 803 static int tegra_gem_get_tiling(struct drm_device *drm, void *data,
 804                                 struct drm_file *file)
 805 {
 806         struct drm_tegra_gem_get_tiling *args = data;
 807         struct drm_gem_object *gem;
 808         struct tegra_bo *bo;
 809         int err = 0;
 810
 811         gem = drm_gem_object_lookup(file, args->handle);
 812         if (!gem)
 813                 return -ENOENT;
 814
 815         bo = to_tegra_bo(gem);
 816
 817         switch (bo->tiling.mode) {
 818         case TEGRA_BO_TILING_MODE_PITCH:
 819                 args->mode = DRM_TEGRA_GEM_TILING_MODE_PITCH;
 820                 args->value = 0;
 821                 break;
 822
 823         case TEGRA_BO_TILING_MODE_TILED:
 824                 args->mode = DRM_TEGRA_GEM_TILING_MODE_TILED;
 825                 args->value = 0;
 826                 break;
 827
 828         case TEGRA_BO_TILING_MODE_BLOCK:
 829                 args->mode = DRM_TEGRA_GEM_TILING_MODE_BLOCK;
 830                 args->value = bo->tiling.value;
 831                 break;
 832
 833         default:
 834                 err = -EINVAL;
 835                 break;
 836         }
 837
 838         drm_gem_object_put_unlocked(gem);
 839
 840         return err;
 841 }
 842
 843 static int tegra_gem_set_flags(struct drm_device *drm, void *data,
 844                                struct drm_file *file)
 845 {
 846         struct drm_tegra_gem_set_flags *args = data;
 847         struct drm_gem_object *gem;
 848         struct tegra_bo *bo;
 849
 850         if (args->flags & ~DRM_TEGRA_GEM_FLAGS)
 851                 return -EINVAL;
 852
 853         gem = drm_gem_object_lookup(file, args->handle);
 854         if (!gem)
 855                 return -ENOENT;
 856
 857         bo = to_tegra_bo(gem);
 858         bo->flags = 0;
 859
 860         if (args->flags & DRM_TEGRA_GEM_BOTTOM_UP)
 861                 bo->flags |= TEGRA_BO_BOTTOM_UP;
 862
 863         drm_gem_object_put_unlocked(gem);
 864
 865         return 0;
 866 }
 867
 868 static int tegra_gem_get_flags(struct drm_device *drm, void *data,
 869                                struct drm_file *file)
 870 {
 871         struct drm_tegra_gem_get_flags *args = data;
 872         struct drm_gem_object *gem;
 873         struct tegra_bo *bo;
 874
 875         gem = drm_gem_object_lookup(file, args->handle);
 876         if (!gem)
 877                 return -ENOENT;
 878
 879         bo = to_tegra_bo(gem);
 880         args->flags = 0;
 881
 882         if (bo->flags & TEGRA_BO_BOTTOM_UP)
 883                 args->flags |= DRM_TEGRA_GEM_BOTTOM_UP;
 884
 885         drm_gem_object_put_unlocked(gem);
 886
 887         return 0;
 888 }
 889 #endif
 890
 891 static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
 892 #ifdef CONFIG_DRM_TEGRA_STAGING
 893         DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create,
 894                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 895         DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap,
 896                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 897         DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read,
 898                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 899         DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr,
 900                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 901         DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait,
 902                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 903         DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel,
 904                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 905         DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel,
 906                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 907         DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt,
 908                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 909         DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit,
 910                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 911         DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base,
 912                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 913         DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling,
 914                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 915         DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling,
 916                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 917         DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags,
 918                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 919         DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags,
 920                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 921 #endif
 922 };
 923
 924 static const struct file_operations tegra_drm_fops = {
 925         .owner = THIS_MODULE,
 926         .open = drm_open,
 927         .release = drm_release,
 928         .unlocked_ioctl = drm_ioctl,
 929         .mmap = tegra_drm_mmap,
 930         .poll = drm_poll,
 931         .read = drm_read,
 932         .compat_ioctl = drm_compat_ioctl,
 933         .llseek = noop_llseek,
 934 };
 935
 936 static int tegra_drm_context_cleanup(int id, void *p, void *data)
 937 {
 938         struct tegra_drm_context *context = p;
 939
 940         tegra_drm_context_free(context);
 941
 942         return 0;
 943 }
 944
 945 static void tegra_drm_postclose(struct drm_device *drm, struct drm_file *file)
 946 {
 947         struct tegra_drm_file *fpriv = file->driver_priv;
 948
 949         mutex_lock(&fpriv->lock);
 950         idr_for_each(&fpriv->contexts, tegra_drm_context_cleanup, NULL);
 951         mutex_unlock(&fpriv->lock);
 952
 953         idr_destroy(&fpriv->contexts);
 954         mutex_destroy(&fpriv->lock);
 955         kfree(fpriv);
 956 }
 957
 958 #ifdef CONFIG_DEBUG_FS
 959 static int tegra_debugfs_framebuffers(struct seq_file *s, void *data)
 960 {
 961         struct drm_info_node *node = (struct drm_info_node *)s->private;
 962         struct drm_device *drm = node->minor->dev;
 963         struct drm_framebuffer *fb;
 964
 965         mutex_lock(&drm->mode_config.fb_lock);
 966
 967         list_for_each_entry(fb, &drm->mode_config.fb_list, head) {
 968                 seq_printf(s, "%3d: user size: %d x %d, depth %d, %d bpp, refcount %d\n",
 969                            fb->base.id, fb->width, fb->height,
 970                            fb->format->depth,
 971                            fb->format->cpp[0] * 8,
 972                            drm_framebuffer_read_refcount(fb));
 973         }
 974
 975         mutex_unlock(&drm->mode_config.fb_lock);
 976
 977         return 0;
 978 }
 979
 980 static int tegra_debugfs_iova(struct seq_file *s, void *data)
 981 {
 982         struct drm_info_node *node = (struct drm_info_node *)s->private;
 983         struct drm_device *drm = node->minor->dev;
 984         struct tegra_drm *tegra = drm->dev_private;
 985         struct drm_printer p = drm_seq_file_printer(s);
 986
 987         if (tegra->domain) {
 988                 mutex_lock(&tegra->mm_lock);
 989                 drm_mm_print(&tegra->mm, &p);
 990                 mutex_unlock(&tegra->mm_lock);
 991         }
 992
 993         return 0;
 994 }
 995
 996 static struct drm_info_list tegra_debugfs_list[] = {
 997         { "framebuffers", tegra_debugfs_framebuffers, 0 },
 998         { "iova", tegra_debugfs_iova, 0 },
 999 };
1000
1001 static int tegra_debugfs_init(struct drm_minor *minor)
1002 {
1003         return drm_debugfs_create_files(tegra_debugfs_list,
1004                                         ARRAY_SIZE(tegra_debugfs_list),
1005                                         minor->debugfs_root, minor);
1006 }
1007 #endif
1008
1009 static struct drm_driver tegra_drm_driver = {
1010         .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
1011                            DRIVER_ATOMIC | DRIVER_RENDER,
1012         .load = tegra_drm_load,
1013         .unload = tegra_drm_unload,
1014         .open = tegra_drm_open,
1015         .postclose = tegra_drm_postclose,
1016         .lastclose = drm_fb_helper_lastclose,
1017
1018 #if defined(CONFIG_DEBUG_FS)
1019         .debugfs_init = tegra_debugfs_init,
1020 #endif
1021
1022         .gem_free_object_unlocked = tegra_bo_free_object,
1023         .gem_vm_ops = &tegra_bo_vm_ops,
1024
1025         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
1026         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
1027         .gem_prime_export = tegra_gem_prime_export,
1028         .gem_prime_import = tegra_gem_prime_import,
1029
1030         .dumb_create = tegra_bo_dumb_create,
1031
1032         .ioctls = tegra_drm_ioctls,
1033         .num_ioctls = ARRAY_SIZE(tegra_drm_ioctls),
1034         .fops = &tegra_drm_fops,
1035
1036         .name = DRIVER_NAME,
1037         .desc = DRIVER_DESC,
1038         .date = DRIVER_DATE,
1039         .major = DRIVER_MAJOR,
1040         .minor = DRIVER_MINOR,
1041         .patchlevel = DRIVER_PATCHLEVEL,
1042 };
1043
1044 int tegra_drm_register_client(struct tegra_drm *tegra,
1045                               struct tegra_drm_client *client)
1046 {
1047         mutex_lock(&tegra->clients_lock);
1048         list_add_tail(&client->list, &tegra->clients);
1049         client->drm = tegra;
1050         mutex_unlock(&tegra->clients_lock);
1051
1052         return 0;
1053 }
1054
1055 int tegra_drm_unregister_client(struct tegra_drm *tegra,
1056                                 struct tegra_drm_client *client)
1057 {
1058         mutex_lock(&tegra->clients_lock);
1059         list_del_init(&client->list);
1060         client->drm = NULL;
1061         mutex_unlock(&tegra->clients_lock);
1062
1063         return 0;
1064 }
1065
1066 struct iommu_group *host1x_client_iommu_attach(struct host1x_client *client,
1067                                                bool shared)
1068 {
1069         struct drm_device *drm = dev_get_drvdata(client->parent);
1070         struct tegra_drm *tegra = drm->dev_private;
1071         struct iommu_group *group = NULL;
1072         int err;
1073
1074         if (tegra->domain) {
1075                 group = iommu_group_get(client->dev);
1076                 if (!group) {
1077                         dev_err(client->dev, "failed to get IOMMU group\n");
1078                         return ERR_PTR(-ENODEV);
1079                 }
1080
1081                 if (!shared || (shared && (group != tegra->group))) {
1082 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)
1083                         if (client->dev->archdata.mapping) {
1084                                 struct dma_iommu_mapping *mapping =
1085                                         to_dma_iommu_mapping(client->dev);
1086                                 arm_iommu_detach_device(client->dev);
1087                                 arm_iommu_release_mapping(mapping);
1088                         }
1089 #endif
1090                         err = iommu_attach_group(tegra->domain, group);
1091                         if (err < 0) {
1092                                 iommu_group_put(group);
1093                                 return ERR_PTR(err);
1094                         }
1095
1096                         if (shared && !tegra->group)
1097                                 tegra->group = group;
1098                 }
1099         }
1100
1101         return group;
1102 }
1103
1104 void host1x_client_iommu_detach(struct host1x_client *client,
1105                                 struct iommu_group *group)
1106 {
1107         struct drm_device *drm = dev_get_drvdata(client->parent);
1108         struct tegra_drm *tegra = drm->dev_private;
1109
1110         if (group) {
1111                 if (group == tegra->group) {
1112                         iommu_detach_group(tegra->domain, group);
1113                         tegra->group = NULL;
1114                 }
1115
1116                 iommu_group_put(group);
1117         }
1118 }
1119
1120 void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size, dma_addr_t *dma)
1121 {
1122         struct iova *alloc;
1123         void *virt;
1124         gfp_t gfp;
1125         int err;
1126
1127         if (tegra->domain)
1128                 size = iova_align(&tegra->carveout.domain, size);
1129         else
1130                 size = PAGE_ALIGN(size);
1131
1132         gfp = GFP_KERNEL | __GFP_ZERO;
1133         if (!tegra->domain) {
1134                 /*
1135                  * Many units only support 32-bit addresses, even on 64-bit
1136                  * SoCs. If there is no IOMMU to translate into a 32-bit IO
1137                  * virtual address space, force allocations to be in the
1138                  * lower 32-bit range.
1139                  */
1140                 gfp |= GFP_DMA;
1141         }
1142
1143         virt = (void *)__get_free_pages(gfp, get_order(size));
1144         if (!virt)
1145                 return ERR_PTR(-ENOMEM);
1146
1147         if (!tegra->domain) {
1148                 /*
1149                  * If IOMMU is disabled, devices address physical memory
1150                  * directly.
1151                  */
1152                 *dma = virt_to_phys(virt);
1153                 return virt;
1154         }
1155
1156         alloc = alloc_iova(&tegra->carveout.domain,
1157                            size >> tegra->carveout.shift,
1158                            tegra->carveout.limit, true);
1159         if (!alloc) {
1160                 err = -EBUSY;
1161                 goto free_pages;
1162         }
1163
1164         *dma = iova_dma_addr(&tegra->carveout.domain, alloc);
1165         err = iommu_map(tegra->domain, *dma, virt_to_phys(virt),
1166                         size, IOMMU_READ | IOMMU_WRITE);
1167         if (err < 0)
1168                 goto free_iova;
1169
1170         return virt;
1171
1172 free_iova:
1173         __free_iova(&tegra->carveout.domain, alloc);
1174 free_pages:
1175         free_pages((unsigned long)virt, get_order(size));
1176
1177         return ERR_PTR(err);
1178 }
1179
1180 void tegra_drm_free(struct tegra_drm *tegra, size_t size, void *virt,
1181                     dma_addr_t dma)
1182 {
1183         if (tegra->domain)
1184                 size = iova_align(&tegra->carveout.domain, size);
1185         else
1186                 size = PAGE_ALIGN(size);
1187
1188         if (tegra->domain) {
1189                 iommu_unmap(tegra->domain, dma, size);
1190                 free_iova(&tegra->carveout.domain,
1191                           iova_pfn(&tegra->carveout.domain, dma));
1192         }
1193
1194         free_pages((unsigned long)virt, get_order(size));
1195 }
1196
1197 static int host1x_drm_probe(struct host1x_device *dev)
1198 {
1199         struct drm_driver *driver = &tegra_drm_driver;
1200         struct drm_device *drm;
1201         int err;
1202
1203         drm = drm_dev_alloc(driver, &dev->dev);
1204         if (IS_ERR(drm))
1205                 return PTR_ERR(drm);
1206
1207         dev_set_drvdata(&dev->dev, drm);
1208
1209         err = drm_fb_helper_remove_conflicting_framebuffers(NULL, "tegradrmfb", false);
1210         if (err < 0)
1211                 goto put;
1212
1213         err = drm_dev_register(drm, 0);
1214         if (err < 0)
1215                 goto put;
1216
1217         return 0;
1218
1219 put:
1220         drm_dev_put(drm);
1221         return err;
1222 }
1223
1224 static int host1x_drm_remove(struct host1x_device *dev)
1225 {
1226         struct drm_device *drm = dev_get_drvdata(&dev->dev);
1227
1228         drm_dev_unregister(drm);
1229         drm_dev_put(drm);
1230
1231         return 0;
1232 }
1233
1234 #ifdef CONFIG_PM_SLEEP
1235 static int host1x_drm_suspend(struct device *dev)
1236 {
1237         struct drm_device *drm = dev_get_drvdata(dev);
1238
1239         return drm_mode_config_helper_suspend(drm);
1240 }
1241
1242 static int host1x_drm_resume(struct device *dev)
1243 {
1244         struct drm_device *drm = dev_get_drvdata(dev);
1245
1246         return drm_mode_config_helper_resume(drm);
1247 }
1248 #endif
1249
1250 static SIMPLE_DEV_PM_OPS(host1x_drm_pm_ops, host1x_drm_suspend,
1251                          host1x_drm_resume);
1252
1253 static const struct of_device_id host1x_drm_subdevs[] = {
1254         { .compatible = "nvidia,tegra20-dc", },
1255         { .compatible = "nvidia,tegra20-hdmi", },
1256         { .compatible = "nvidia,tegra20-gr2d", },
1257         { .compatible = "nvidia,tegra20-gr3d", },
1258         { .compatible = "nvidia,tegra30-dc", },
1259         { .compatible = "nvidia,tegra30-hdmi", },
1260         { .compatible = "nvidia,tegra30-gr2d", },
1261         { .compatible = "nvidia,tegra30-gr3d", },
1262         { .compatible = "nvidia,tegra114-dsi", },
1263         { .compatible = "nvidia,tegra114-hdmi", },
1264         { .compatible = "nvidia,tegra114-gr3d", },
1265         { .compatible = "nvidia,tegra124-dc", },
1266         { .compatible = "nvidia,tegra124-sor", },
1267         { .compatible = "nvidia,tegra124-hdmi", },
1268         { .compatible = "nvidia,tegra124-dsi", },
1269         { .compatible = "nvidia,tegra124-vic", },
1270         { .compatible = "nvidia,tegra132-dsi", },
1271         { .compatible = "nvidia,tegra210-dc", },
1272         { .compatible = "nvidia,tegra210-dsi", },
1273         { .compatible = "nvidia,tegra210-sor", },
1274         { .compatible = "nvidia,tegra210-sor1", },
1275         { .compatible = "nvidia,tegra210-vic", },
1276         { .compatible = "nvidia,tegra186-display", },
1277         { .compatible = "nvidia,tegra186-dc", },
1278         { .compatible = "nvidia,tegra186-sor", },
1279         { .compatible = "nvidia,tegra186-sor1", },
1280         { .compatible = "nvidia,tegra186-vic", },
1281         { .compatible = "nvidia,tegra194-display", },
1282         { .compatible = "nvidia,tegra194-dc", },
1283         { .compatible = "nvidia,tegra194-sor", },
1284         { .compatible = "nvidia,tegra194-vic", },
1285         { /* sentinel */ }
1286 };
1287
1288 static struct host1x_driver host1x_drm_driver = {
1289         .driver = {
1290                 .name = "drm",
1291                 .pm = &host1x_drm_pm_ops,
1292         },
1293         .probe = host1x_drm_probe,
1294         .remove = host1x_drm_remove,
1295         .subdevs = host1x_drm_subdevs,
1296 };
1297
1298 static struct platform_driver * const drivers[] = {
1299         &tegra_display_hub_driver,
1300         &tegra_dc_driver,
1301         &tegra_hdmi_driver,
1302         &tegra_dsi_driver,
1303         &tegra_dpaux_driver,
1304         &tegra_sor_driver,
1305         &tegra_gr2d_driver,
1306         &tegra_gr3d_driver,
1307         &tegra_vic_driver,
1308 };
1309
1310 static int __init host1x_drm_init(void)
1311 {
1312         int err;
1313
1314         err = host1x_driver_register(&host1x_drm_driver);
1315         if (err < 0)
1316                 return err;
1317
1318         err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
1319         if (err < 0)
1320                 goto unregister_host1x;
1321
1322         return 0;
1323
1324 unregister_host1x:
1325         host1x_driver_unregister(&host1x_drm_driver);
1326         return err;
1327 }
1328 module_init(host1x_drm_init);
1329
1330 static void __exit host1x_drm_exit(void)
1331 {
1332         platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
1333         host1x_driver_unregister(&host1x_drm_driver);
1334 }
1335 module_exit(host1x_drm_exit);
1336
1337 MODULE_AUTHOR("Thierry Reding <[email protected]>");
1338 MODULE_DESCRIPTION("NVIDIA Tegra DRM driver");
1339 MODULE_LICENSE("GPL v2");