drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <[email protected]>
  29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30  *    Dave Airlie
  31  */
  32 #include <drm/ttm/ttm_bo_api.h>
  33 #include <drm/ttm/ttm_bo_driver.h>
  34 #include <drm/ttm/ttm_placement.h>
  35 #include <drm/ttm/ttm_module.h>
  36 #include <drm/ttm/ttm_page_alloc.h>
  37 #include <drm/drmP.h>
  38 #include <drm/amdgpu_drm.h>
  39 #include <linux/seq_file.h>
  40 #include <linux/slab.h>
  41 #include <linux/swiotlb.h>
  42 #include <linux/swap.h>
  43 #include <linux/pagemap.h>
  44 #include <linux/debugfs.h>
  45 #include <linux/iommu.h>
  46 #include "amdgpu.h"
  47 #include "amdgpu_object.h"
  48 #include "amdgpu_trace.h"
  49 #include "amdgpu_amdkfd.h"
  50 #include "bif/bif_4_1_d.h"
  51
  52 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
  53
  54 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
  55                              struct ttm_mem_reg *mem, unsigned num_pages,
  56                              uint64_t offset, unsigned window,
  57                              struct amdgpu_ring *ring,
  58                              uint64_t *addr);
  59
  60 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
  61 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
  62
  63 /*
  64  * Global memory.
  65  */
  66
  67 /**
  68  * amdgpu_ttm_mem_global_init - Initialize and acquire reference to
  69  * memory object
  70  *
  71  * @ref: Object for initialization.
  72  *
  73  * This is called by drm_global_item_ref() when an object is being
  74  * initialized.
  75  */
  76 static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref)
  77 {
  78         return ttm_mem_global_init(ref->object);
  79 }
  80
  81 /**
  82  * amdgpu_ttm_mem_global_release - Drop reference to a memory object
  83  *
  84  * @ref: Object being removed
  85  *
  86  * This is called by drm_global_item_unref() when an object is being
  87  * released.
  88  */
  89 static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
  90 {
  91         ttm_mem_global_release(ref->object);
  92 }
  93
  94 /**
  95  * amdgpu_ttm_global_init - Initialize global TTM memory reference
  96  *                                                      structures.
  97  *
  98  * @adev:       AMDGPU device for which the global structures need to be
  99  *                      registered.
 100  *
 101  * This is called as part of the AMDGPU ttm init from amdgpu_ttm_init()
 102  * during bring up.
 103  */
 104 static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 105 {
 106         struct drm_global_reference *global_ref;
 107         int r;
 108
 109         /* ensure reference is false in case init fails */
 110         adev->mman.mem_global_referenced = false;
 111
 112         global_ref = &adev->mman.mem_global_ref;
 113         global_ref->global_type = DRM_GLOBAL_TTM_MEM;
 114         global_ref->size = sizeof(struct ttm_mem_global);
 115         global_ref->init = &amdgpu_ttm_mem_global_init;
 116         global_ref->release = &amdgpu_ttm_mem_global_release;
 117         r = drm_global_item_ref(global_ref);
 118         if (r) {
 119                 DRM_ERROR("Failed setting up TTM memory accounting "
 120                           "subsystem.\n");
 121                 goto error_mem;
 122         }
 123
 124         adev->mman.bo_global_ref.mem_glob =
 125                 adev->mman.mem_global_ref.object;
 126         global_ref = &adev->mman.bo_global_ref.ref;
 127         global_ref->global_type = DRM_GLOBAL_TTM_BO;
 128         global_ref->size = sizeof(struct ttm_bo_global);
 129         global_ref->init = &ttm_bo_global_init;
 130         global_ref->release = &ttm_bo_global_release;
 131         r = drm_global_item_ref(global_ref);
 132         if (r) {
 133                 DRM_ERROR("Failed setting up TTM BO subsystem.\n");
 134                 goto error_bo;
 135         }
 136
 137         mutex_init(&adev->mman.gtt_window_lock);
 138
 139         adev->mman.mem_global_referenced = true;
 140
 141         return 0;
 142
 143 error_bo:
 144         drm_global_item_unref(&adev->mman.mem_global_ref);
 145 error_mem:
 146         return r;
 147 }
 148
 149 static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
 150 {
 151         if (adev->mman.mem_global_referenced) {
 152                 mutex_destroy(&adev->mman.gtt_window_lock);
 153                 drm_global_item_unref(&adev->mman.bo_global_ref.ref);
 154                 drm_global_item_unref(&adev->mman.mem_global_ref);
 155                 adev->mman.mem_global_referenced = false;
 156         }
 157 }
 158
 159 static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
 160 {
 161         return 0;
 162 }
 163
 164 /**
 165  * amdgpu_init_mem_type -       Initialize a memory manager for a specific
 166  *                                                      type of memory request.
 167  *
 168  * @bdev:       The TTM BO device object (contains a reference to
 169  *                      amdgpu_device)
 170  * @type:       The type of memory requested
 171  * @man:
 172  *
 173  * This is called by ttm_bo_init_mm() when a buffer object is being
 174  * initialized.
 175  */
 176 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 177                                 struct ttm_mem_type_manager *man)
 178 {
 179         struct amdgpu_device *adev;
 180
 181         adev = amdgpu_ttm_adev(bdev);
 182
 183         switch (type) {
 184         case TTM_PL_SYSTEM:
 185                 /* System memory */
 186                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 187                 man->available_caching = TTM_PL_MASK_CACHING;
 188                 man->default_caching = TTM_PL_FLAG_CACHED;
 189                 break;
 190         case TTM_PL_TT:
 191                 /* GTT memory  */
 192                 man->func = &amdgpu_gtt_mgr_func;
 193                 man->gpu_offset = adev->gmc.gart_start;
 194                 man->available_caching = TTM_PL_MASK_CACHING;
 195                 man->default_caching = TTM_PL_FLAG_CACHED;
 196                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
 197                 break;
 198         case TTM_PL_VRAM:
 199                 /* "On-card" video ram */
 200                 man->func = &amdgpu_vram_mgr_func;
 201                 man->gpu_offset = adev->gmc.vram_start;
 202                 man->flags = TTM_MEMTYPE_FLAG_FIXED |
 203                              TTM_MEMTYPE_FLAG_MAPPABLE;
 204                 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
 205                 man->default_caching = TTM_PL_FLAG_WC;
 206                 break;
 207         case AMDGPU_PL_GDS:
 208         case AMDGPU_PL_GWS:
 209         case AMDGPU_PL_OA:
 210                 /* On-chip GDS memory*/
 211                 man->func = &ttm_bo_manager_func;
 212                 man->gpu_offset = 0;
 213                 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
 214                 man->available_caching = TTM_PL_FLAG_UNCACHED;
 215                 man->default_caching = TTM_PL_FLAG_UNCACHED;
 216                 break;
 217         default:
 218                 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
 219                 return -EINVAL;
 220         }
 221         return 0;
 222 }
 223
 224 /**
 225  * amdgpu_evict_flags - Compute placement flags
 226  *
 227  * @bo: The buffer object to evict
 228  * @placement: Possible destination(s) for evicted BO
 229  *
 230  * Fill in placement data when ttm_bo_evict() is called
 231  */
 232 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 233                                 struct ttm_placement *placement)
 234 {
 235         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 236         struct amdgpu_bo *abo;
 237         static const struct ttm_place placements = {
 238                 .fpfn = 0,
 239                 .lpfn = 0,
 240                 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
 241         };
 242
 243         /* Don't handle scatter gather BOs */
 244         if (bo->type == ttm_bo_type_sg) {
 245                 placement->num_placement = 0;
 246                 placement->num_busy_placement = 0;
 247                 return;
 248         }
 249
 250         /* Object isn't an AMDGPU object so ignore */
 251         if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
 252                 placement->placement = &placements;
 253                 placement->busy_placement = &placements;
 254                 placement->num_placement = 1;
 255                 placement->num_busy_placement = 1;
 256                 return;
 257         }
 258
 259         abo = ttm_to_amdgpu_bo(bo);
 260         switch (bo->mem.mem_type) {
 261         case TTM_PL_VRAM:
 262                 if (!adev->mman.buffer_funcs_enabled) {
 263                         /* Move to system memory */
 264                         amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 265                 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 266                            !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
 267                            amdgpu_bo_in_cpu_visible_vram(abo)) {
 268
 269                         /* Try evicting to the CPU inaccessible part of VRAM
 270                          * first, but only set GTT as busy placement, so this
 271                          * BO will be evicted to GTT rather than causing other
 272                          * BOs to be evicted from VRAM
 273                          */
 274                         amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
 275                                                          AMDGPU_GEM_DOMAIN_GTT);
 276                         abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
 277                         abo->placements[0].lpfn = 0;
 278                         abo->placement.busy_placement = &abo->placements[1];
 279                         abo->placement.num_busy_placement = 1;
 280                 } else {
 281                         /* Move to GTT memory */
 282                         amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
 283                 }
 284                 break;
 285         case TTM_PL_TT:
 286         default:
 287                 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 288         }
 289         *placement = abo->placement;
 290 }
 291
 292 /**
 293  * amdgpu_verify_access - Verify access for a mmap call
 294  *
 295  * @bo:         The buffer object to map
 296  * @filp:       The file pointer from the process performing the mmap
 297  *
 298  * This is called by ttm_bo_mmap() to verify whether a process
 299  * has the right to mmap a BO to their process space.
 300  */
 301 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 302 {
 303         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
 304
 305         /*
 306          * Don't verify access for KFD BOs. They don't have a GEM
 307          * object associated with them.
 308          */
 309         if (abo->kfd_bo)
 310                 return 0;
 311
 312         if (amdgpu_ttm_tt_get_usermm(bo->ttm))
 313                 return -EPERM;
 314         return drm_vma_node_verify_access(&abo->gem_base.vma_node,
 315                                           filp->private_data);
 316 }
 317
 318 /**
 319  * amdgpu_move_null - Register memory for a buffer object
 320  *
 321  * @bo:                 The bo to assign the memory to
 322  * @new_mem:    The memory to be assigned.
 323  *
 324  * Assign the memory from new_mem to the memory of the buffer object
 325  * bo.
 326  */
 327 static void amdgpu_move_null(struct ttm_buffer_object *bo,
 328                              struct ttm_mem_reg *new_mem)
 329 {
 330         struct ttm_mem_reg *old_mem = &bo->mem;
 331
 332         BUG_ON(old_mem->mm_node != NULL);
 333         *old_mem = *new_mem;
 334         new_mem->mm_node = NULL;
 335 }
 336
 337 /**
 338  * amdgpu_mm_node_addr -        Compute the GPU relative offset of a GTT
 339  *                                                      buffer.
 340  */
 341 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 342                                     struct drm_mm_node *mm_node,
 343                                     struct ttm_mem_reg *mem)
 344 {
 345         uint64_t addr = 0;
 346
 347         if (mem->mem_type != TTM_PL_TT || amdgpu_gtt_mgr_has_gart_addr(mem)) {
 348                 addr = mm_node->start << PAGE_SHIFT;
 349                 addr += bo->bdev->man[mem->mem_type].gpu_offset;
 350         }
 351         return addr;
 352 }
 353
 354 /**
 355  * amdgpu_find_mm_node -        Helper function finds the drm_mm_node
 356  *                                              corresponding to @offset. It also modifies
 357  *                                                      the offset to be within the drm_mm_node
 358  *                                                      returned
 359  */
 360 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
 361                                                unsigned long *offset)
 362 {
 363         struct drm_mm_node *mm_node = mem->mm_node;
 364
 365         while (*offset >= (mm_node->size << PAGE_SHIFT)) {
 366                 *offset -= (mm_node->size << PAGE_SHIFT);
 367                 ++mm_node;
 368         }
 369         return mm_node;
 370 }
 371
 372 /**
 373  * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
 374  *
 375  * The function copies @size bytes from {src->mem + src->offset} to
 376  * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
 377  * move and different for a BO to BO copy.
 378  *
 379  * @f: Returns the last fence if multiple jobs are submitted.
 380  */
 381 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 382                                struct amdgpu_copy_mem *src,
 383                                struct amdgpu_copy_mem *dst,
 384                                uint64_t size,
 385                                struct reservation_object *resv,
 386                                struct dma_fence **f)
 387 {
 388         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 389         struct drm_mm_node *src_mm, *dst_mm;
 390         uint64_t src_node_start, dst_node_start, src_node_size,
 391                  dst_node_size, src_page_offset, dst_page_offset;
 392         struct dma_fence *fence = NULL;
 393         int r = 0;
 394         const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
 395                                         AMDGPU_GPU_PAGE_SIZE);
 396
 397         if (!adev->mman.buffer_funcs_enabled) {
 398                 DRM_ERROR("Trying to move memory with ring turned off.\n");
 399                 return -EINVAL;
 400         }
 401
 402         src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
 403         src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
 404                                              src->offset;
 405         src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
 406         src_page_offset = src_node_start & (PAGE_SIZE - 1);
 407
 408         dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
 409         dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
 410                                              dst->offset;
 411         dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
 412         dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
 413
 414         mutex_lock(&adev->mman.gtt_window_lock);
 415
 416         while (size) {
 417                 unsigned long cur_size;
 418                 uint64_t from = src_node_start, to = dst_node_start;
 419                 struct dma_fence *next;
 420
 421                 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
 422                  * begins at an offset, then adjust the size accordingly
 423                  */
 424                 cur_size = min3(min(src_node_size, dst_node_size), size,
 425                                 GTT_MAX_BYTES);
 426                 if (cur_size + src_page_offset > GTT_MAX_BYTES ||
 427                     cur_size + dst_page_offset > GTT_MAX_BYTES)
 428                         cur_size -= max(src_page_offset, dst_page_offset);
 429
 430                 /* Map only what needs to be accessed. Map src to window 0 and
 431                  * dst to window 1
 432                  */
 433                 if (src->mem->mem_type == TTM_PL_TT &&
 434                     !amdgpu_gtt_mgr_has_gart_addr(src->mem)) {
 435                         r = amdgpu_map_buffer(src->bo, src->mem,
 436                                         PFN_UP(cur_size + src_page_offset),
 437                                         src_node_start, 0, ring,
 438                                         &from);
 439                         if (r)
 440                                 goto error;
 441                         /* Adjust the offset because amdgpu_map_buffer returns
 442                          * start of mapped page
 443                          */
 444                         from += src_page_offset;
 445                 }
 446
 447                 if (dst->mem->mem_type == TTM_PL_TT &&
 448                     !amdgpu_gtt_mgr_has_gart_addr(dst->mem)) {
 449                         r = amdgpu_map_buffer(dst->bo, dst->mem,
 450                                         PFN_UP(cur_size + dst_page_offset),
 451                                         dst_node_start, 1, ring,
 452                                         &to);
 453                         if (r)
 454                                 goto error;
 455                         to += dst_page_offset;
 456                 }
 457
 458                 r = amdgpu_copy_buffer(ring, from, to, cur_size,
 459                                        resv, &next, false, true);
 460                 if (r)
 461                         goto error;
 462
 463                 dma_fence_put(fence);
 464                 fence = next;
 465
 466                 size -= cur_size;
 467                 if (!size)
 468                         break;
 469
 470                 src_node_size -= cur_size;
 471                 if (!src_node_size) {
 472                         src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
 473                                                              src->mem);
 474                         src_node_size = (src_mm->size << PAGE_SHIFT);
 475                 } else {
 476                         src_node_start += cur_size;
 477                         src_page_offset = src_node_start & (PAGE_SIZE - 1);
 478                 }
 479                 dst_node_size -= cur_size;
 480                 if (!dst_node_size) {
 481                         dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
 482                                                              dst->mem);
 483                         dst_node_size = (dst_mm->size << PAGE_SHIFT);
 484                 } else {
 485                         dst_node_start += cur_size;
 486                         dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
 487                 }
 488         }
 489 error:
 490         mutex_unlock(&adev->mman.gtt_window_lock);
 491         if (f)
 492                 *f = dma_fence_get(fence);
 493         dma_fence_put(fence);
 494         return r;
 495 }
 496
 497 /**
 498  * amdgpu_move_blit - Copy an entire buffer to another buffer
 499  *
 500  * This is a helper called by amdgpu_bo_move() and
 501  * amdgpu_move_vram_ram() to help move buffers to and from VRAM.
 502  */
 503 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 504                             bool evict, bool no_wait_gpu,
 505                             struct ttm_mem_reg *new_mem,
 506                             struct ttm_mem_reg *old_mem)
 507 {
 508         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 509         struct amdgpu_copy_mem src, dst;
 510         struct dma_fence *fence = NULL;
 511         int r;
 512
 513         src.bo = bo;
 514         dst.bo = bo;
 515         src.mem = old_mem;
 516         dst.mem = new_mem;
 517         src.offset = 0;
 518         dst.offset = 0;
 519
 520         r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
 521                                        new_mem->num_pages << PAGE_SHIFT,
 522                                        bo->resv, &fence);
 523         if (r)
 524                 goto error;
 525
 526         r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
 527         dma_fence_put(fence);
 528         return r;
 529
 530 error:
 531         if (fence)
 532                 dma_fence_wait(fence, false);
 533         dma_fence_put(fence);
 534         return r;
 535 }
 536
 537 /**
 538  * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
 539  *
 540  * Called by amdgpu_bo_move().
 541  */
 542 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
 543                                 struct ttm_operation_ctx *ctx,
 544                                 struct ttm_mem_reg *new_mem)
 545 {
 546         struct amdgpu_device *adev;
 547         struct ttm_mem_reg *old_mem = &bo->mem;
 548         struct ttm_mem_reg tmp_mem;
 549         struct ttm_place placements;
 550         struct ttm_placement placement;
 551         int r;
 552
 553         adev = amdgpu_ttm_adev(bo->bdev);
 554
 555         /* create space/pages for new_mem in GTT space */
 556         tmp_mem = *new_mem;
 557         tmp_mem.mm_node = NULL;
 558         placement.num_placement = 1;
 559         placement.placement = &placements;
 560         placement.num_busy_placement = 1;
 561         placement.busy_placement = &placements;
 562         placements.fpfn = 0;
 563         placements.lpfn = 0;
 564         placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 565         r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
 566         if (unlikely(r)) {
 567                 return r;
 568         }
 569
 570         /* set caching flags */
 571         r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
 572         if (unlikely(r)) {
 573                 goto out_cleanup;
 574         }
 575
 576         /* Bind the memory to the GTT space */
 577         r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
 578         if (unlikely(r)) {
 579                 goto out_cleanup;
 580         }
 581
 582         /* blit VRAM to GTT */
 583         r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, &tmp_mem, old_mem);
 584         if (unlikely(r)) {
 585                 goto out_cleanup;
 586         }
 587
 588         /* move BO (in tmp_mem) to new_mem */
 589         r = ttm_bo_move_ttm(bo, ctx, new_mem);
 590 out_cleanup:
 591         ttm_bo_mem_put(bo, &tmp_mem);
 592         return r;
 593 }
 594
 595 /**
 596  * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
 597  *
 598  * Called by amdgpu_bo_move().
 599  */
 600 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 601                                 struct ttm_operation_ctx *ctx,
 602                                 struct ttm_mem_reg *new_mem)
 603 {
 604         struct amdgpu_device *adev;
 605         struct ttm_mem_reg *old_mem = &bo->mem;
 606         struct ttm_mem_reg tmp_mem;
 607         struct ttm_placement placement;
 608         struct ttm_place placements;
 609         int r;
 610
 611         adev = amdgpu_ttm_adev(bo->bdev);
 612
 613         /* make space in GTT for old_mem buffer */
 614         tmp_mem = *new_mem;
 615         tmp_mem.mm_node = NULL;
 616         placement.num_placement = 1;
 617         placement.placement = &placements;
 618         placement.num_busy_placement = 1;
 619         placement.busy_placement = &placements;
 620         placements.fpfn = 0;
 621         placements.lpfn = 0;
 622         placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 623         r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
 624         if (unlikely(r)) {
 625                 return r;
 626         }
 627
 628         /* move/bind old memory to GTT space */
 629         r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
 630         if (unlikely(r)) {
 631                 goto out_cleanup;
 632         }
 633
 634         /* copy to VRAM */
 635         r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, new_mem, old_mem);
 636         if (unlikely(r)) {
 637                 goto out_cleanup;
 638         }
 639 out_cleanup:
 640         ttm_bo_mem_put(bo, &tmp_mem);
 641         return r;
 642 }
 643
 644 /**
 645  * amdgpu_bo_move - Move a buffer object to a new memory location
 646  *
 647  * Called by ttm_bo_handle_move_mem()
 648  */
 649 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 650                           struct ttm_operation_ctx *ctx,
 651                           struct ttm_mem_reg *new_mem)
 652 {
 653         struct amdgpu_device *adev;
 654         struct amdgpu_bo *abo;
 655         struct ttm_mem_reg *old_mem = &bo->mem;
 656         int r;
 657
 658         /* Can't move a pinned BO */
 659         abo = ttm_to_amdgpu_bo(bo);
 660         if (WARN_ON_ONCE(abo->pin_count > 0))
 661                 return -EINVAL;
 662
 663         adev = amdgpu_ttm_adev(bo->bdev);
 664
 665         if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
 666                 amdgpu_move_null(bo, new_mem);
 667                 return 0;
 668         }
 669         if ((old_mem->mem_type == TTM_PL_TT &&
 670              new_mem->mem_type == TTM_PL_SYSTEM) ||
 671             (old_mem->mem_type == TTM_PL_SYSTEM &&
 672              new_mem->mem_type == TTM_PL_TT)) {
 673                 /* bind is enough */
 674                 amdgpu_move_null(bo, new_mem);
 675                 return 0;
 676         }
 677
 678         if (!adev->mman.buffer_funcs_enabled)
 679                 goto memcpy;
 680
 681         if (old_mem->mem_type == TTM_PL_VRAM &&
 682             new_mem->mem_type == TTM_PL_SYSTEM) {
 683                 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
 684         } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
 685                    new_mem->mem_type == TTM_PL_VRAM) {
 686                 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
 687         } else {
 688                 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
 689                                      new_mem, old_mem);
 690         }
 691
 692         if (r) {
 693 memcpy:
 694                 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
 695                 if (r) {
 696                         return r;
 697                 }
 698         }
 699
 700         if (bo->type == ttm_bo_type_device &&
 701             new_mem->mem_type == TTM_PL_VRAM &&
 702             old_mem->mem_type != TTM_PL_VRAM) {
 703                 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
 704                  * accesses the BO after it's moved.
 705                  */
 706                 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 707         }
 708
 709         /* update statistics */
 710         atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
 711         return 0;
 712 }
 713
 714 /**
 715  * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
 716  *
 717  * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
 718  */
 719 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 720 {
 721         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 722         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 723         struct drm_mm_node *mm_node = mem->mm_node;
 724
 725         mem->bus.addr = NULL;
 726         mem->bus.offset = 0;
 727         mem->bus.size = mem->num_pages << PAGE_SHIFT;
 728         mem->bus.base = 0;
 729         mem->bus.is_iomem = false;
 730         if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
 731                 return -EINVAL;
 732         switch (mem->mem_type) {
 733         case TTM_PL_SYSTEM:
 734                 /* system memory */
 735                 return 0;
 736         case TTM_PL_TT:
 737                 break;
 738         case TTM_PL_VRAM:
 739                 mem->bus.offset = mem->start << PAGE_SHIFT;
 740                 /* check if it's visible */
 741                 if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
 742                         return -EINVAL;
 743                 /* Only physically contiguous buffers apply. In a contiguous
 744                  * buffer, size of the first mm_node would match the number of
 745                  * pages in ttm_mem_reg.
 746                  */
 747                 if (adev->mman.aper_base_kaddr &&
 748                     (mm_node->size == mem->num_pages))
 749                         mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
 750                                         mem->bus.offset;
 751
 752                 mem->bus.base = adev->gmc.aper_base;
 753                 mem->bus.is_iomem = true;
 754                 break;
 755         default:
 756                 return -EINVAL;
 757         }
 758         return 0;
 759 }
 760
 761 static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 762 {
 763 }
 764
 765 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
 766                                            unsigned long page_offset)
 767 {
 768         struct drm_mm_node *mm;
 769         unsigned long offset = (page_offset << PAGE_SHIFT);
 770
 771         mm = amdgpu_find_mm_node(&bo->mem, &offset);
 772         return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
 773                 (offset >> PAGE_SHIFT);
 774 }
 775
 776 /*
 777  * TTM backend functions.
 778  */
 779 struct amdgpu_ttm_gup_task_list {
 780         struct list_head        list;
 781         struct task_struct      *task;
 782 };
 783
 784 struct amdgpu_ttm_tt {
 785         struct ttm_dma_tt       ttm;
 786         u64                     offset;
 787         uint64_t                userptr;
 788         struct task_struct      *usertask;
 789         uint32_t                userflags;
 790         spinlock_t              guptasklock;
 791         struct list_head        guptasks;
 792         atomic_t                mmu_invalidations;
 793         uint32_t                last_set_pages;
 794 };
 795
 796 /**
 797  * amdgpu_ttm_tt_get_user_pages -       Pin pages of memory pointed to
 798  *                                                                      by a USERPTR pointer to memory
 799  *
 800  * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
 801  * This provides a wrapper around the get_user_pages() call to provide
 802  * device accessible pages that back user memory.
 803  */
 804 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 805 {
 806         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 807         struct mm_struct *mm = gtt->usertask->mm;
 808         unsigned int flags = 0;
 809         unsigned pinned = 0;
 810         int r;
 811
 812         if (!mm) /* Happens during process shutdown */
 813                 return -ESRCH;
 814
 815         if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
 816                 flags |= FOLL_WRITE;
 817
 818         down_read(&mm->mmap_sem);
 819
 820         if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
 821                 /* check that we only use anonymous memory
 822                    to prevent problems with writeback */
 823                 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
 824                 struct vm_area_struct *vma;
 825
 826                 vma = find_vma(mm, gtt->userptr);
 827                 if (!vma || vma->vm_file || vma->vm_end < end) {
 828                         up_read(&mm->mmap_sem);
 829                         return -EPERM;
 830                 }
 831         }
 832
 833         /* loop enough times using contiguous pages of memory */
 834         do {
 835                 unsigned num_pages = ttm->num_pages - pinned;
 836                 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
 837                 struct page **p = pages + pinned;
 838                 struct amdgpu_ttm_gup_task_list guptask;
 839
 840                 guptask.task = current;
 841                 spin_lock(&gtt->guptasklock);
 842                 list_add(&guptask.list, &gtt->guptasks);
 843                 spin_unlock(&gtt->guptasklock);
 844
 845                 if (mm == current->mm)
 846                         r = get_user_pages(userptr, num_pages, flags, p, NULL);
 847                 else
 848                         r = get_user_pages_remote(gtt->usertask,
 849                                         mm, userptr, num_pages,
 850                                         flags, p, NULL, NULL);
 851
 852                 spin_lock(&gtt->guptasklock);
 853                 list_del(&guptask.list);
 854                 spin_unlock(&gtt->guptasklock);
 855
 856                 if (r < 0)
 857                         goto release_pages;
 858
 859                 pinned += r;
 860
 861         } while (pinned < ttm->num_pages);
 862
 863         up_read(&mm->mmap_sem);
 864         return 0;
 865
 866 release_pages:
 867         release_pages(pages, pinned);
 868         up_read(&mm->mmap_sem);
 869         return r;
 870 }
 871
 872 /**
 873  * amdgpu_ttm_tt_set_user_pages -       Copy pages in, putting old pages
 874  *                                                                      as necessary.
 875  *
 876  * Called by amdgpu_cs_list_validate().  This creates the page list
 877  * that backs user memory and will ultimately be mapped into the device
 878  * address space.
 879  */
 880 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
 881 {
 882         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 883         unsigned i;
 884
 885         gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
 886         for (i = 0; i < ttm->num_pages; ++i) {
 887                 if (ttm->pages[i])
 888                         put_page(ttm->pages[i]);
 889
 890                 ttm->pages[i] = pages ? pages[i] : NULL;
 891         }
 892 }
 893
 894 /**
 895  * amdgpu_ttm_tt_mark_user_page - Mark pages as dirty
 896  *
 897  * Called while unpinning userptr pages
 898  */
 899 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
 900 {
 901         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 902         unsigned i;
 903
 904         for (i = 0; i < ttm->num_pages; ++i) {
 905                 struct page *page = ttm->pages[i];
 906
 907                 if (!page)
 908                         continue;
 909
 910                 if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
 911                         set_page_dirty(page);
 912
 913                 mark_page_accessed(page);
 914         }
 915 }
 916
 917 /**
 918  * amdgpu_ttm_tt_pin_userptr -  prepare the sg table with the
 919  *                                                              user pages
 920  *
 921  * Called by amdgpu_ttm_backend_bind()
 922  **/
 923 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 924 {
 925         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 926         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 927         unsigned nents;
 928         int r;
 929
 930         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 931         enum dma_data_direction direction = write ?
 932                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 933
 934         /* Allocate an SG array and squash pages into it */
 935         r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
 936                                       ttm->num_pages << PAGE_SHIFT,
 937                                       GFP_KERNEL);
 938         if (r)
 939                 goto release_sg;
 940
 941         /* Map SG to device */
 942         r = -ENOMEM;
 943         nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 944         if (nents != ttm->sg->nents)
 945                 goto release_sg;
 946
 947         /* convert SG to linear array of pages and dma addresses */
 948         drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
 949                                          gtt->ttm.dma_address, ttm->num_pages);
 950
 951         return 0;
 952
 953 release_sg:
 954         kfree(ttm->sg);
 955         return r;
 956 }
 957
 958 /**
 959  * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
 960  */
 961 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 962 {
 963         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 964         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 965
 966         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 967         enum dma_data_direction direction = write ?
 968                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 969
 970         /* double check that we don't free the table twice */
 971         if (!ttm->sg->sgl)
 972                 return;
 973
 974         /* unmap the pages mapped to the device */
 975         dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 976
 977         /* mark the pages as dirty */
 978         amdgpu_ttm_tt_mark_user_pages(ttm);
 979
 980         sg_free_table(ttm->sg);
 981 }
 982
 983 int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
 984                                 struct ttm_buffer_object *tbo,
 985                                 uint64_t flags)
 986 {
 987         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
 988         struct ttm_tt *ttm = tbo->ttm;
 989         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 990         int r;
 991
 992         if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
 993                 uint64_t page_idx = 1;
 994
 995                 r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
 996                                 ttm->pages, gtt->ttm.dma_address, flags);
 997                 if (r)
 998                         goto gart_bind_fail;
 999
1000                 /* Patch mtype of the second part BO */
1001                 flags &=  ~AMDGPU_PTE_MTYPE_MASK;
1002                 flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC);
1003
1004                 r = amdgpu_gart_bind(adev,
1005                                 gtt->offset + (page_idx << PAGE_SHIFT),
1006                                 ttm->num_pages - page_idx,
1007                                 &ttm->pages[page_idx],
1008                                 &(gtt->ttm.dma_address[page_idx]), flags);
1009         } else {
1010                 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1011                                      ttm->pages, gtt->ttm.dma_address, flags);
1012         }
1013
1014 gart_bind_fail:
1015         if (r)
1016                 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1017                           ttm->num_pages, gtt->offset);
1018
1019         return r;
1020 }
1021
1022 /**
1023  * amdgpu_ttm_backend_bind - Bind GTT memory
1024  *
1025  * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
1026  * This handles binding GTT memory to the device address space.
1027  */
1028 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
1029                                    struct ttm_mem_reg *bo_mem)
1030 {
1031         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1032         struct amdgpu_ttm_tt *gtt = (void*)ttm;
1033         uint64_t flags;
1034         int r = 0;
1035
1036         if (gtt->userptr) {
1037                 r = amdgpu_ttm_tt_pin_userptr(ttm);
1038                 if (r) {
1039                         DRM_ERROR("failed to pin userptr\n");
1040                         return r;
1041                 }
1042         }
1043         if (!ttm->num_pages) {
1044                 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
1045                      ttm->num_pages, bo_mem, ttm);
1046         }
1047
1048         if (bo_mem->mem_type == AMDGPU_PL_GDS ||
1049             bo_mem->mem_type == AMDGPU_PL_GWS ||
1050             bo_mem->mem_type == AMDGPU_PL_OA)
1051                 return -EINVAL;
1052
1053         if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
1054                 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
1055                 return 0;
1056         }
1057
1058         /* compute PTE flags relevant to this BO memory */
1059         flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1060
1061         /* bind pages into GART page tables */
1062         gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
1063         r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1064                 ttm->pages, gtt->ttm.dma_address, flags);
1065
1066         if (r)
1067                 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1068                           ttm->num_pages, gtt->offset);
1069         return r;
1070 }
1071
1072 /**
1073  * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
1074  */
1075 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1076 {
1077         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1078         struct ttm_operation_ctx ctx = { false, false };
1079         struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1080         struct ttm_mem_reg tmp;
1081         struct ttm_placement placement;
1082         struct ttm_place placements;
1083         uint64_t flags;
1084         int r;
1085
1086         if (bo->mem.mem_type != TTM_PL_TT ||
1087             amdgpu_gtt_mgr_has_gart_addr(&bo->mem))
1088                 return 0;
1089
1090         /* allocate GTT space */
1091         tmp = bo->mem;
1092         tmp.mm_node = NULL;
1093         placement.num_placement = 1;
1094         placement.placement = &placements;
1095         placement.num_busy_placement = 1;
1096         placement.busy_placement = &placements;
1097         placements.fpfn = 0;
1098         placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1099         placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
1100                 TTM_PL_FLAG_TT;
1101
1102         r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1103         if (unlikely(r))
1104                 return r;
1105
1106         /* compute PTE flags for this buffer object */
1107         flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1108
1109         /* Bind pages */
1110         gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1111         r = amdgpu_ttm_gart_bind(adev, bo, flags);
1112         if (unlikely(r)) {
1113                 ttm_bo_mem_put(bo, &tmp);
1114                 return r;
1115         }
1116
1117         ttm_bo_mem_put(bo, &bo->mem);
1118         bo->mem = tmp;
1119         bo->offset = (bo->mem.start << PAGE_SHIFT) +
1120                 bo->bdev->man[bo->mem.mem_type].gpu_offset;
1121
1122         return 0;
1123 }
1124
1125 /**
1126  * amdgpu_ttm_recover_gart - Rebind GTT pages
1127  *
1128  * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1129  * rebind GTT pages during a GPU reset.
1130  */
1131 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1132 {
1133         struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1134         uint64_t flags;
1135         int r;
1136
1137         if (!tbo->ttm)
1138                 return 0;
1139
1140         flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1141         r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1142
1143         return r;
1144 }
1145
1146 /**
1147  * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1148  *
1149  * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1150  * ttm_tt_destroy().
1151  */
1152 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
1153 {
1154         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1155         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1156         int r;
1157
1158         /* if the pages have userptr pinning then clear that first */
1159         if (gtt->userptr)
1160                 amdgpu_ttm_tt_unpin_userptr(ttm);
1161
1162         if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1163                 return 0;
1164
1165         /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1166         r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1167         if (r)
1168                 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
1169                           gtt->ttm.ttm.num_pages, gtt->offset);
1170         return r;
1171 }
1172
1173 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
1174 {
1175         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1176
1177         if (gtt->usertask)
1178                 put_task_struct(gtt->usertask);
1179
1180         ttm_dma_tt_fini(&gtt->ttm);
1181         kfree(gtt);
1182 }
1183
1184 static struct ttm_backend_func amdgpu_backend_func = {
1185         .bind = &amdgpu_ttm_backend_bind,
1186         .unbind = &amdgpu_ttm_backend_unbind,
1187         .destroy = &amdgpu_ttm_backend_destroy,
1188 };
1189
1190 /**
1191  * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1192  *
1193  * @bo: The buffer object to create a GTT ttm_tt object around
1194  *
1195  * Called by ttm_tt_create().
1196  */
1197 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1198                                            uint32_t page_flags)
1199 {
1200         struct amdgpu_device *adev;
1201         struct amdgpu_ttm_tt *gtt;
1202
1203         adev = amdgpu_ttm_adev(bo->bdev);
1204
1205         gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1206         if (gtt == NULL) {
1207                 return NULL;
1208         }
1209         gtt->ttm.ttm.func = &amdgpu_backend_func;
1210
1211         /* allocate space for the uninitialized page entries */
1212         if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
1213                 kfree(gtt);
1214                 return NULL;
1215         }
1216         return &gtt->ttm.ttm;
1217 }
1218
1219 /**
1220  * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1221  *
1222  * Map the pages of a ttm_tt object to an address space visible
1223  * to the underlying device.
1224  */
1225 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
1226                         struct ttm_operation_ctx *ctx)
1227 {
1228         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1229         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1230         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1231
1232         /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1233         if (gtt && gtt->userptr) {
1234                 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1235                 if (!ttm->sg)
1236                         return -ENOMEM;
1237
1238                 ttm->page_flags |= TTM_PAGE_FLAG_SG;
1239                 ttm->state = tt_unbound;
1240                 return 0;
1241         }
1242
1243         if (slave && ttm->sg) {
1244                 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1245                                                  gtt->ttm.dma_address,
1246                                                  ttm->num_pages);
1247                 ttm->state = tt_unbound;
1248                 return 0;
1249         }
1250
1251 #ifdef CONFIG_SWIOTLB
1252         if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1253                 return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
1254         }
1255 #endif
1256
1257         /* fall back to generic helper to populate the page array
1258          * and map them to the device */
1259         return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
1260 }
1261
1262 /**
1263  * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1264  *
1265  * Unmaps pages of a ttm_tt object from the device address space and
1266  * unpopulates the page array backing it.
1267  */
1268 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
1269 {
1270         struct amdgpu_device *adev;
1271         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1272         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1273
1274         if (gtt && gtt->userptr) {
1275                 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1276                 kfree(ttm->sg);
1277                 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1278                 return;
1279         }
1280
1281         if (slave)
1282                 return;
1283
1284         adev = amdgpu_ttm_adev(ttm->bdev);
1285
1286 #ifdef CONFIG_SWIOTLB
1287         if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1288                 ttm_dma_unpopulate(&gtt->ttm, adev->dev);
1289                 return;
1290         }
1291 #endif
1292
1293         /* fall back to generic helper to unmap and unpopulate array */
1294         ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
1295 }
1296
1297 /**
1298  * amdgpu_ttm_tt_set_userptr -  Initialize userptr GTT ttm_tt
1299  *                                                              for the current task
1300  *
1301  * @ttm: The ttm_tt object to bind this userptr object to
1302  * @addr:  The address in the current tasks VM space to use
1303  * @flags: Requirements of userptr object.
1304  *
1305  * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
1306  * to current task
1307  */
1308 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1309                               uint32_t flags)
1310 {
1311         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1312
1313         if (gtt == NULL)
1314                 return -EINVAL;
1315
1316         gtt->userptr = addr;
1317         gtt->userflags = flags;
1318
1319         if (gtt->usertask)
1320                 put_task_struct(gtt->usertask);
1321         gtt->usertask = current->group_leader;
1322         get_task_struct(gtt->usertask);
1323
1324         spin_lock_init(&gtt->guptasklock);
1325         INIT_LIST_HEAD(&gtt->guptasks);
1326         atomic_set(&gtt->mmu_invalidations, 0);
1327         gtt->last_set_pages = 0;
1328
1329         return 0;
1330 }
1331
1332 /**
1333  * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1334  */
1335 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1336 {
1337         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1338
1339         if (gtt == NULL)
1340                 return NULL;
1341
1342         if (gtt->usertask == NULL)
1343                 return NULL;
1344
1345         return gtt->usertask->mm;
1346 }
1347
1348 /**
1349  * amdgpu_ttm_tt_affect_userptr -       Determine if a ttm_tt object lays
1350  *                                                                      inside an address range for the
1351  *                                                                      current task.
1352  *
1353  */
1354 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1355                                   unsigned long end)
1356 {
1357         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1358         struct amdgpu_ttm_gup_task_list *entry;
1359         unsigned long size;
1360
1361         if (gtt == NULL || !gtt->userptr)
1362                 return false;
1363
1364         /* Return false if no part of the ttm_tt object lies within
1365          * the range
1366          */
1367         size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
1368         if (gtt->userptr > end || gtt->userptr + size <= start)
1369                 return false;
1370
1371         /* Search the lists of tasks that hold this mapping and see
1372          * if current is one of them.  If it is return false.
1373          */
1374         spin_lock(&gtt->guptasklock);
1375         list_for_each_entry(entry, &gtt->guptasks, list) {
1376                 if (entry->task == current) {
1377                         spin_unlock(&gtt->guptasklock);
1378                         return false;
1379                 }
1380         }
1381         spin_unlock(&gtt->guptasklock);
1382
1383         atomic_inc(&gtt->mmu_invalidations);
1384
1385         return true;
1386 }
1387
1388 /**
1389  * amdgpu_ttm_tt_userptr_invalidated -  Has the ttm_tt object been
1390  *                                                                              invalidated?
1391  */
1392 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
1393                                        int *last_invalidated)
1394 {
1395         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1396         int prev_invalidated = *last_invalidated;
1397
1398         *last_invalidated = atomic_read(&gtt->mmu_invalidations);
1399         return prev_invalidated != *last_invalidated;
1400 }
1401
1402 /**
1403  * amdgpu_ttm_tt_userptr_needs_pages -  Have the pages backing this
1404  *                                                                              ttm_tt object been invalidated
1405  *                                                                              since the last time they've
1406  *                                                                              been set?
1407  */
1408 bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
1409 {
1410         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1411
1412         if (gtt == NULL || !gtt->userptr)
1413                 return false;
1414
1415         return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
1416 }
1417
1418 /**
1419  * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1420  */
1421 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1422 {
1423         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1424
1425         if (gtt == NULL)
1426                 return false;
1427
1428         return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1429 }
1430
1431 /**
1432  * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1433  *
1434  * @ttm: The ttm_tt object to compute the flags for
1435  * @mem: The memory registry backing this ttm_tt object
1436  */
1437 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1438                                  struct ttm_mem_reg *mem)
1439 {
1440         uint64_t flags = 0;
1441
1442         if (mem && mem->mem_type != TTM_PL_SYSTEM)
1443                 flags |= AMDGPU_PTE_VALID;
1444
1445         if (mem && mem->mem_type == TTM_PL_TT) {
1446                 flags |= AMDGPU_PTE_SYSTEM;
1447
1448                 if (ttm->caching_state == tt_cached)
1449                         flags |= AMDGPU_PTE_SNOOPED;
1450         }
1451
1452         flags |= adev->gart.gart_pte_flags;
1453         flags |= AMDGPU_PTE_READABLE;
1454
1455         if (!amdgpu_ttm_tt_is_readonly(ttm))
1456                 flags |= AMDGPU_PTE_WRITEABLE;
1457
1458         return flags;
1459 }
1460
1461 /**
1462  * amdgpu_ttm_bo_eviction_valuable -    Check to see if we can evict
1463  *                                                                              a buffer object.
1464  *
1465  * Return true if eviction is sensible.  Called by
1466  * ttm_mem_evict_first() on behalf of ttm_bo_mem_force_space()
1467  * which tries to evict buffer objects until it can find space
1468  * for a new object and by ttm_bo_force_list_clean() which is
1469  * used to clean out a memory space.
1470  */
1471 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1472                                             const struct ttm_place *place)
1473 {
1474         unsigned long num_pages = bo->mem.num_pages;
1475         struct drm_mm_node *node = bo->mem.mm_node;
1476         struct reservation_object_list *flist;
1477         struct dma_fence *f;
1478         int i;
1479
1480         /* If bo is a KFD BO, check if the bo belongs to the current process.
1481          * If true, then return false as any KFD process needs all its BOs to
1482          * be resident to run successfully
1483          */
1484         flist = reservation_object_get_list(bo->resv);
1485         if (flist) {
1486                 for (i = 0; i < flist->shared_count; ++i) {
1487                         f = rcu_dereference_protected(flist->shared[i],
1488                                 reservation_object_held(bo->resv));
1489                         if (amdkfd_fence_check_mm(f, current->mm))
1490                                 return false;
1491                 }
1492         }
1493
1494         switch (bo->mem.mem_type) {
1495         case TTM_PL_TT:
1496                 return true;
1497
1498         case TTM_PL_VRAM:
1499                 /* Check each drm MM node individually */
1500                 while (num_pages) {
1501                         if (place->fpfn < (node->start + node->size) &&
1502                             !(place->lpfn && place->lpfn <= node->start))
1503                                 return true;
1504
1505                         num_pages -= node->size;
1506                         ++node;
1507                 }
1508                 return false;
1509
1510         default:
1511                 break;
1512         }
1513
1514         return ttm_bo_eviction_valuable(bo, place);
1515 }
1516
1517 /**
1518  * amdgpu_ttm_access_memory -   Read or Write memory that backs a
1519  *                                                              buffer object.
1520  *
1521  * @bo:  The buffer object to read/write
1522  * @offset:  Offset into buffer object
1523  * @buf:  Secondary buffer to write/read from
1524  * @len: Length in bytes of access
1525  * @write:  true if writing
1526  *
1527  * This is used to access VRAM that backs a buffer object via MMIO
1528  * access for debugging purposes.
1529  */
1530 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1531                                     unsigned long offset,
1532                                     void *buf, int len, int write)
1533 {
1534         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1535         struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1536         struct drm_mm_node *nodes;
1537         uint32_t value = 0;
1538         int ret = 0;
1539         uint64_t pos;
1540         unsigned long flags;
1541
1542         if (bo->mem.mem_type != TTM_PL_VRAM)
1543                 return -EIO;
1544
1545         nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1546         pos = (nodes->start << PAGE_SHIFT) + offset;
1547
1548         while (len && pos < adev->gmc.mc_vram_size) {
1549                 uint64_t aligned_pos = pos & ~(uint64_t)3;
1550                 uint32_t bytes = 4 - (pos & 3);
1551                 uint32_t shift = (pos & 3) * 8;
1552                 uint32_t mask = 0xffffffff << shift;
1553
1554                 if (len < bytes) {
1555                         mask &= 0xffffffff >> (bytes - len) * 8;
1556                         bytes = len;
1557                 }
1558
1559                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1560                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1561                 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1562                 if (!write || mask != 0xffffffff)
1563                         value = RREG32_NO_KIQ(mmMM_DATA);
1564                 if (write) {
1565                         value &= ~mask;
1566                         value |= (*(uint32_t *)buf << shift) & mask;
1567                         WREG32_NO_KIQ(mmMM_DATA, value);
1568                 }
1569                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1570                 if (!write) {
1571                         value = (value & mask) >> shift;
1572                         memcpy(buf, &value, bytes);
1573                 }
1574
1575                 ret += bytes;
1576                 buf = (uint8_t *)buf + bytes;
1577                 pos += bytes;
1578                 len -= bytes;
1579                 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1580                         ++nodes;
1581                         pos = (nodes->start << PAGE_SHIFT);
1582                 }
1583         }
1584
1585         return ret;
1586 }
1587
1588 static struct ttm_bo_driver amdgpu_bo_driver = {
1589         .ttm_tt_create = &amdgpu_ttm_tt_create,
1590         .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1591         .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1592         .invalidate_caches = &amdgpu_invalidate_caches,
1593         .init_mem_type = &amdgpu_init_mem_type,
1594         .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1595         .evict_flags = &amdgpu_evict_flags,
1596         .move = &amdgpu_bo_move,
1597         .verify_access = &amdgpu_verify_access,
1598         .move_notify = &amdgpu_bo_move_notify,
1599         .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1600         .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1601         .io_mem_free = &amdgpu_ttm_io_mem_free,
1602         .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1603         .access_memory = &amdgpu_ttm_access_memory
1604 };
1605
1606 /*
1607  * Firmware Reservation functions
1608  */
1609 /**
1610  * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1611  *
1612  * @adev: amdgpu_device pointer
1613  *
1614  * free fw reserved vram if it has been reserved.
1615  */
1616 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1617 {
1618         amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
1619                 NULL, &adev->fw_vram_usage.va);
1620 }
1621
1622 /**
1623  * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1624  *
1625  * @adev: amdgpu_device pointer
1626  *
1627  * create bo vram reservation from fw.
1628  */
1629 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1630 {
1631         struct ttm_operation_ctx ctx = { false, false };
1632         struct amdgpu_bo_param bp;
1633         int r = 0;
1634         int i;
1635         u64 vram_size = adev->gmc.visible_vram_size;
1636         u64 offset = adev->fw_vram_usage.start_offset;
1637         u64 size = adev->fw_vram_usage.size;
1638         struct amdgpu_bo *bo;
1639
1640         memset(&bp, 0, sizeof(bp));
1641         bp.size = adev->fw_vram_usage.size;
1642         bp.byte_align = PAGE_SIZE;
1643         bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
1644         bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1645                 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1646         bp.type = ttm_bo_type_kernel;
1647         bp.resv = NULL;
1648         adev->fw_vram_usage.va = NULL;
1649         adev->fw_vram_usage.reserved_bo = NULL;
1650
1651         if (adev->fw_vram_usage.size > 0 &&
1652                 adev->fw_vram_usage.size <= vram_size) {
1653
1654                 r = amdgpu_bo_create(adev, &bp,
1655                                      &adev->fw_vram_usage.reserved_bo);
1656                 if (r)
1657                         goto error_create;
1658
1659                 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
1660                 if (r)
1661                         goto error_reserve;
1662
1663                 /* remove the original mem node and create a new one at the
1664                  * request position
1665                  */
1666                 bo = adev->fw_vram_usage.reserved_bo;
1667                 offset = ALIGN(offset, PAGE_SIZE);
1668                 for (i = 0; i < bo->placement.num_placement; ++i) {
1669                         bo->placements[i].fpfn = offset >> PAGE_SHIFT;
1670                         bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
1671                 }
1672
1673                 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
1674                 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
1675                                      &bo->tbo.mem, &ctx);
1676                 if (r)
1677                         goto error_pin;
1678
1679                 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
1680                         AMDGPU_GEM_DOMAIN_VRAM,
1681                         adev->fw_vram_usage.start_offset,
1682                         (adev->fw_vram_usage.start_offset +
1683                         adev->fw_vram_usage.size));
1684                 if (r)
1685                         goto error_pin;
1686                 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
1687                         &adev->fw_vram_usage.va);
1688                 if (r)
1689                         goto error_kmap;
1690
1691                 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1692         }
1693         return r;
1694
1695 error_kmap:
1696         amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
1697 error_pin:
1698         amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1699 error_reserve:
1700         amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
1701 error_create:
1702         adev->fw_vram_usage.va = NULL;
1703         adev->fw_vram_usage.reserved_bo = NULL;
1704         return r;
1705 }
1706 /**
1707  * amdgpu_ttm_init -    Init the memory management (ttm) as well as
1708  *                                              various gtt/vram related fields.
1709  *
1710  * This initializes all of the memory space pools that the TTM layer
1711  * will need such as the GTT space (system memory mapped to the device),
1712  * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1713  * can be mapped per VMID.
1714  */
1715 int amdgpu_ttm_init(struct amdgpu_device *adev)
1716 {
1717         uint64_t gtt_size;
1718         int r;
1719         u64 vis_vram_limit;
1720
1721         /* initialize global references for vram/gtt */
1722         r = amdgpu_ttm_global_init(adev);
1723         if (r) {
1724                 return r;
1725         }
1726         /* No others user of address space so set it to 0 */
1727         r = ttm_bo_device_init(&adev->mman.bdev,
1728                                adev->mman.bo_global_ref.ref.object,
1729                                &amdgpu_bo_driver,
1730                                adev->ddev->anon_inode->i_mapping,
1731                                DRM_FILE_PAGE_OFFSET,
1732                                adev->need_dma32);
1733         if (r) {
1734                 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1735                 return r;
1736         }
1737         adev->mman.initialized = true;
1738
1739         /* We opt to avoid OOM on system pages allocations */
1740         adev->mman.bdev.no_retry = true;
1741
1742         /* Initialize VRAM pool with all of VRAM divided into pages */
1743         r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1744                                 adev->gmc.real_vram_size >> PAGE_SHIFT);
1745         if (r) {
1746                 DRM_ERROR("Failed initializing VRAM heap.\n");
1747                 return r;
1748         }
1749
1750         /* Reduce size of CPU-visible VRAM if requested */
1751         vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1752         if (amdgpu_vis_vram_limit > 0 &&
1753             vis_vram_limit <= adev->gmc.visible_vram_size)
1754                 adev->gmc.visible_vram_size = vis_vram_limit;
1755
1756         /* Change the size here instead of the init above so only lpfn is affected */
1757         amdgpu_ttm_set_buffer_funcs_status(adev, false);
1758 #ifdef CONFIG_64BIT
1759         adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1760                                                 adev->gmc.visible_vram_size);
1761 #endif
1762
1763         /*
1764          *The reserved vram for firmware must be pinned to the specified
1765          *place on the VRAM, so reserve it early.
1766          */
1767         r = amdgpu_ttm_fw_reserve_vram_init(adev);
1768         if (r) {
1769                 return r;
1770         }
1771
1772         /* allocate memory as required for VGA
1773          * This is used for VGA emulation and pre-OS scanout buffers to
1774          * avoid display artifacts while transitioning between pre-OS
1775          * and driver.  */
1776         if (adev->gmc.stolen_size) {
1777                 r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
1778                                             AMDGPU_GEM_DOMAIN_VRAM,
1779                                             &adev->stolen_vga_memory,
1780                                             NULL, NULL);
1781                 if (r)
1782                         return r;
1783         }
1784         DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1785                  (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1786
1787         /* Compute GTT size, either bsaed on 3/4th the size of RAM size
1788          * or whatever the user passed on module init */
1789         if (amdgpu_gtt_size == -1) {
1790                 struct sysinfo si;
1791
1792                 si_meminfo(&si);
1793                 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1794                                adev->gmc.mc_vram_size),
1795                                ((uint64_t)si.totalram * si.mem_unit * 3/4));
1796         }
1797         else
1798                 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1799
1800         /* Initialize GTT memory pool */
1801         r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1802         if (r) {
1803                 DRM_ERROR("Failed initializing GTT heap.\n");
1804                 return r;
1805         }
1806         DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1807                  (unsigned)(gtt_size / (1024 * 1024)));
1808
1809         /* Initialize various on-chip memory pools */
1810         adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;
1811         adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;
1812         adev->gds.mem.cs_partition_size = adev->gds.mem.cs_partition_size << AMDGPU_GDS_SHIFT;
1813         adev->gds.gws.total_size = adev->gds.gws.total_size << AMDGPU_GWS_SHIFT;
1814         adev->gds.gws.gfx_partition_size = adev->gds.gws.gfx_partition_size << AMDGPU_GWS_SHIFT;
1815         adev->gds.gws.cs_partition_size = adev->gds.gws.cs_partition_size << AMDGPU_GWS_SHIFT;
1816         adev->gds.oa.total_size = adev->gds.oa.total_size << AMDGPU_OA_SHIFT;
1817         adev->gds.oa.gfx_partition_size = adev->gds.oa.gfx_partition_size << AMDGPU_OA_SHIFT;
1818         adev->gds.oa.cs_partition_size = adev->gds.oa.cs_partition_size << AMDGPU_OA_SHIFT;
1819         /* GDS Memory */
1820         if (adev->gds.mem.total_size) {
1821                 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
1822                                    adev->gds.mem.total_size >> PAGE_SHIFT);
1823                 if (r) {
1824                         DRM_ERROR("Failed initializing GDS heap.\n");
1825                         return r;
1826                 }
1827         }
1828
1829         /* GWS */
1830         if (adev->gds.gws.total_size) {
1831                 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
1832                                    adev->gds.gws.total_size >> PAGE_SHIFT);
1833                 if (r) {
1834                         DRM_ERROR("Failed initializing gws heap.\n");
1835                         return r;
1836                 }
1837         }
1838
1839         /* OA */
1840         if (adev->gds.oa.total_size) {
1841                 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
1842                                    adev->gds.oa.total_size >> PAGE_SHIFT);
1843                 if (r) {
1844                         DRM_ERROR("Failed initializing oa heap.\n");
1845                         return r;
1846                 }
1847         }
1848
1849         /* Register debugfs entries for amdgpu_ttm */
1850         r = amdgpu_ttm_debugfs_init(adev);
1851         if (r) {
1852                 DRM_ERROR("Failed to init debugfs\n");
1853                 return r;
1854         }
1855         return 0;
1856 }
1857
1858 /**
1859  * amdgpu_ttm_late_init -       Handle any late initialization for
1860  *                                                      amdgpu_ttm
1861  */
1862 void amdgpu_ttm_late_init(struct amdgpu_device *adev)
1863 {
1864         /* return the VGA stolen memory (if any) back to VRAM */
1865         amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
1866 }
1867
1868 /**
1869  * amdgpu_ttm_fini - De-initialize the TTM memory pools
1870  */
1871 void amdgpu_ttm_fini(struct amdgpu_device *adev)
1872 {
1873         if (!adev->mman.initialized)
1874                 return;
1875
1876         amdgpu_ttm_debugfs_fini(adev);
1877         amdgpu_ttm_fw_reserve_vram_fini(adev);
1878         if (adev->mman.aper_base_kaddr)
1879                 iounmap(adev->mman.aper_base_kaddr);
1880         adev->mman.aper_base_kaddr = NULL;
1881
1882         ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1883         ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1884         if (adev->gds.mem.total_size)
1885                 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
1886         if (adev->gds.gws.total_size)
1887                 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
1888         if (adev->gds.oa.total_size)
1889                 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1890         ttm_bo_device_release(&adev->mman.bdev);
1891         amdgpu_ttm_global_fini(adev);
1892         adev->mman.initialized = false;
1893         DRM_INFO("amdgpu: ttm finalized\n");
1894 }
1895
1896 /**
1897  * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
1898  *
1899  * @adev: amdgpu_device pointer
1900  * @enable: true when we can use buffer functions.
1901  *
1902  * Enable/disable use of buffer functions during suspend/resume. This should
1903  * only be called at bootup or when userspace isn't running.
1904  */
1905 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1906 {
1907         struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
1908         uint64_t size;
1909         int r;
1910
1911         if (!adev->mman.initialized || adev->in_gpu_reset ||
1912             adev->mman.buffer_funcs_enabled == enable)
1913                 return;
1914
1915         if (enable) {
1916                 struct amdgpu_ring *ring;
1917                 struct drm_sched_rq *rq;
1918
1919                 ring = adev->mman.buffer_funcs_ring;
1920                 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
1921                 r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
1922                 if (r) {
1923                         DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
1924                                   r);
1925                         return;
1926                 }
1927         } else {
1928                 drm_sched_entity_destroy(adev->mman.entity.sched,
1929                                          &adev->mman.entity);
1930         }
1931
1932         /* this just adjusts TTM size idea, which sets lpfn to the correct value */
1933         if (enable)
1934                 size = adev->gmc.real_vram_size;
1935         else
1936                 size = adev->gmc.visible_vram_size;
1937         man->size = size >> PAGE_SHIFT;
1938         adev->mman.buffer_funcs_enabled = enable;
1939 }
1940
1941 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
1942 {
1943         struct drm_file *file_priv;
1944         struct amdgpu_device *adev;
1945
1946         if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
1947                 return -EINVAL;
1948
1949         file_priv = filp->private_data;
1950         adev = file_priv->minor->dev->dev_private;
1951         if (adev == NULL)
1952                 return -EINVAL;
1953
1954         return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
1955 }
1956
1957 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
1958                              struct ttm_mem_reg *mem, unsigned num_pages,
1959                              uint64_t offset, unsigned window,
1960                              struct amdgpu_ring *ring,
1961                              uint64_t *addr)
1962 {
1963         struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
1964         struct amdgpu_device *adev = ring->adev;
1965         struct ttm_tt *ttm = bo->ttm;
1966         struct amdgpu_job *job;
1967         unsigned num_dw, num_bytes;
1968         dma_addr_t *dma_address;
1969         struct dma_fence *fence;
1970         uint64_t src_addr, dst_addr;
1971         uint64_t flags;
1972         int r;
1973
1974         BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
1975                AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
1976
1977         *addr = adev->gmc.gart_start;
1978         *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
1979                 AMDGPU_GPU_PAGE_SIZE;
1980
1981         num_dw = adev->mman.buffer_funcs->copy_num_dw;
1982         while (num_dw & 0x7)
1983                 num_dw++;
1984
1985         num_bytes = num_pages * 8;
1986
1987         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
1988         if (r)
1989                 return r;
1990
1991         src_addr = num_dw * 4;
1992         src_addr += job->ibs[0].gpu_addr;
1993
1994         dst_addr = adev->gart.table_addr;
1995         dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
1996         amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
1997                                 dst_addr, num_bytes);
1998
1999         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2000         WARN_ON(job->ibs[0].length_dw > num_dw);
2001
2002         dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
2003         flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
2004         r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
2005                             &job->ibs[0].ptr[num_dw]);
2006         if (r)
2007                 goto error_free;
2008
2009         r = amdgpu_job_submit(job, &adev->mman.entity,
2010                               AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
2011         if (r)
2012                 goto error_free;
2013
2014         dma_fence_put(fence);
2015
2016         return r;
2017
2018 error_free:
2019         amdgpu_job_free(job);
2020         return r;
2021 }
2022
2023 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
2024                        uint64_t dst_offset, uint32_t byte_count,
2025                        struct reservation_object *resv,
2026                        struct dma_fence **fence, bool direct_submit,
2027                        bool vm_needs_flush)
2028 {
2029         struct amdgpu_device *adev = ring->adev;
2030         struct amdgpu_job *job;
2031
2032         uint32_t max_bytes;
2033         unsigned num_loops, num_dw;
2034         unsigned i;
2035         int r;
2036
2037         if (direct_submit && !ring->ready) {
2038                 DRM_ERROR("Trying to move memory with ring turned off.\n");
2039                 return -EINVAL;
2040         }
2041
2042         max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2043         num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2044         num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;
2045
2046         /* for IB padding */
2047         while (num_dw & 0x7)
2048                 num_dw++;
2049
2050         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2051         if (r)
2052                 return r;
2053
2054         job->vm_needs_flush = vm_needs_flush;
2055         if (resv) {
2056                 r = amdgpu_sync_resv(adev, &job->sync, resv,
2057                                      AMDGPU_FENCE_OWNER_UNDEFINED,
2058                                      false);
2059                 if (r) {
2060                         DRM_ERROR("sync failed (%d).\n", r);
2061                         goto error_free;
2062                 }
2063         }
2064
2065         for (i = 0; i < num_loops; i++) {
2066                 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2067
2068                 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2069                                         dst_offset, cur_size_in_bytes);
2070
2071                 src_offset += cur_size_in_bytes;
2072                 dst_offset += cur_size_in_bytes;
2073                 byte_count -= cur_size_in_bytes;
2074         }
2075
2076         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2077         WARN_ON(job->ibs[0].length_dw > num_dw);
2078         if (direct_submit)
2079                 r = amdgpu_job_submit_direct(job, ring, fence);
2080         else
2081                 r = amdgpu_job_submit(job, &adev->mman.entity,
2082                                       AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2083         if (r)
2084                 goto error_free;
2085
2086         return r;
2087
2088 error_free:
2089         amdgpu_job_free(job);
2090         DRM_ERROR("Error scheduling IBs (%d)\n", r);
2091         return r;
2092 }
2093
2094 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2095                        uint32_t src_data,
2096                        struct reservation_object *resv,
2097                        struct dma_fence **fence)
2098 {
2099         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2100         uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2101         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2102
2103         struct drm_mm_node *mm_node;
2104         unsigned long num_pages;
2105         unsigned int num_loops, num_dw;
2106
2107         struct amdgpu_job *job;
2108         int r;
2109
2110         if (!adev->mman.buffer_funcs_enabled) {
2111                 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2112                 return -EINVAL;
2113         }
2114
2115         if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2116                 r = amdgpu_ttm_alloc_gart(&bo->tbo);
2117                 if (r)
2118                         return r;
2119         }
2120
2121         num_pages = bo->tbo.num_pages;
2122         mm_node = bo->tbo.mem.mm_node;
2123         num_loops = 0;
2124         while (num_pages) {
2125                 uint32_t byte_count = mm_node->size << PAGE_SHIFT;
2126
2127                 num_loops += DIV_ROUND_UP(byte_count, max_bytes);
2128                 num_pages -= mm_node->size;
2129                 ++mm_node;
2130         }
2131         num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2132
2133         /* for IB padding */
2134         num_dw += 64;
2135
2136         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2137         if (r)
2138                 return r;
2139
2140         if (resv) {
2141                 r = amdgpu_sync_resv(adev, &job->sync, resv,
2142                                      AMDGPU_FENCE_OWNER_UNDEFINED, false);
2143                 if (r) {
2144                         DRM_ERROR("sync failed (%d).\n", r);
2145                         goto error_free;
2146                 }
2147         }
2148
2149         num_pages = bo->tbo.num_pages;
2150         mm_node = bo->tbo.mem.mm_node;
2151
2152         while (num_pages) {
2153                 uint32_t byte_count = mm_node->size << PAGE_SHIFT;
2154                 uint64_t dst_addr;
2155
2156                 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2157                 while (byte_count) {
2158                         uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2159
2160                         amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
2161                                                 dst_addr, cur_size_in_bytes);
2162
2163                         dst_addr += cur_size_in_bytes;
2164                         byte_count -= cur_size_in_bytes;
2165                 }
2166
2167                 num_pages -= mm_node->size;
2168                 ++mm_node;
2169         }
2170
2171         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2172         WARN_ON(job->ibs[0].length_dw > num_dw);
2173         r = amdgpu_job_submit(job, &adev->mman.entity,
2174                               AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2175         if (r)
2176                 goto error_free;
2177
2178         return 0;
2179
2180 error_free:
2181         amdgpu_job_free(job);
2182         return r;
2183 }
2184
2185 #if defined(CONFIG_DEBUG_FS)
2186
2187 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
2188 {
2189         struct drm_info_node *node = (struct drm_info_node *)m->private;
2190         unsigned ttm_pl = *(int *)node->info_ent->data;
2191         struct drm_device *dev = node->minor->dev;
2192         struct amdgpu_device *adev = dev->dev_private;
2193         struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
2194         struct drm_printer p = drm_seq_file_printer(m);
2195
2196         man->func->debug(man, &p);
2197         return 0;
2198 }
2199
2200 static int ttm_pl_vram = TTM_PL_VRAM;
2201 static int ttm_pl_tt = TTM_PL_TT;
2202
2203 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
2204         {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, &ttm_pl_vram},
2205         {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, &ttm_pl_tt},
2206         {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
2207 #ifdef CONFIG_SWIOTLB
2208         {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
2209 #endif
2210 };
2211
2212 /**
2213  * amdgpu_ttm_vram_read - Linear read access to VRAM
2214  *
2215  * Accesses VRAM via MMIO for debugging purposes.
2216  */
2217 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2218                                     size_t size, loff_t *pos)
2219 {
2220         struct amdgpu_device *adev = file_inode(f)->i_private;
2221         ssize_t result = 0;
2222         int r;
2223
2224         if (size & 0x3 || *pos & 0x3)
2225                 return -EINVAL;
2226
2227         if (*pos >= adev->gmc.mc_vram_size)
2228                 return -ENXIO;
2229
2230         while (size) {
2231                 unsigned long flags;
2232                 uint32_t value;
2233
2234                 if (*pos >= adev->gmc.mc_vram_size)
2235                         return result;
2236
2237                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2238                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2239                 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2240                 value = RREG32_NO_KIQ(mmMM_DATA);
2241                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2242
2243                 r = put_user(value, (uint32_t *)buf);
2244                 if (r)
2245                         return r;
2246
2247                 result += 4;
2248                 buf += 4;
2249                 *pos += 4;
2250                 size -= 4;
2251         }
2252
2253         return result;
2254 }
2255
2256 /**
2257  * amdgpu_ttm_vram_write - Linear write access to VRAM
2258  *
2259  * Accesses VRAM via MMIO for debugging purposes.
2260  */
2261 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2262                                     size_t size, loff_t *pos)
2263 {
2264         struct amdgpu_device *adev = file_inode(f)->i_private;
2265         ssize_t result = 0;
2266         int r;
2267
2268         if (size & 0x3 || *pos & 0x3)
2269                 return -EINVAL;
2270
2271         if (*pos >= adev->gmc.mc_vram_size)
2272                 return -ENXIO;
2273
2274         while (size) {
2275                 unsigned long flags;
2276                 uint32_t value;
2277
2278                 if (*pos >= adev->gmc.mc_vram_size)
2279                         return result;
2280
2281                 r = get_user(value, (uint32_t *)buf);
2282                 if (r)
2283                         return r;
2284
2285                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2286                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2287                 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2288                 WREG32_NO_KIQ(mmMM_DATA, value);
2289                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2290
2291                 result += 4;
2292                 buf += 4;
2293                 *pos += 4;
2294                 size -= 4;
2295         }
2296
2297         return result;
2298 }
2299
2300 static const struct file_operations amdgpu_ttm_vram_fops = {
2301         .owner = THIS_MODULE,
2302         .read = amdgpu_ttm_vram_read,
2303         .write = amdgpu_ttm_vram_write,
2304         .llseek = default_llseek,
2305 };
2306
2307 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2308
2309 /**
2310  * amdgpu_ttm_gtt_read - Linear read access to GTT memory
2311  */
2312 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
2313                                    size_t size, loff_t *pos)
2314 {
2315         struct amdgpu_device *adev = file_inode(f)->i_private;
2316         ssize_t result = 0;
2317         int r;
2318
2319         while (size) {
2320                 loff_t p = *pos / PAGE_SIZE;
2321                 unsigned off = *pos & ~PAGE_MASK;
2322                 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
2323                 struct page *page;
2324                 void *ptr;
2325
2326                 if (p >= adev->gart.num_cpu_pages)
2327                         return result;
2328
2329                 page = adev->gart.pages[p];
2330                 if (page) {
2331                         ptr = kmap(page);
2332                         ptr += off;
2333
2334                         r = copy_to_user(buf, ptr, cur_size);
2335                         kunmap(adev->gart.pages[p]);
2336                 } else
2337                         r = clear_user(buf, cur_size);
2338
2339                 if (r)
2340                         return -EFAULT;
2341
2342                 result += cur_size;
2343                 buf += cur_size;
2344                 *pos += cur_size;
2345                 size -= cur_size;
2346         }
2347
2348         return result;
2349 }
2350
2351 static const struct file_operations amdgpu_ttm_gtt_fops = {
2352         .owner = THIS_MODULE,
2353         .read = amdgpu_ttm_gtt_read,
2354         .llseek = default_llseek
2355 };
2356
2357 #endif
2358
2359 /**
2360  * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2361  *
2362  * This function is used to read memory that has been mapped to the
2363  * GPU and the known addresses are not physical addresses but instead
2364  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2365  */
2366 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2367                                  size_t size, loff_t *pos)
2368 {
2369         struct amdgpu_device *adev = file_inode(f)->i_private;
2370         struct iommu_domain *dom;
2371         ssize_t result = 0;
2372         int r;
2373
2374         /* retrieve the IOMMU domain if any for this device */
2375         dom = iommu_get_domain_for_dev(adev->dev);
2376
2377         while (size) {
2378                 phys_addr_t addr = *pos & PAGE_MASK;
2379                 loff_t off = *pos & ~PAGE_MASK;
2380                 size_t bytes = PAGE_SIZE - off;
2381                 unsigned long pfn;
2382                 struct page *p;
2383                 void *ptr;
2384
2385                 bytes = bytes < size ? bytes : size;
2386
2387                 /* Translate the bus address to a physical address.  If
2388                  * the domain is NULL it means there is no IOMMU active
2389                  * and the address translation is the identity
2390                  */
2391                 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2392
2393                 pfn = addr >> PAGE_SHIFT;
2394                 if (!pfn_valid(pfn))
2395                         return -EPERM;
2396
2397                 p = pfn_to_page(pfn);
2398                 if (p->mapping != adev->mman.bdev.dev_mapping)
2399                         return -EPERM;
2400
2401                 ptr = kmap(p);
2402                 r = copy_to_user(buf, ptr + off, bytes);
2403                 kunmap(p);
2404                 if (r)
2405                         return -EFAULT;
2406
2407                 size -= bytes;
2408                 *pos += bytes;
2409                 result += bytes;
2410         }
2411
2412         return result;
2413 }
2414
2415 /**
2416  * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2417  *
2418  * This function is used to write memory that has been mapped to the
2419  * GPU and the known addresses are not physical addresses but instead
2420  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2421  */
2422 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2423                                  size_t size, loff_t *pos)
2424 {
2425         struct amdgpu_device *adev = file_inode(f)->i_private;
2426         struct iommu_domain *dom;
2427         ssize_t result = 0;
2428         int r;
2429
2430         dom = iommu_get_domain_for_dev(adev->dev);
2431
2432         while (size) {
2433                 phys_addr_t addr = *pos & PAGE_MASK;
2434                 loff_t off = *pos & ~PAGE_MASK;
2435                 size_t bytes = PAGE_SIZE - off;
2436                 unsigned long pfn;
2437                 struct page *p;
2438                 void *ptr;
2439
2440                 bytes = bytes < size ? bytes : size;
2441
2442                 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2443
2444                 pfn = addr >> PAGE_SHIFT;
2445                 if (!pfn_valid(pfn))
2446                         return -EPERM;
2447
2448                 p = pfn_to_page(pfn);
2449                 if (p->mapping != adev->mman.bdev.dev_mapping)
2450                         return -EPERM;
2451
2452                 ptr = kmap(p);
2453                 r = copy_from_user(ptr + off, buf, bytes);
2454                 kunmap(p);
2455                 if (r)
2456                         return -EFAULT;
2457
2458                 size -= bytes;
2459                 *pos += bytes;
2460                 result += bytes;
2461         }
2462
2463         return result;
2464 }
2465
2466 static const struct file_operations amdgpu_ttm_iomem_fops = {
2467         .owner = THIS_MODULE,
2468         .read = amdgpu_iomem_read,
2469         .write = amdgpu_iomem_write,
2470         .llseek = default_llseek
2471 };
2472
2473 static const struct {
2474         char *name;
2475         const struct file_operations *fops;
2476         int domain;
2477 } ttm_debugfs_entries[] = {
2478         { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
2479 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2480         { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
2481 #endif
2482         { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2483 };
2484
2485 #endif
2486
2487 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2488 {
2489 #if defined(CONFIG_DEBUG_FS)
2490         unsigned count;
2491
2492         struct drm_minor *minor = adev->ddev->primary;
2493         struct dentry *ent, *root = minor->debugfs_root;
2494
2495         for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
2496                 ent = debugfs_create_file(
2497                                 ttm_debugfs_entries[count].name,
2498                                 S_IFREG | S_IRUGO, root,
2499                                 adev,
2500                                 ttm_debugfs_entries[count].fops);
2501                 if (IS_ERR(ent))
2502                         return PTR_ERR(ent);
2503                 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2504                         i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2505                 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2506                         i_size_write(ent->d_inode, adev->gmc.gart_size);
2507                 adev->mman.debugfs_entries[count] = ent;
2508         }
2509
2510         count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
2511
2512 #ifdef CONFIG_SWIOTLB
2513         if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
2514                 --count;
2515 #endif
2516
2517         return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
2518 #else
2519         return 0;
2520 #endif
2521 }
2522
2523 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
2524 {
2525 #if defined(CONFIG_DEBUG_FS)
2526         unsigned i;
2527
2528         for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
2529                 debugfs_remove(adev->mman.debugfs_entries[i]);
2530 #endif
2531 }