drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <[email protected]>
  29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30  *    Dave Airlie
  31  */
  32
  33 #include <linux/dma-mapping.h>
  34 #include <linux/iommu.h>
  35 #include <linux/hmm.h>
  36 #include <linux/pagemap.h>
  37 #include <linux/sched/task.h>
  38 #include <linux/seq_file.h>
  39 #include <linux/slab.h>
  40 #include <linux/swap.h>
  41 #include <linux/swiotlb.h>
  42
  43 #include <drm/ttm/ttm_bo_api.h>
  44 #include <drm/ttm/ttm_bo_driver.h>
  45 #include <drm/ttm/ttm_placement.h>
  46 #include <drm/ttm/ttm_module.h>
  47 #include <drm/ttm/ttm_page_alloc.h>
  48
  49 #include <drm/drm_debugfs.h>
  50 #include <drm/amdgpu_drm.h>
  51
  52 #include "amdgpu.h"
  53 #include "amdgpu_object.h"
  54 #include "amdgpu_trace.h"
  55 #include "amdgpu_amdkfd.h"
  56 #include "amdgpu_sdma.h"
  57 #include "bif/bif_4_1_d.h"
  58
  59 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
  60                              struct ttm_mem_reg *mem, unsigned num_pages,
  61                              uint64_t offset, unsigned window,
  62                              struct amdgpu_ring *ring,
  63                              uint64_t *addr);
  64
  65 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
  66 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
  67
  68 static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
  69 {
  70         return 0;
  71 }
  72
  73 /**
  74  * amdgpu_init_mem_type - Initialize a memory manager for a specific type of
  75  * memory request.
  76  *
  77  * @bdev: The TTM BO device object (contains a reference to amdgpu_device)
  78  * @type: The type of memory requested
  79  * @man: The memory type manager for each domain
  80  *
  81  * This is called by ttm_bo_init_mm() when a buffer object is being
  82  * initialized.
  83  */
  84 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
  85                                 struct ttm_mem_type_manager *man)
  86 {
  87         struct amdgpu_device *adev;
  88
  89         adev = amdgpu_ttm_adev(bdev);
  90
  91         switch (type) {
  92         case TTM_PL_SYSTEM:
  93                 /* System memory */
  94                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
  95                 man->available_caching = TTM_PL_MASK_CACHING;
  96                 man->default_caching = TTM_PL_FLAG_CACHED;
  97                 break;
  98         case TTM_PL_TT:
  99                 /* GTT memory  */
 100                 man->func = &amdgpu_gtt_mgr_func;
 101                 man->gpu_offset = adev->gmc.gart_start;
 102                 man->available_caching = TTM_PL_MASK_CACHING;
 103                 man->default_caching = TTM_PL_FLAG_CACHED;
 104                 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
 105                 break;
 106         case TTM_PL_VRAM:
 107                 /* "On-card" video ram */
 108                 man->func = &amdgpu_vram_mgr_func;
 109                 man->gpu_offset = adev->gmc.vram_start;
 110                 man->flags = TTM_MEMTYPE_FLAG_FIXED |
 111                              TTM_MEMTYPE_FLAG_MAPPABLE;
 112                 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
 113                 man->default_caching = TTM_PL_FLAG_WC;
 114                 break;
 115         case AMDGPU_PL_GDS:
 116         case AMDGPU_PL_GWS:
 117         case AMDGPU_PL_OA:
 118                 /* On-chip GDS memory*/
 119                 man->func = &ttm_bo_manager_func;
 120                 man->gpu_offset = 0;
 121                 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
 122                 man->available_caching = TTM_PL_FLAG_UNCACHED;
 123                 man->default_caching = TTM_PL_FLAG_UNCACHED;
 124                 break;
 125         default:
 126                 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
 127                 return -EINVAL;
 128         }
 129         return 0;
 130 }
 131
 132 /**
 133  * amdgpu_evict_flags - Compute placement flags
 134  *
 135  * @bo: The buffer object to evict
 136  * @placement: Possible destination(s) for evicted BO
 137  *
 138  * Fill in placement data when ttm_bo_evict() is called
 139  */
 140 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 141                                 struct ttm_placement *placement)
 142 {
 143         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 144         struct amdgpu_bo *abo;
 145         static const struct ttm_place placements = {
 146                 .fpfn = 0,
 147                 .lpfn = 0,
 148                 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
 149         };
 150
 151         /* Don't handle scatter gather BOs */
 152         if (bo->type == ttm_bo_type_sg) {
 153                 placement->num_placement = 0;
 154                 placement->num_busy_placement = 0;
 155                 return;
 156         }
 157
 158         /* Object isn't an AMDGPU object so ignore */
 159         if (!amdgpu_bo_is_amdgpu_bo(bo)) {
 160                 placement->placement = &placements;
 161                 placement->busy_placement = &placements;
 162                 placement->num_placement = 1;
 163                 placement->num_busy_placement = 1;
 164                 return;
 165         }
 166
 167         abo = ttm_to_amdgpu_bo(bo);
 168         switch (bo->mem.mem_type) {
 169         case AMDGPU_PL_GDS:
 170         case AMDGPU_PL_GWS:
 171         case AMDGPU_PL_OA:
 172                 placement->num_placement = 0;
 173                 placement->num_busy_placement = 0;
 174                 return;
 175
 176         case TTM_PL_VRAM:
 177                 if (!adev->mman.buffer_funcs_enabled) {
 178                         /* Move to system memory */
 179                         amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 180                 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
 181                            !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
 182                            amdgpu_bo_in_cpu_visible_vram(abo)) {
 183
 184                         /* Try evicting to the CPU inaccessible part of VRAM
 185                          * first, but only set GTT as busy placement, so this
 186                          * BO will be evicted to GTT rather than causing other
 187                          * BOs to be evicted from VRAM
 188                          */
 189                         amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
 190                                                          AMDGPU_GEM_DOMAIN_GTT);
 191                         abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
 192                         abo->placements[0].lpfn = 0;
 193                         abo->placement.busy_placement = &abo->placements[1];
 194                         abo->placement.num_busy_placement = 1;
 195                 } else {
 196                         /* Move to GTT memory */
 197                         amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
 198                 }
 199                 break;
 200         case TTM_PL_TT:
 201         default:
 202                 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 203                 break;
 204         }
 205         *placement = abo->placement;
 206 }
 207
 208 /**
 209  * amdgpu_verify_access - Verify access for a mmap call
 210  *
 211  * @bo: The buffer object to map
 212  * @filp: The file pointer from the process performing the mmap
 213  *
 214  * This is called by ttm_bo_mmap() to verify whether a process
 215  * has the right to mmap a BO to their process space.
 216  */
 217 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
 218 {
 219         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
 220
 221         /*
 222          * Don't verify access for KFD BOs. They don't have a GEM
 223          * object associated with them.
 224          */
 225         if (abo->kfd_bo)
 226                 return 0;
 227
 228         if (amdgpu_ttm_tt_get_usermm(bo->ttm))
 229                 return -EPERM;
 230         return drm_vma_node_verify_access(&abo->gem_base.vma_node,
 231                                           filp->private_data);
 232 }
 233
 234 /**
 235  * amdgpu_move_null - Register memory for a buffer object
 236  *
 237  * @bo: The bo to assign the memory to
 238  * @new_mem: The memory to be assigned.
 239  *
 240  * Assign the memory from new_mem to the memory of the buffer object bo.
 241  */
 242 static void amdgpu_move_null(struct ttm_buffer_object *bo,
 243                              struct ttm_mem_reg *new_mem)
 244 {
 245         struct ttm_mem_reg *old_mem = &bo->mem;
 246
 247         BUG_ON(old_mem->mm_node != NULL);
 248         *old_mem = *new_mem;
 249         new_mem->mm_node = NULL;
 250 }
 251
 252 /**
 253  * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer.
 254  *
 255  * @bo: The bo to assign the memory to.
 256  * @mm_node: Memory manager node for drm allocator.
 257  * @mem: The region where the bo resides.
 258  *
 259  */
 260 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 261                                     struct drm_mm_node *mm_node,
 262                                     struct ttm_mem_reg *mem)
 263 {
 264         uint64_t addr = 0;
 265
 266         if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {
 267                 addr = mm_node->start << PAGE_SHIFT;
 268                 addr += bo->bdev->man[mem->mem_type].gpu_offset;
 269         }
 270         return addr;
 271 }
 272
 273 /**
 274  * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to
 275  * @offset. It also modifies the offset to be within the drm_mm_node returned
 276  *
 277  * @mem: The region where the bo resides.
 278  * @offset: The offset that drm_mm_node is used for finding.
 279  *
 280  */
 281 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
 282                                                unsigned long *offset)
 283 {
 284         struct drm_mm_node *mm_node = mem->mm_node;
 285
 286         while (*offset >= (mm_node->size << PAGE_SHIFT)) {
 287                 *offset -= (mm_node->size << PAGE_SHIFT);
 288                 ++mm_node;
 289         }
 290         return mm_node;
 291 }
 292
 293 /**
 294  * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
 295  *
 296  * The function copies @size bytes from {src->mem + src->offset} to
 297  * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
 298  * move and different for a BO to BO copy.
 299  *
 300  * @f: Returns the last fence if multiple jobs are submitted.
 301  */
 302 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 303                                struct amdgpu_copy_mem *src,
 304                                struct amdgpu_copy_mem *dst,
 305                                uint64_t size,
 306                                struct reservation_object *resv,
 307                                struct dma_fence **f)
 308 {
 309         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 310         struct drm_mm_node *src_mm, *dst_mm;
 311         uint64_t src_node_start, dst_node_start, src_node_size,
 312                  dst_node_size, src_page_offset, dst_page_offset;
 313         struct dma_fence *fence = NULL;
 314         int r = 0;
 315         const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
 316                                         AMDGPU_GPU_PAGE_SIZE);
 317
 318         if (!adev->mman.buffer_funcs_enabled) {
 319                 DRM_ERROR("Trying to move memory with ring turned off.\n");
 320                 return -EINVAL;
 321         }
 322
 323         src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
 324         src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
 325                                              src->offset;
 326         src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
 327         src_page_offset = src_node_start & (PAGE_SIZE - 1);
 328
 329         dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
 330         dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
 331                                              dst->offset;
 332         dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
 333         dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
 334
 335         mutex_lock(&adev->mman.gtt_window_lock);
 336
 337         while (size) {
 338                 unsigned long cur_size;
 339                 uint64_t from = src_node_start, to = dst_node_start;
 340                 struct dma_fence *next;
 341
 342                 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
 343                  * begins at an offset, then adjust the size accordingly
 344                  */
 345                 cur_size = min3(min(src_node_size, dst_node_size), size,
 346                                 GTT_MAX_BYTES);
 347                 if (cur_size + src_page_offset > GTT_MAX_BYTES ||
 348                     cur_size + dst_page_offset > GTT_MAX_BYTES)
 349                         cur_size -= max(src_page_offset, dst_page_offset);
 350
 351                 /* Map only what needs to be accessed. Map src to window 0 and
 352                  * dst to window 1
 353                  */
 354                 if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {
 355                         r = amdgpu_map_buffer(src->bo, src->mem,
 356                                         PFN_UP(cur_size + src_page_offset),
 357                                         src_node_start, 0, ring,
 358                                         &from);
 359                         if (r)
 360                                 goto error;
 361                         /* Adjust the offset because amdgpu_map_buffer returns
 362                          * start of mapped page
 363                          */
 364                         from += src_page_offset;
 365                 }
 366
 367                 if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {
 368                         r = amdgpu_map_buffer(dst->bo, dst->mem,
 369                                         PFN_UP(cur_size + dst_page_offset),
 370                                         dst_node_start, 1, ring,
 371                                         &to);
 372                         if (r)
 373                                 goto error;
 374                         to += dst_page_offset;
 375                 }
 376
 377                 r = amdgpu_copy_buffer(ring, from, to, cur_size,
 378                                        resv, &next, false, true);
 379                 if (r)
 380                         goto error;
 381
 382                 dma_fence_put(fence);
 383                 fence = next;
 384
 385                 size -= cur_size;
 386                 if (!size)
 387                         break;
 388
 389                 src_node_size -= cur_size;
 390                 if (!src_node_size) {
 391                         src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
 392                                                              src->mem);
 393                         src_node_size = (src_mm->size << PAGE_SHIFT);
 394                         src_page_offset = 0;
 395                 } else {
 396                         src_node_start += cur_size;
 397                         src_page_offset = src_node_start & (PAGE_SIZE - 1);
 398                 }
 399                 dst_node_size -= cur_size;
 400                 if (!dst_node_size) {
 401                         dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
 402                                                              dst->mem);
 403                         dst_node_size = (dst_mm->size << PAGE_SHIFT);
 404                         dst_page_offset = 0;
 405                 } else {
 406                         dst_node_start += cur_size;
 407                         dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
 408                 }
 409         }
 410 error:
 411         mutex_unlock(&adev->mman.gtt_window_lock);
 412         if (f)
 413                 *f = dma_fence_get(fence);
 414         dma_fence_put(fence);
 415         return r;
 416 }
 417
 418 /**
 419  * amdgpu_move_blit - Copy an entire buffer to another buffer
 420  *
 421  * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
 422  * help move buffers to and from VRAM.
 423  */
 424 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 425                             bool evict, bool no_wait_gpu,
 426                             struct ttm_mem_reg *new_mem,
 427                             struct ttm_mem_reg *old_mem)
 428 {
 429         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 430         struct amdgpu_copy_mem src, dst;
 431         struct dma_fence *fence = NULL;
 432         int r;
 433
 434         src.bo = bo;
 435         dst.bo = bo;
 436         src.mem = old_mem;
 437         dst.mem = new_mem;
 438         src.offset = 0;
 439         dst.offset = 0;
 440
 441         r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
 442                                        new_mem->num_pages << PAGE_SHIFT,
 443                                        bo->resv, &fence);
 444         if (r)
 445                 goto error;
 446
 447         /* Always block for VM page tables before committing the new location */
 448         if (bo->type == ttm_bo_type_kernel)
 449                 r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);
 450         else
 451                 r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
 452         dma_fence_put(fence);
 453         return r;
 454
 455 error:
 456         if (fence)
 457                 dma_fence_wait(fence, false);
 458         dma_fence_put(fence);
 459         return r;
 460 }
 461
 462 /**
 463  * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
 464  *
 465  * Called by amdgpu_bo_move().
 466  */
 467 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
 468                                 struct ttm_operation_ctx *ctx,
 469                                 struct ttm_mem_reg *new_mem)
 470 {
 471         struct amdgpu_device *adev;
 472         struct ttm_mem_reg *old_mem = &bo->mem;
 473         struct ttm_mem_reg tmp_mem;
 474         struct ttm_place placements;
 475         struct ttm_placement placement;
 476         int r;
 477
 478         adev = amdgpu_ttm_adev(bo->bdev);
 479
 480         /* create space/pages for new_mem in GTT space */
 481         tmp_mem = *new_mem;
 482         tmp_mem.mm_node = NULL;
 483         placement.num_placement = 1;
 484         placement.placement = &placements;
 485         placement.num_busy_placement = 1;
 486         placement.busy_placement = &placements;
 487         placements.fpfn = 0;
 488         placements.lpfn = 0;
 489         placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 490         r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
 491         if (unlikely(r)) {
 492                 pr_err("Failed to find GTT space for blit from VRAM\n");
 493                 return r;
 494         }
 495
 496         /* set caching flags */
 497         r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
 498         if (unlikely(r)) {
 499                 goto out_cleanup;
 500         }
 501
 502         /* Bind the memory to the GTT space */
 503         r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
 504         if (unlikely(r)) {
 505                 goto out_cleanup;
 506         }
 507
 508         /* blit VRAM to GTT */
 509         r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);
 510         if (unlikely(r)) {
 511                 goto out_cleanup;
 512         }
 513
 514         /* move BO (in tmp_mem) to new_mem */
 515         r = ttm_bo_move_ttm(bo, ctx, new_mem);
 516 out_cleanup:
 517         ttm_bo_mem_put(bo, &tmp_mem);
 518         return r;
 519 }
 520
 521 /**
 522  * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
 523  *
 524  * Called by amdgpu_bo_move().
 525  */
 526 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
 527                                 struct ttm_operation_ctx *ctx,
 528                                 struct ttm_mem_reg *new_mem)
 529 {
 530         struct amdgpu_device *adev;
 531         struct ttm_mem_reg *old_mem = &bo->mem;
 532         struct ttm_mem_reg tmp_mem;
 533         struct ttm_placement placement;
 534         struct ttm_place placements;
 535         int r;
 536
 537         adev = amdgpu_ttm_adev(bo->bdev);
 538
 539         /* make space in GTT for old_mem buffer */
 540         tmp_mem = *new_mem;
 541         tmp_mem.mm_node = NULL;
 542         placement.num_placement = 1;
 543         placement.placement = &placements;
 544         placement.num_busy_placement = 1;
 545         placement.busy_placement = &placements;
 546         placements.fpfn = 0;
 547         placements.lpfn = 0;
 548         placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 549         r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
 550         if (unlikely(r)) {
 551                 pr_err("Failed to find GTT space for blit to VRAM\n");
 552                 return r;
 553         }
 554
 555         /* move/bind old memory to GTT space */
 556         r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
 557         if (unlikely(r)) {
 558                 goto out_cleanup;
 559         }
 560
 561         /* copy to VRAM */
 562         r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem);
 563         if (unlikely(r)) {
 564                 goto out_cleanup;
 565         }
 566 out_cleanup:
 567         ttm_bo_mem_put(bo, &tmp_mem);
 568         return r;
 569 }
 570
 571 /**
 572  * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
 573  *
 574  * Called by amdgpu_bo_move()
 575  */
 576 static bool amdgpu_mem_visible(struct amdgpu_device *adev,
 577                                struct ttm_mem_reg *mem)
 578 {
 579         struct drm_mm_node *nodes = mem->mm_node;
 580
 581         if (mem->mem_type == TTM_PL_SYSTEM ||
 582             mem->mem_type == TTM_PL_TT)
 583                 return true;
 584         if (mem->mem_type != TTM_PL_VRAM)
 585                 return false;
 586
 587         /* ttm_mem_reg_ioremap only supports contiguous memory */
 588         if (nodes->size != mem->num_pages)
 589                 return false;
 590
 591         return ((nodes->start + nodes->size) << PAGE_SHIFT)
 592                 <= adev->gmc.visible_vram_size;
 593 }
 594
 595 /**
 596  * amdgpu_bo_move - Move a buffer object to a new memory location
 597  *
 598  * Called by ttm_bo_handle_move_mem()
 599  */
 600 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 601                           struct ttm_operation_ctx *ctx,
 602                           struct ttm_mem_reg *new_mem)
 603 {
 604         struct amdgpu_device *adev;
 605         struct amdgpu_bo *abo;
 606         struct ttm_mem_reg *old_mem = &bo->mem;
 607         int r;
 608
 609         /* Can't move a pinned BO */
 610         abo = ttm_to_amdgpu_bo(bo);
 611         if (WARN_ON_ONCE(abo->pin_count > 0))
 612                 return -EINVAL;
 613
 614         adev = amdgpu_ttm_adev(bo->bdev);
 615
 616         if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
 617                 amdgpu_move_null(bo, new_mem);
 618                 return 0;
 619         }
 620         if ((old_mem->mem_type == TTM_PL_TT &&
 621              new_mem->mem_type == TTM_PL_SYSTEM) ||
 622             (old_mem->mem_type == TTM_PL_SYSTEM &&
 623              new_mem->mem_type == TTM_PL_TT)) {
 624                 /* bind is enough */
 625                 amdgpu_move_null(bo, new_mem);
 626                 return 0;
 627         }
 628         if (old_mem->mem_type == AMDGPU_PL_GDS ||
 629             old_mem->mem_type == AMDGPU_PL_GWS ||
 630             old_mem->mem_type == AMDGPU_PL_OA ||
 631             new_mem->mem_type == AMDGPU_PL_GDS ||
 632             new_mem->mem_type == AMDGPU_PL_GWS ||
 633             new_mem->mem_type == AMDGPU_PL_OA) {
 634                 /* Nothing to save here */
 635                 amdgpu_move_null(bo, new_mem);
 636                 return 0;
 637         }
 638
 639         if (!adev->mman.buffer_funcs_enabled) {
 640                 r = -ENODEV;
 641                 goto memcpy;
 642         }
 643
 644         if (old_mem->mem_type == TTM_PL_VRAM &&
 645             new_mem->mem_type == TTM_PL_SYSTEM) {
 646                 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
 647         } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
 648                    new_mem->mem_type == TTM_PL_VRAM) {
 649                 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
 650         } else {
 651                 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
 652                                      new_mem, old_mem);
 653         }
 654
 655         if (r) {
 656 memcpy:
 657                 /* Check that all memory is CPU accessible */
 658                 if (!amdgpu_mem_visible(adev, old_mem) ||
 659                     !amdgpu_mem_visible(adev, new_mem)) {
 660                         pr_err("Move buffer fallback to memcpy unavailable\n");
 661                         return r;
 662                 }
 663
 664                 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
 665                 if (r)
 666                         return r;
 667         }
 668
 669         if (bo->type == ttm_bo_type_device &&
 670             new_mem->mem_type == TTM_PL_VRAM &&
 671             old_mem->mem_type != TTM_PL_VRAM) {
 672                 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
 673                  * accesses the BO after it's moved.
 674                  */
 675                 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 676         }
 677
 678         /* update statistics */
 679         atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
 680         return 0;
 681 }
 682
 683 /**
 684  * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
 685  *
 686  * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
 687  */
 688 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 689 {
 690         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 691         struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 692         struct drm_mm_node *mm_node = mem->mm_node;
 693
 694         mem->bus.addr = NULL;
 695         mem->bus.offset = 0;
 696         mem->bus.size = mem->num_pages << PAGE_SHIFT;
 697         mem->bus.base = 0;
 698         mem->bus.is_iomem = false;
 699         if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
 700                 return -EINVAL;
 701         switch (mem->mem_type) {
 702         case TTM_PL_SYSTEM:
 703                 /* system memory */
 704                 return 0;
 705         case TTM_PL_TT:
 706                 break;
 707         case TTM_PL_VRAM:
 708                 mem->bus.offset = mem->start << PAGE_SHIFT;
 709                 /* check if it's visible */
 710                 if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
 711                         return -EINVAL;
 712                 /* Only physically contiguous buffers apply. In a contiguous
 713                  * buffer, size of the first mm_node would match the number of
 714                  * pages in ttm_mem_reg.
 715                  */
 716                 if (adev->mman.aper_base_kaddr &&
 717                     (mm_node->size == mem->num_pages))
 718                         mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
 719                                         mem->bus.offset;
 720
 721                 mem->bus.base = adev->gmc.aper_base;
 722                 mem->bus.is_iomem = true;
 723                 break;
 724         default:
 725                 return -EINVAL;
 726         }
 727         return 0;
 728 }
 729
 730 static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 731 {
 732 }
 733
 734 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
 735                                            unsigned long page_offset)
 736 {
 737         struct drm_mm_node *mm;
 738         unsigned long offset = (page_offset << PAGE_SHIFT);
 739
 740         mm = amdgpu_find_mm_node(&bo->mem, &offset);
 741         return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
 742                 (offset >> PAGE_SHIFT);
 743 }
 744
 745 /*
 746  * TTM backend functions.
 747  */
 748 struct amdgpu_ttm_tt {
 749         struct ttm_dma_tt       ttm;
 750         u64                     offset;
 751         uint64_t                userptr;
 752         struct task_struct      *usertask;
 753         uint32_t                userflags;
 754 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
 755         struct hmm_range        *range;
 756 #endif
 757 };
 758
 759 /**
 760  * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
 761  * memory and start HMM tracking CPU page table update
 762  *
 763  * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
 764  * once afterwards to stop HMM tracking
 765  */
 766 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
 767
 768 #define MAX_RETRY_HMM_RANGE_FAULT       16
 769
 770 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
 771 {
 772         struct hmm_mirror *mirror = bo->mn ? &bo->mn->mirror : NULL;
 773         struct ttm_tt *ttm = bo->tbo.ttm;
 774         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 775         struct mm_struct *mm = gtt->usertask->mm;
 776         unsigned long start = gtt->userptr;
 777         struct vm_area_struct *vma;
 778         struct hmm_range *range;
 779         unsigned long i;
 780         uint64_t *pfns;
 781         int retry = 0;
 782         int r = 0;
 783
 784         if (!mm) /* Happens during process shutdown */
 785                 return -ESRCH;
 786
 787         if (unlikely(!mirror)) {
 788                 DRM_DEBUG_DRIVER("Failed to get hmm_mirror\n");
 789                 r = -EFAULT;
 790                 goto out;
 791         }
 792
 793         vma = find_vma(mm, start);
 794         if (unlikely(!vma || start < vma->vm_start)) {
 795                 r = -EFAULT;
 796                 goto out;
 797         }
 798         if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
 799                 vma->vm_file)) {
 800                 r = -EPERM;
 801                 goto out;
 802         }
 803
 804         range = kzalloc(sizeof(*range), GFP_KERNEL);
 805         if (unlikely(!range)) {
 806                 r = -ENOMEM;
 807                 goto out;
 808         }
 809
 810         pfns = kvmalloc_array(ttm->num_pages, sizeof(*pfns), GFP_KERNEL);
 811         if (unlikely(!pfns)) {
 812                 r = -ENOMEM;
 813                 goto out_free_ranges;
 814         }
 815
 816         amdgpu_hmm_init_range(range);
 817         range->default_flags = range->flags[HMM_PFN_VALID];
 818         range->default_flags |= amdgpu_ttm_tt_is_readonly(ttm) ?
 819                                 0 : range->flags[HMM_PFN_WRITE];
 820         range->pfn_flags_mask = 0;
 821         range->pfns = pfns;
 822         hmm_range_register(range, mirror, start,
 823                            start + ttm->num_pages * PAGE_SIZE, PAGE_SHIFT);
 824
 825 retry:
 826         /*
 827          * Just wait for range to be valid, safe to ignore return value as we
 828          * will use the return value of hmm_range_fault() below under the
 829          * mmap_sem to ascertain the validity of the range.
 830          */
 831         hmm_range_wait_until_valid(range, HMM_RANGE_DEFAULT_TIMEOUT);
 832
 833         down_read(&mm->mmap_sem);
 834
 835         r = hmm_range_fault(range, true);
 836         if (unlikely(r < 0)) {
 837                 if (likely(r == -EAGAIN)) {
 838                         /*
 839                          * return -EAGAIN, mmap_sem is dropped
 840                          */
 841                         if (retry++ < MAX_RETRY_HMM_RANGE_FAULT)
 842                                 goto retry;
 843                         else
 844                                 pr_err("Retry hmm fault too many times\n");
 845                 }
 846
 847                 goto out_up_read;
 848         }
 849
 850         up_read(&mm->mmap_sem);
 851
 852         for (i = 0; i < ttm->num_pages; i++) {
 853                 pages[i] = hmm_device_entry_to_page(range, pfns[i]);
 854                 if (unlikely(!pages[i])) {
 855                         pr_err("Page fault failed for pfn[%lu] = 0x%llx\n",
 856                                i, pfns[i]);
 857                         r = -ENOMEM;
 858
 859                         goto out_free_pfns;
 860                 }
 861         }
 862
 863         gtt->range = range;
 864
 865         return 0;
 866
 867 out_up_read:
 868         if (likely(r != -EAGAIN))
 869                 up_read(&mm->mmap_sem);
 870 out_free_pfns:
 871         hmm_range_unregister(range);
 872         kvfree(pfns);
 873 out_free_ranges:
 874         kfree(range);
 875 out:
 876         return r;
 877 }
 878
 879 /**
 880  * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change
 881  * Check if the pages backing this ttm range have been invalidated
 882  *
 883  * Returns: true if pages are still valid
 884  */
 885 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
 886 {
 887         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 888         bool r = false;
 889
 890         if (!gtt || !gtt->userptr)
 891                 return false;
 892
 893         DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n",
 894                 gtt->userptr, ttm->num_pages);
 895
 896         WARN_ONCE(!gtt->range || !gtt->range->pfns,
 897                 "No user pages to check\n");
 898
 899         if (gtt->range) {
 900                 r = hmm_range_valid(gtt->range);
 901                 hmm_range_unregister(gtt->range);
 902
 903                 kvfree(gtt->range->pfns);
 904                 kfree(gtt->range);
 905                 gtt->range = NULL;
 906         }
 907
 908         return r;
 909 }
 910 #endif
 911
 912 /**
 913  * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
 914  *
 915  * Called by amdgpu_cs_list_validate(). This creates the page list
 916  * that backs user memory and will ultimately be mapped into the device
 917  * address space.
 918  */
 919 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
 920 {
 921         unsigned long i;
 922
 923         for (i = 0; i < ttm->num_pages; ++i)
 924                 ttm->pages[i] = pages ? pages[i] : NULL;
 925 }
 926
 927 /**
 928  * amdgpu_ttm_tt_pin_userptr -  prepare the sg table with the user pages
 929  *
 930  * Called by amdgpu_ttm_backend_bind()
 931  **/
 932 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 933 {
 934         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 935         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 936         unsigned nents;
 937         int r;
 938
 939         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 940         enum dma_data_direction direction = write ?
 941                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 942
 943         /* Allocate an SG array and squash pages into it */
 944         r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
 945                                       ttm->num_pages << PAGE_SHIFT,
 946                                       GFP_KERNEL);
 947         if (r)
 948                 goto release_sg;
 949
 950         /* Map SG to device */
 951         r = -ENOMEM;
 952         nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 953         if (nents != ttm->sg->nents)
 954                 goto release_sg;
 955
 956         /* convert SG to linear array of pages and dma addresses */
 957         drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
 958                                          gtt->ttm.dma_address, ttm->num_pages);
 959
 960         return 0;
 961
 962 release_sg:
 963         kfree(ttm->sg);
 964         return r;
 965 }
 966
 967 /**
 968  * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
 969  */
 970 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 971 {
 972         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 973         struct amdgpu_ttm_tt *gtt = (void *)ttm;
 974
 975         int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
 976         enum dma_data_direction direction = write ?
 977                 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 978
 979         /* double check that we don't free the table twice */
 980         if (!ttm->sg->sgl)
 981                 return;
 982
 983         /* unmap the pages mapped to the device */
 984         dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
 985
 986         sg_free_table(ttm->sg);
 987
 988 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
 989         if (gtt->range &&
 990             ttm->pages[0] == hmm_device_entry_to_page(gtt->range,
 991                                                       gtt->range->pfns[0]))
 992                 WARN_ONCE(1, "Missing get_user_page_done\n");
 993 #endif
 994 }
 995
 996 int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
 997                                 struct ttm_buffer_object *tbo,
 998                                 uint64_t flags)
 999 {
1000         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
1001         struct ttm_tt *ttm = tbo->ttm;
1002         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1003         int r;
1004
1005         if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
1006                 uint64_t page_idx = 1;
1007
1008                 r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
1009                                 ttm->pages, gtt->ttm.dma_address, flags);
1010                 if (r)
1011                         goto gart_bind_fail;
1012
1013                 /* Patch mtype of the second part BO */
1014                 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1015                 flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC);
1016
1017                 r = amdgpu_gart_bind(adev,
1018                                 gtt->offset + (page_idx << PAGE_SHIFT),
1019                                 ttm->num_pages - page_idx,
1020                                 &ttm->pages[page_idx],
1021                                 &(gtt->ttm.dma_address[page_idx]), flags);
1022         } else {
1023                 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1024                                      ttm->pages, gtt->ttm.dma_address, flags);
1025         }
1026
1027 gart_bind_fail:
1028         if (r)
1029                 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1030                           ttm->num_pages, gtt->offset);
1031
1032         return r;
1033 }
1034
1035 /**
1036  * amdgpu_ttm_backend_bind - Bind GTT memory
1037  *
1038  * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
1039  * This handles binding GTT memory to the device address space.
1040  */
1041 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
1042                                    struct ttm_mem_reg *bo_mem)
1043 {
1044         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1045         struct amdgpu_ttm_tt *gtt = (void*)ttm;
1046         uint64_t flags;
1047         int r = 0;
1048
1049         if (gtt->userptr) {
1050                 r = amdgpu_ttm_tt_pin_userptr(ttm);
1051                 if (r) {
1052                         DRM_ERROR("failed to pin userptr\n");
1053                         return r;
1054                 }
1055         }
1056         if (!ttm->num_pages) {
1057                 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
1058                      ttm->num_pages, bo_mem, ttm);
1059         }
1060
1061         if (bo_mem->mem_type == AMDGPU_PL_GDS ||
1062             bo_mem->mem_type == AMDGPU_PL_GWS ||
1063             bo_mem->mem_type == AMDGPU_PL_OA)
1064                 return -EINVAL;
1065
1066         if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
1067                 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
1068                 return 0;
1069         }
1070
1071         /* compute PTE flags relevant to this BO memory */
1072         flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1073
1074         /* bind pages into GART page tables */
1075         gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
1076         r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1077                 ttm->pages, gtt->ttm.dma_address, flags);
1078
1079         if (r)
1080                 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1081                           ttm->num_pages, gtt->offset);
1082         return r;
1083 }
1084
1085 /**
1086  * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
1087  */
1088 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1089 {
1090         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1091         struct ttm_operation_ctx ctx = { false, false };
1092         struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1093         struct ttm_mem_reg tmp;
1094         struct ttm_placement placement;
1095         struct ttm_place placements;
1096         uint64_t addr, flags;
1097         int r;
1098
1099         if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1100                 return 0;
1101
1102         addr = amdgpu_gmc_agp_addr(bo);
1103         if (addr != AMDGPU_BO_INVALID_OFFSET) {
1104                 bo->mem.start = addr >> PAGE_SHIFT;
1105         } else {
1106
1107                 /* allocate GART space */
1108                 tmp = bo->mem;
1109                 tmp.mm_node = NULL;
1110                 placement.num_placement = 1;
1111                 placement.placement = &placements;
1112                 placement.num_busy_placement = 1;
1113                 placement.busy_placement = &placements;
1114                 placements.fpfn = 0;
1115                 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1116                 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
1117                         TTM_PL_FLAG_TT;
1118
1119                 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1120                 if (unlikely(r))
1121                         return r;
1122
1123                 /* compute PTE flags for this buffer object */
1124                 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1125
1126                 /* Bind pages */
1127                 gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1128                 r = amdgpu_ttm_gart_bind(adev, bo, flags);
1129                 if (unlikely(r)) {
1130                         ttm_bo_mem_put(bo, &tmp);
1131                         return r;
1132                 }
1133
1134                 ttm_bo_mem_put(bo, &bo->mem);
1135                 bo->mem = tmp;
1136         }
1137
1138         bo->offset = (bo->mem.start << PAGE_SHIFT) +
1139                 bo->bdev->man[bo->mem.mem_type].gpu_offset;
1140
1141         return 0;
1142 }
1143
1144 /**
1145  * amdgpu_ttm_recover_gart - Rebind GTT pages
1146  *
1147  * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1148  * rebind GTT pages during a GPU reset.
1149  */
1150 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1151 {
1152         struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1153         uint64_t flags;
1154         int r;
1155
1156         if (!tbo->ttm)
1157                 return 0;
1158
1159         flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1160         r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1161
1162         return r;
1163 }
1164
1165 /**
1166  * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1167  *
1168  * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1169  * ttm_tt_destroy().
1170  */
1171 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
1172 {
1173         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1174         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1175         int r;
1176
1177         /* if the pages have userptr pinning then clear that first */
1178         if (gtt->userptr)
1179                 amdgpu_ttm_tt_unpin_userptr(ttm);
1180
1181         if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1182                 return 0;
1183
1184         /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1185         r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1186         if (r)
1187                 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
1188                           gtt->ttm.ttm.num_pages, gtt->offset);
1189         return r;
1190 }
1191
1192 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
1193 {
1194         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1195
1196         if (gtt->usertask)
1197                 put_task_struct(gtt->usertask);
1198
1199         ttm_dma_tt_fini(&gtt->ttm);
1200         kfree(gtt);
1201 }
1202
1203 static struct ttm_backend_func amdgpu_backend_func = {
1204         .bind = &amdgpu_ttm_backend_bind,
1205         .unbind = &amdgpu_ttm_backend_unbind,
1206         .destroy = &amdgpu_ttm_backend_destroy,
1207 };
1208
1209 /**
1210  * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1211  *
1212  * @bo: The buffer object to create a GTT ttm_tt object around
1213  *
1214  * Called by ttm_tt_create().
1215  */
1216 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1217                                            uint32_t page_flags)
1218 {
1219         struct amdgpu_device *adev;
1220         struct amdgpu_ttm_tt *gtt;
1221
1222         adev = amdgpu_ttm_adev(bo->bdev);
1223
1224         gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1225         if (gtt == NULL) {
1226                 return NULL;
1227         }
1228         gtt->ttm.ttm.func = &amdgpu_backend_func;
1229
1230         /* allocate space for the uninitialized page entries */
1231         if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
1232                 kfree(gtt);
1233                 return NULL;
1234         }
1235         return &gtt->ttm.ttm;
1236 }
1237
1238 /**
1239  * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1240  *
1241  * Map the pages of a ttm_tt object to an address space visible
1242  * to the underlying device.
1243  */
1244 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
1245                         struct ttm_operation_ctx *ctx)
1246 {
1247         struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1248         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1249         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1250
1251         /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1252         if (gtt && gtt->userptr) {
1253                 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1254                 if (!ttm->sg)
1255                         return -ENOMEM;
1256
1257                 ttm->page_flags |= TTM_PAGE_FLAG_SG;
1258                 ttm->state = tt_unbound;
1259                 return 0;
1260         }
1261
1262         if (slave && ttm->sg) {
1263                 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1264                                                  gtt->ttm.dma_address,
1265                                                  ttm->num_pages);
1266                 ttm->state = tt_unbound;
1267                 return 0;
1268         }
1269
1270 #ifdef CONFIG_SWIOTLB
1271         if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1272                 return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
1273         }
1274 #endif
1275
1276         /* fall back to generic helper to populate the page array
1277          * and map them to the device */
1278         return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
1279 }
1280
1281 /**
1282  * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1283  *
1284  * Unmaps pages of a ttm_tt object from the device address space and
1285  * unpopulates the page array backing it.
1286  */
1287 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
1288 {
1289         struct amdgpu_device *adev;
1290         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1291         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1292
1293         if (gtt && gtt->userptr) {
1294                 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1295                 kfree(ttm->sg);
1296                 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1297                 return;
1298         }
1299
1300         if (slave)
1301                 return;
1302
1303         adev = amdgpu_ttm_adev(ttm->bdev);
1304
1305 #ifdef CONFIG_SWIOTLB
1306         if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1307                 ttm_dma_unpopulate(&gtt->ttm, adev->dev);
1308                 return;
1309         }
1310 #endif
1311
1312         /* fall back to generic helper to unmap and unpopulate array */
1313         ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
1314 }
1315
1316 /**
1317  * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1318  * task
1319  *
1320  * @ttm: The ttm_tt object to bind this userptr object to
1321  * @addr:  The address in the current tasks VM space to use
1322  * @flags: Requirements of userptr object.
1323  *
1324  * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
1325  * to current task
1326  */
1327 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1328                               uint32_t flags)
1329 {
1330         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1331
1332         if (gtt == NULL)
1333                 return -EINVAL;
1334
1335         gtt->userptr = addr;
1336         gtt->userflags = flags;
1337
1338         if (gtt->usertask)
1339                 put_task_struct(gtt->usertask);
1340         gtt->usertask = current->group_leader;
1341         get_task_struct(gtt->usertask);
1342
1343         return 0;
1344 }
1345
1346 /**
1347  * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1348  */
1349 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1350 {
1351         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1352
1353         if (gtt == NULL)
1354                 return NULL;
1355
1356         if (gtt->usertask == NULL)
1357                 return NULL;
1358
1359         return gtt->usertask->mm;
1360 }
1361
1362 /**
1363  * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1364  * address range for the current task.
1365  *
1366  */
1367 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1368                                   unsigned long end)
1369 {
1370         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1371         unsigned long size;
1372
1373         if (gtt == NULL || !gtt->userptr)
1374                 return false;
1375
1376         /* Return false if no part of the ttm_tt object lies within
1377          * the range
1378          */
1379         size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
1380         if (gtt->userptr > end || gtt->userptr + size <= start)
1381                 return false;
1382
1383         return true;
1384 }
1385
1386 /**
1387  * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1388  */
1389 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1390 {
1391         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1392
1393         if (gtt == NULL || !gtt->userptr)
1394                 return false;
1395
1396         return true;
1397 }
1398
1399 /**
1400  * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1401  */
1402 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1403 {
1404         struct amdgpu_ttm_tt *gtt = (void *)ttm;
1405
1406         if (gtt == NULL)
1407                 return false;
1408
1409         return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1410 }
1411
1412 /**
1413  * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1414  *
1415  * @ttm: The ttm_tt object to compute the flags for
1416  * @mem: The memory registry backing this ttm_tt object
1417  *
1418  * Figure out the flags to use for a VM PDE (Page Directory Entry).
1419  */
1420 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
1421 {
1422         uint64_t flags = 0;
1423
1424         if (mem && mem->mem_type != TTM_PL_SYSTEM)
1425                 flags |= AMDGPU_PTE_VALID;
1426
1427         if (mem && mem->mem_type == TTM_PL_TT) {
1428                 flags |= AMDGPU_PTE_SYSTEM;
1429
1430                 if (ttm->caching_state == tt_cached)
1431                         flags |= AMDGPU_PTE_SNOOPED;
1432         }
1433
1434         return flags;
1435 }
1436
1437 /**
1438  * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1439  *
1440  * @ttm: The ttm_tt object to compute the flags for
1441  * @mem: The memory registry backing this ttm_tt object
1442
1443  * Figure out the flags to use for a VM PTE (Page Table Entry).
1444  */
1445 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1446                                  struct ttm_mem_reg *mem)
1447 {
1448         uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1449
1450         flags |= adev->gart.gart_pte_flags;
1451         flags |= AMDGPU_PTE_READABLE;
1452
1453         if (!amdgpu_ttm_tt_is_readonly(ttm))
1454                 flags |= AMDGPU_PTE_WRITEABLE;
1455
1456         return flags;
1457 }
1458
1459 /**
1460  * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1461  * object.
1462  *
1463  * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1464  * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1465  * it can find space for a new object and by ttm_bo_force_list_clean() which is
1466  * used to clean out a memory space.
1467  */
1468 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1469                                             const struct ttm_place *place)
1470 {
1471         unsigned long num_pages = bo->mem.num_pages;
1472         struct drm_mm_node *node = bo->mem.mm_node;
1473         struct reservation_object_list *flist;
1474         struct dma_fence *f;
1475         int i;
1476
1477         /* Don't evict VM page tables while they are busy, otherwise we can't
1478          * cleanly handle page faults.
1479          */
1480         if (bo->type == ttm_bo_type_kernel &&
1481             !reservation_object_test_signaled_rcu(bo->resv, true))
1482                 return false;
1483
1484         /* If bo is a KFD BO, check if the bo belongs to the current process.
1485          * If true, then return false as any KFD process needs all its BOs to
1486          * be resident to run successfully
1487          */
1488         flist = reservation_object_get_list(bo->resv);
1489         if (flist) {
1490                 for (i = 0; i < flist->shared_count; ++i) {
1491                         f = rcu_dereference_protected(flist->shared[i],
1492                                 reservation_object_held(bo->resv));
1493                         if (amdkfd_fence_check_mm(f, current->mm))
1494                                 return false;
1495                 }
1496         }
1497
1498         switch (bo->mem.mem_type) {
1499         case TTM_PL_TT:
1500                 return true;
1501
1502         case TTM_PL_VRAM:
1503                 /* Check each drm MM node individually */
1504                 while (num_pages) {
1505                         if (place->fpfn < (node->start + node->size) &&
1506                             !(place->lpfn && place->lpfn <= node->start))
1507                                 return true;
1508
1509                         num_pages -= node->size;
1510                         ++node;
1511                 }
1512                 return false;
1513
1514         default:
1515                 break;
1516         }
1517
1518         return ttm_bo_eviction_valuable(bo, place);
1519 }
1520
1521 /**
1522  * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1523  *
1524  * @bo:  The buffer object to read/write
1525  * @offset:  Offset into buffer object
1526  * @buf:  Secondary buffer to write/read from
1527  * @len: Length in bytes of access
1528  * @write:  true if writing
1529  *
1530  * This is used to access VRAM that backs a buffer object via MMIO
1531  * access for debugging purposes.
1532  */
1533 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1534                                     unsigned long offset,
1535                                     void *buf, int len, int write)
1536 {
1537         struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1538         struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1539         struct drm_mm_node *nodes;
1540         uint32_t value = 0;
1541         int ret = 0;
1542         uint64_t pos;
1543         unsigned long flags;
1544
1545         if (bo->mem.mem_type != TTM_PL_VRAM)
1546                 return -EIO;
1547
1548         nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1549         pos = (nodes->start << PAGE_SHIFT) + offset;
1550
1551         while (len && pos < adev->gmc.mc_vram_size) {
1552                 uint64_t aligned_pos = pos & ~(uint64_t)3;
1553                 uint32_t bytes = 4 - (pos & 3);
1554                 uint32_t shift = (pos & 3) * 8;
1555                 uint32_t mask = 0xffffffff << shift;
1556
1557                 if (len < bytes) {
1558                         mask &= 0xffffffff >> (bytes - len) * 8;
1559                         bytes = len;
1560                 }
1561
1562                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1563                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1564                 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1565                 if (!write || mask != 0xffffffff)
1566                         value = RREG32_NO_KIQ(mmMM_DATA);
1567                 if (write) {
1568                         value &= ~mask;
1569                         value |= (*(uint32_t *)buf << shift) & mask;
1570                         WREG32_NO_KIQ(mmMM_DATA, value);
1571                 }
1572                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1573                 if (!write) {
1574                         value = (value & mask) >> shift;
1575                         memcpy(buf, &value, bytes);
1576                 }
1577
1578                 ret += bytes;
1579                 buf = (uint8_t *)buf + bytes;
1580                 pos += bytes;
1581                 len -= bytes;
1582                 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1583                         ++nodes;
1584                         pos = (nodes->start << PAGE_SHIFT);
1585                 }
1586         }
1587
1588         return ret;
1589 }
1590
1591 static struct ttm_bo_driver amdgpu_bo_driver = {
1592         .ttm_tt_create = &amdgpu_ttm_tt_create,
1593         .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1594         .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1595         .invalidate_caches = &amdgpu_invalidate_caches,
1596         .init_mem_type = &amdgpu_init_mem_type,
1597         .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1598         .evict_flags = &amdgpu_evict_flags,
1599         .move = &amdgpu_bo_move,
1600         .verify_access = &amdgpu_verify_access,
1601         .move_notify = &amdgpu_bo_move_notify,
1602         .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1603         .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1604         .io_mem_free = &amdgpu_ttm_io_mem_free,
1605         .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1606         .access_memory = &amdgpu_ttm_access_memory,
1607         .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
1608 };
1609
1610 /*
1611  * Firmware Reservation functions
1612  */
1613 /**
1614  * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1615  *
1616  * @adev: amdgpu_device pointer
1617  *
1618  * free fw reserved vram if it has been reserved.
1619  */
1620 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1621 {
1622         amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
1623                 NULL, &adev->fw_vram_usage.va);
1624 }
1625
1626 /**
1627  * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1628  *
1629  * @adev: amdgpu_device pointer
1630  *
1631  * create bo vram reservation from fw.
1632  */
1633 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1634 {
1635         struct ttm_operation_ctx ctx = { false, false };
1636         struct amdgpu_bo_param bp;
1637         int r = 0;
1638         int i;
1639         u64 vram_size = adev->gmc.visible_vram_size;
1640         u64 offset = adev->fw_vram_usage.start_offset;
1641         u64 size = adev->fw_vram_usage.size;
1642         struct amdgpu_bo *bo;
1643
1644         memset(&bp, 0, sizeof(bp));
1645         bp.size = adev->fw_vram_usage.size;
1646         bp.byte_align = PAGE_SIZE;
1647         bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
1648         bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1649                 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1650         bp.type = ttm_bo_type_kernel;
1651         bp.resv = NULL;
1652         adev->fw_vram_usage.va = NULL;
1653         adev->fw_vram_usage.reserved_bo = NULL;
1654
1655         if (adev->fw_vram_usage.size > 0 &&
1656                 adev->fw_vram_usage.size <= vram_size) {
1657
1658                 r = amdgpu_bo_create(adev, &bp,
1659                                      &adev->fw_vram_usage.reserved_bo);
1660                 if (r)
1661                         goto error_create;
1662
1663                 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
1664                 if (r)
1665                         goto error_reserve;
1666
1667                 /* remove the original mem node and create a new one at the
1668                  * request position
1669                  */
1670                 bo = adev->fw_vram_usage.reserved_bo;
1671                 offset = ALIGN(offset, PAGE_SIZE);
1672                 for (i = 0; i < bo->placement.num_placement; ++i) {
1673                         bo->placements[i].fpfn = offset >> PAGE_SHIFT;
1674                         bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
1675                 }
1676
1677                 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
1678                 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
1679                                      &bo->tbo.mem, &ctx);
1680                 if (r)
1681                         goto error_pin;
1682
1683                 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
1684                         AMDGPU_GEM_DOMAIN_VRAM,
1685                         adev->fw_vram_usage.start_offset,
1686                         (adev->fw_vram_usage.start_offset +
1687                         adev->fw_vram_usage.size));
1688                 if (r)
1689                         goto error_pin;
1690                 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
1691                         &adev->fw_vram_usage.va);
1692                 if (r)
1693                         goto error_kmap;
1694
1695                 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1696         }
1697         return r;
1698
1699 error_kmap:
1700         amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
1701 error_pin:
1702         amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1703 error_reserve:
1704         amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
1705 error_create:
1706         adev->fw_vram_usage.va = NULL;
1707         adev->fw_vram_usage.reserved_bo = NULL;
1708         return r;
1709 }
1710 /**
1711  * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1712  * gtt/vram related fields.
1713  *
1714  * This initializes all of the memory space pools that the TTM layer
1715  * will need such as the GTT space (system memory mapped to the device),
1716  * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1717  * can be mapped per VMID.
1718  */
1719 int amdgpu_ttm_init(struct amdgpu_device *adev)
1720 {
1721         uint64_t gtt_size;
1722         int r;
1723         u64 vis_vram_limit;
1724
1725         mutex_init(&adev->mman.gtt_window_lock);
1726
1727         /* No others user of address space so set it to 0 */
1728         r = ttm_bo_device_init(&adev->mman.bdev,
1729                                &amdgpu_bo_driver,
1730                                adev->ddev->anon_inode->i_mapping,
1731                                adev->need_dma32);
1732         if (r) {
1733                 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1734                 return r;
1735         }
1736         adev->mman.initialized = true;
1737
1738         /* We opt to avoid OOM on system pages allocations */
1739         adev->mman.bdev.no_retry = true;
1740
1741         /* Initialize VRAM pool with all of VRAM divided into pages */
1742         r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1743                                 adev->gmc.real_vram_size >> PAGE_SHIFT);
1744         if (r) {
1745                 DRM_ERROR("Failed initializing VRAM heap.\n");
1746                 return r;
1747         }
1748
1749         /* Reduce size of CPU-visible VRAM if requested */
1750         vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1751         if (amdgpu_vis_vram_limit > 0 &&
1752             vis_vram_limit <= adev->gmc.visible_vram_size)
1753                 adev->gmc.visible_vram_size = vis_vram_limit;
1754
1755         /* Change the size here instead of the init above so only lpfn is affected */
1756         amdgpu_ttm_set_buffer_funcs_status(adev, false);
1757 #ifdef CONFIG_64BIT
1758         adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1759                                                 adev->gmc.visible_vram_size);
1760 #endif
1761
1762         /*
1763          *The reserved vram for firmware must be pinned to the specified
1764          *place on the VRAM, so reserve it early.
1765          */
1766         r = amdgpu_ttm_fw_reserve_vram_init(adev);
1767         if (r) {
1768                 return r;
1769         }
1770
1771         /* allocate memory as required for VGA
1772          * This is used for VGA emulation and pre-OS scanout buffers to
1773          * avoid display artifacts while transitioning between pre-OS
1774          * and driver.  */
1775         r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
1776                                     AMDGPU_GEM_DOMAIN_VRAM,
1777                                     &adev->stolen_vga_memory,
1778                                     NULL, NULL);
1779         if (r)
1780                 return r;
1781         DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1782                  (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1783
1784         /* Compute GTT size, either bsaed on 3/4th the size of RAM size
1785          * or whatever the user passed on module init */
1786         if (amdgpu_gtt_size == -1) {
1787                 struct sysinfo si;
1788
1789                 si_meminfo(&si);
1790                 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1791                                adev->gmc.mc_vram_size),
1792                                ((uint64_t)si.totalram * si.mem_unit * 3/4));
1793         }
1794         else
1795                 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1796
1797         /* Initialize GTT memory pool */
1798         r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1799         if (r) {
1800                 DRM_ERROR("Failed initializing GTT heap.\n");
1801                 return r;
1802         }
1803         DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1804                  (unsigned)(gtt_size / (1024 * 1024)));
1805
1806         /* Initialize various on-chip memory pools */
1807         r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
1808                            adev->gds.gds_size);
1809         if (r) {
1810                 DRM_ERROR("Failed initializing GDS heap.\n");
1811                 return r;
1812         }
1813
1814         r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
1815                            adev->gds.gws_size);
1816         if (r) {
1817                 DRM_ERROR("Failed initializing gws heap.\n");
1818                 return r;
1819         }
1820
1821         r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
1822                            adev->gds.oa_size);
1823         if (r) {
1824                 DRM_ERROR("Failed initializing oa heap.\n");
1825                 return r;
1826         }
1827
1828         /* Register debugfs entries for amdgpu_ttm */
1829         r = amdgpu_ttm_debugfs_init(adev);
1830         if (r) {
1831                 DRM_ERROR("Failed to init debugfs\n");
1832                 return r;
1833         }
1834         return 0;
1835 }
1836
1837 /**
1838  * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm
1839  */
1840 void amdgpu_ttm_late_init(struct amdgpu_device *adev)
1841 {
1842         /* return the VGA stolen memory (if any) back to VRAM */
1843         amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
1844 }
1845
1846 /**
1847  * amdgpu_ttm_fini - De-initialize the TTM memory pools
1848  */
1849 void amdgpu_ttm_fini(struct amdgpu_device *adev)
1850 {
1851         if (!adev->mman.initialized)
1852                 return;
1853
1854         amdgpu_ttm_debugfs_fini(adev);
1855         amdgpu_ttm_fw_reserve_vram_fini(adev);
1856         if (adev->mman.aper_base_kaddr)
1857                 iounmap(adev->mman.aper_base_kaddr);
1858         adev->mman.aper_base_kaddr = NULL;
1859
1860         ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1861         ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1862         ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
1863         ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
1864         ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1865         ttm_bo_device_release(&adev->mman.bdev);
1866         adev->mman.initialized = false;
1867         DRM_INFO("amdgpu: ttm finalized\n");
1868 }
1869
1870 /**
1871  * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
1872  *
1873  * @adev: amdgpu_device pointer
1874  * @enable: true when we can use buffer functions.
1875  *
1876  * Enable/disable use of buffer functions during suspend/resume. This should
1877  * only be called at bootup or when userspace isn't running.
1878  */
1879 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1880 {
1881         struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
1882         uint64_t size;
1883         int r;
1884
1885         if (!adev->mman.initialized || adev->in_gpu_reset ||
1886             adev->mman.buffer_funcs_enabled == enable)
1887                 return;
1888
1889         if (enable) {
1890                 struct amdgpu_ring *ring;
1891                 struct drm_sched_rq *rq;
1892
1893                 ring = adev->mman.buffer_funcs_ring;
1894                 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
1895                 r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
1896                 if (r) {
1897                         DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
1898                                   r);
1899                         return;
1900                 }
1901         } else {
1902                 drm_sched_entity_destroy(&adev->mman.entity);
1903                 dma_fence_put(man->move);
1904                 man->move = NULL;
1905         }
1906
1907         /* this just adjusts TTM size idea, which sets lpfn to the correct value */
1908         if (enable)
1909                 size = adev->gmc.real_vram_size;
1910         else
1911                 size = adev->gmc.visible_vram_size;
1912         man->size = size >> PAGE_SHIFT;
1913         adev->mman.buffer_funcs_enabled = enable;
1914 }
1915
1916 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
1917 {
1918         struct drm_file *file_priv = filp->private_data;
1919         struct amdgpu_device *adev = file_priv->minor->dev->dev_private;
1920
1921         if (adev == NULL)
1922                 return -EINVAL;
1923
1924         return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
1925 }
1926
1927 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
1928                              struct ttm_mem_reg *mem, unsigned num_pages,
1929                              uint64_t offset, unsigned window,
1930                              struct amdgpu_ring *ring,
1931                              uint64_t *addr)
1932 {
1933         struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
1934         struct amdgpu_device *adev = ring->adev;
1935         struct ttm_tt *ttm = bo->ttm;
1936         struct amdgpu_job *job;
1937         unsigned num_dw, num_bytes;
1938         dma_addr_t *dma_address;
1939         struct dma_fence *fence;
1940         uint64_t src_addr, dst_addr;
1941         uint64_t flags;
1942         int r;
1943
1944         BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
1945                AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
1946
1947         *addr = adev->gmc.gart_start;
1948         *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
1949                 AMDGPU_GPU_PAGE_SIZE;
1950
1951         num_dw = adev->mman.buffer_funcs->copy_num_dw;
1952         while (num_dw & 0x7)
1953                 num_dw++;
1954
1955         num_bytes = num_pages * 8;
1956
1957         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
1958         if (r)
1959                 return r;
1960
1961         src_addr = num_dw * 4;
1962         src_addr += job->ibs[0].gpu_addr;
1963
1964         dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
1965         dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
1966         amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
1967                                 dst_addr, num_bytes);
1968
1969         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
1970         WARN_ON(job->ibs[0].length_dw > num_dw);
1971
1972         dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
1973         flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
1974         r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
1975                             &job->ibs[0].ptr[num_dw]);
1976         if (r)
1977                 goto error_free;
1978
1979         r = amdgpu_job_submit(job, &adev->mman.entity,
1980                               AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
1981         if (r)
1982                 goto error_free;
1983
1984         dma_fence_put(fence);
1985
1986         return r;
1987
1988 error_free:
1989         amdgpu_job_free(job);
1990         return r;
1991 }
1992
1993 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
1994                        uint64_t dst_offset, uint32_t byte_count,
1995                        struct reservation_object *resv,
1996                        struct dma_fence **fence, bool direct_submit,
1997                        bool vm_needs_flush)
1998 {
1999         struct amdgpu_device *adev = ring->adev;
2000         struct amdgpu_job *job;
2001
2002         uint32_t max_bytes;
2003         unsigned num_loops, num_dw;
2004         unsigned i;
2005         int r;
2006
2007         if (direct_submit && !ring->sched.ready) {
2008                 DRM_ERROR("Trying to move memory with ring turned off.\n");
2009                 return -EINVAL;
2010         }
2011
2012         max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2013         num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2014         num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;
2015
2016         /* for IB padding */
2017         while (num_dw & 0x7)
2018                 num_dw++;
2019
2020         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2021         if (r)
2022                 return r;
2023
2024         if (vm_needs_flush) {
2025                 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
2026                 job->vm_needs_flush = true;
2027         }
2028         if (resv) {
2029                 r = amdgpu_sync_resv(adev, &job->sync, resv,
2030                                      AMDGPU_FENCE_OWNER_UNDEFINED,
2031                                      false);
2032                 if (r) {
2033                         DRM_ERROR("sync failed (%d).\n", r);
2034                         goto error_free;
2035                 }
2036         }
2037
2038         for (i = 0; i < num_loops; i++) {
2039                 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2040
2041                 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2042                                         dst_offset, cur_size_in_bytes);
2043
2044                 src_offset += cur_size_in_bytes;
2045                 dst_offset += cur_size_in_bytes;
2046                 byte_count -= cur_size_in_bytes;
2047         }
2048
2049         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2050         WARN_ON(job->ibs[0].length_dw > num_dw);
2051         if (direct_submit)
2052                 r = amdgpu_job_submit_direct(job, ring, fence);
2053         else
2054                 r = amdgpu_job_submit(job, &adev->mman.entity,
2055                                       AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2056         if (r)
2057                 goto error_free;
2058
2059         return r;
2060
2061 error_free:
2062         amdgpu_job_free(job);
2063         DRM_ERROR("Error scheduling IBs (%d)\n", r);
2064         return r;
2065 }
2066
2067 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2068                        uint32_t src_data,
2069                        struct reservation_object *resv,
2070                        struct dma_fence **fence)
2071 {
2072         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2073         uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2074         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2075
2076         struct drm_mm_node *mm_node;
2077         unsigned long num_pages;
2078         unsigned int num_loops, num_dw;
2079
2080         struct amdgpu_job *job;
2081         int r;
2082
2083         if (!adev->mman.buffer_funcs_enabled) {
2084                 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2085                 return -EINVAL;
2086         }
2087
2088         if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2089                 r = amdgpu_ttm_alloc_gart(&bo->tbo);
2090                 if (r)
2091                         return r;
2092         }
2093
2094         num_pages = bo->tbo.num_pages;
2095         mm_node = bo->tbo.mem.mm_node;
2096         num_loops = 0;
2097         while (num_pages) {
2098                 uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2099
2100                 num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes);
2101                 num_pages -= mm_node->size;
2102                 ++mm_node;
2103         }
2104         num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2105
2106         /* for IB padding */
2107         num_dw += 64;
2108
2109         r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2110         if (r)
2111                 return r;
2112
2113         if (resv) {
2114                 r = amdgpu_sync_resv(adev, &job->sync, resv,
2115                                      AMDGPU_FENCE_OWNER_UNDEFINED, false);
2116                 if (r) {
2117                         DRM_ERROR("sync failed (%d).\n", r);
2118                         goto error_free;
2119                 }
2120         }
2121
2122         num_pages = bo->tbo.num_pages;
2123         mm_node = bo->tbo.mem.mm_node;
2124
2125         while (num_pages) {
2126                 uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2127                 uint64_t dst_addr;
2128
2129                 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2130                 while (byte_count) {
2131                         uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count,
2132                                                            max_bytes);
2133
2134                         amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
2135                                                 dst_addr, cur_size_in_bytes);
2136
2137                         dst_addr += cur_size_in_bytes;
2138                         byte_count -= cur_size_in_bytes;
2139                 }
2140
2141                 num_pages -= mm_node->size;
2142                 ++mm_node;
2143         }
2144
2145         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2146         WARN_ON(job->ibs[0].length_dw > num_dw);
2147         r = amdgpu_job_submit(job, &adev->mman.entity,
2148                               AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2149         if (r)
2150                 goto error_free;
2151
2152         return 0;
2153
2154 error_free:
2155         amdgpu_job_free(job);
2156         return r;
2157 }
2158
2159 #if defined(CONFIG_DEBUG_FS)
2160
2161 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
2162 {
2163         struct drm_info_node *node = (struct drm_info_node *)m->private;
2164         unsigned ttm_pl = (uintptr_t)node->info_ent->data;
2165         struct drm_device *dev = node->minor->dev;
2166         struct amdgpu_device *adev = dev->dev_private;
2167         struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
2168         struct drm_printer p = drm_seq_file_printer(m);
2169
2170         man->func->debug(man, &p);
2171         return 0;
2172 }
2173
2174 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
2175         {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM},
2176         {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT},
2177         {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS},
2178         {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS},
2179         {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA},
2180         {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
2181 #ifdef CONFIG_SWIOTLB
2182         {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
2183 #endif
2184 };
2185
2186 /**
2187  * amdgpu_ttm_vram_read - Linear read access to VRAM
2188  *
2189  * Accesses VRAM via MMIO for debugging purposes.
2190  */
2191 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2192                                     size_t size, loff_t *pos)
2193 {
2194         struct amdgpu_device *adev = file_inode(f)->i_private;
2195         ssize_t result = 0;
2196         int r;
2197
2198         if (size & 0x3 || *pos & 0x3)
2199                 return -EINVAL;
2200
2201         if (*pos >= adev->gmc.mc_vram_size)
2202                 return -ENXIO;
2203
2204         while (size) {
2205                 unsigned long flags;
2206                 uint32_t value;
2207
2208                 if (*pos >= adev->gmc.mc_vram_size)
2209                         return result;
2210
2211                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2212                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2213                 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2214                 value = RREG32_NO_KIQ(mmMM_DATA);
2215                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2216
2217                 r = put_user(value, (uint32_t *)buf);
2218                 if (r)
2219                         return r;
2220
2221                 result += 4;
2222                 buf += 4;
2223                 *pos += 4;
2224                 size -= 4;
2225         }
2226
2227         return result;
2228 }
2229
2230 /**
2231  * amdgpu_ttm_vram_write - Linear write access to VRAM
2232  *
2233  * Accesses VRAM via MMIO for debugging purposes.
2234  */
2235 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2236                                     size_t size, loff_t *pos)
2237 {
2238         struct amdgpu_device *adev = file_inode(f)->i_private;
2239         ssize_t result = 0;
2240         int r;
2241
2242         if (size & 0x3 || *pos & 0x3)
2243                 return -EINVAL;
2244
2245         if (*pos >= adev->gmc.mc_vram_size)
2246                 return -ENXIO;
2247
2248         while (size) {
2249                 unsigned long flags;
2250                 uint32_t value;
2251
2252                 if (*pos >= adev->gmc.mc_vram_size)
2253                         return result;
2254
2255                 r = get_user(value, (uint32_t *)buf);
2256                 if (r)
2257                         return r;
2258
2259                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2260                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2261                 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2262                 WREG32_NO_KIQ(mmMM_DATA, value);
2263                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2264
2265                 result += 4;
2266                 buf += 4;
2267                 *pos += 4;
2268                 size -= 4;
2269         }
2270
2271         return result;
2272 }
2273
2274 static const struct file_operations amdgpu_ttm_vram_fops = {
2275         .owner = THIS_MODULE,
2276         .read = amdgpu_ttm_vram_read,
2277         .write = amdgpu_ttm_vram_write,
2278         .llseek = default_llseek,
2279 };
2280
2281 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2282
2283 /**
2284  * amdgpu_ttm_gtt_read - Linear read access to GTT memory
2285  */
2286 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
2287                                    size_t size, loff_t *pos)
2288 {
2289         struct amdgpu_device *adev = file_inode(f)->i_private;
2290         ssize_t result = 0;
2291         int r;
2292
2293         while (size) {
2294                 loff_t p = *pos / PAGE_SIZE;
2295                 unsigned off = *pos & ~PAGE_MASK;
2296                 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
2297                 struct page *page;
2298                 void *ptr;
2299
2300                 if (p >= adev->gart.num_cpu_pages)
2301                         return result;
2302
2303                 page = adev->gart.pages[p];
2304                 if (page) {
2305                         ptr = kmap(page);
2306                         ptr += off;
2307
2308                         r = copy_to_user(buf, ptr, cur_size);
2309                         kunmap(adev->gart.pages[p]);
2310                 } else
2311                         r = clear_user(buf, cur_size);
2312
2313                 if (r)
2314                         return -EFAULT;
2315
2316                 result += cur_size;
2317                 buf += cur_size;
2318                 *pos += cur_size;
2319                 size -= cur_size;
2320         }
2321
2322         return result;
2323 }
2324
2325 static const struct file_operations amdgpu_ttm_gtt_fops = {
2326         .owner = THIS_MODULE,
2327         .read = amdgpu_ttm_gtt_read,
2328         .llseek = default_llseek
2329 };
2330
2331 #endif
2332
2333 /**
2334  * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2335  *
2336  * This function is used to read memory that has been mapped to the
2337  * GPU and the known addresses are not physical addresses but instead
2338  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2339  */
2340 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2341                                  size_t size, loff_t *pos)
2342 {
2343         struct amdgpu_device *adev = file_inode(f)->i_private;
2344         struct iommu_domain *dom;
2345         ssize_t result = 0;
2346         int r;
2347
2348         /* retrieve the IOMMU domain if any for this device */
2349         dom = iommu_get_domain_for_dev(adev->dev);
2350
2351         while (size) {
2352                 phys_addr_t addr = *pos & PAGE_MASK;
2353                 loff_t off = *pos & ~PAGE_MASK;
2354                 size_t bytes = PAGE_SIZE - off;
2355                 unsigned long pfn;
2356                 struct page *p;
2357                 void *ptr;
2358
2359                 bytes = bytes < size ? bytes : size;
2360
2361                 /* Translate the bus address to a physical address.  If
2362                  * the domain is NULL it means there is no IOMMU active
2363                  * and the address translation is the identity
2364                  */
2365                 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2366
2367                 pfn = addr >> PAGE_SHIFT;
2368                 if (!pfn_valid(pfn))
2369                         return -EPERM;
2370
2371                 p = pfn_to_page(pfn);
2372                 if (p->mapping != adev->mman.bdev.dev_mapping)
2373                         return -EPERM;
2374
2375                 ptr = kmap(p);
2376                 r = copy_to_user(buf, ptr + off, bytes);
2377                 kunmap(p);
2378                 if (r)
2379                         return -EFAULT;
2380
2381                 size -= bytes;
2382                 *pos += bytes;
2383                 result += bytes;
2384         }
2385
2386         return result;
2387 }
2388
2389 /**
2390  * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2391  *
2392  * This function is used to write memory that has been mapped to the
2393  * GPU and the known addresses are not physical addresses but instead
2394  * bus addresses (e.g., what you'd put in an IB or ring buffer).
2395  */
2396 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2397                                  size_t size, loff_t *pos)
2398 {
2399         struct amdgpu_device *adev = file_inode(f)->i_private;
2400         struct iommu_domain *dom;
2401         ssize_t result = 0;
2402         int r;
2403
2404         dom = iommu_get_domain_for_dev(adev->dev);
2405
2406         while (size) {
2407                 phys_addr_t addr = *pos & PAGE_MASK;
2408                 loff_t off = *pos & ~PAGE_MASK;
2409                 size_t bytes = PAGE_SIZE - off;
2410                 unsigned long pfn;
2411                 struct page *p;
2412                 void *ptr;
2413
2414                 bytes = bytes < size ? bytes : size;
2415
2416                 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2417
2418                 pfn = addr >> PAGE_SHIFT;
2419                 if (!pfn_valid(pfn))
2420                         return -EPERM;
2421
2422                 p = pfn_to_page(pfn);
2423                 if (p->mapping != adev->mman.bdev.dev_mapping)
2424                         return -EPERM;
2425
2426                 ptr = kmap(p);
2427                 r = copy_from_user(ptr + off, buf, bytes);
2428                 kunmap(p);
2429                 if (r)
2430                         return -EFAULT;
2431
2432                 size -= bytes;
2433                 *pos += bytes;
2434                 result += bytes;
2435         }
2436
2437         return result;
2438 }
2439
2440 static const struct file_operations amdgpu_ttm_iomem_fops = {
2441         .owner = THIS_MODULE,
2442         .read = amdgpu_iomem_read,
2443         .write = amdgpu_iomem_write,
2444         .llseek = default_llseek
2445 };
2446
2447 static const struct {
2448         char *name;
2449         const struct file_operations *fops;
2450         int domain;
2451 } ttm_debugfs_entries[] = {
2452         { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
2453 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2454         { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
2455 #endif
2456         { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2457 };
2458
2459 #endif
2460
2461 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2462 {
2463 #if defined(CONFIG_DEBUG_FS)
2464         unsigned count;
2465
2466         struct drm_minor *minor = adev->ddev->primary;
2467         struct dentry *ent, *root = minor->debugfs_root;
2468
2469         for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
2470                 ent = debugfs_create_file(
2471                                 ttm_debugfs_entries[count].name,
2472                                 S_IFREG | S_IRUGO, root,
2473                                 adev,
2474                                 ttm_debugfs_entries[count].fops);
2475                 if (IS_ERR(ent))
2476                         return PTR_ERR(ent);
2477                 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2478                         i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2479                 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2480                         i_size_write(ent->d_inode, adev->gmc.gart_size);
2481                 adev->mman.debugfs_entries[count] = ent;
2482         }
2483
2484         count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
2485
2486 #ifdef CONFIG_SWIOTLB
2487         if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
2488                 --count;
2489 #endif
2490
2491         return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
2492 #else
2493         return 0;
2494 #endif
2495 }
2496
2497 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
2498 {
2499 #if defined(CONFIG_DEBUG_FS)
2500         unsigned i;
2501
2502         for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
2503                 debugfs_remove(adev->mman.debugfs_entries[i]);
2504 #endif
2505 }