drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/dma-fence-array.h>
  29 #include <linux/interval_tree_generic.h>
  30 #include <linux/idr.h>
  31 #include <drm/drmP.h>
  32 #include <drm/amdgpu_drm.h>
  33 #include "amdgpu.h"
  34 #include "amdgpu_trace.h"
  35
  36 /*
  37  * GPUVM
  38  * GPUVM is similar to the legacy gart on older asics, however
  39  * rather than there being a single global gart table
  40  * for the entire GPU, there are multiple VM page tables active
  41  * at any given time.  The VM page tables can contain a mix
  42  * vram pages and system memory pages and system memory pages
  43  * can be mapped as snooped (cached system pages) or unsnooped
  44  * (uncached system pages).
  45  * Each VM has an ID associated with it and there is a page table
  46  * associated with each VMID.  When execting a command buffer,
  47  * the kernel tells the the ring what VMID to use for that command
  48  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  49  * The userspace drivers maintain their own address space and the kernel
  50  * sets up their pages tables accordingly when they submit their
  51  * command buffers and a VMID is assigned.
  52  * Cayman/Trinity support up to 8 active VMs at any given time;
  53  * SI supports 16.
  54  */
  55
  56 #define START(node) ((node)->start)
  57 #define LAST(node) ((node)->last)
  58
  59 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  60                      START, LAST, static, amdgpu_vm_it)
  61
  62 #undef START
  63 #undef LAST
  64
  65 /* Local structure. Encapsulate some VM table update parameters to reduce
  66  * the number of function parameters
  67  */
  68 struct amdgpu_pte_update_params {
  69         /* amdgpu device we do this update for */
  70         struct amdgpu_device *adev;
  71         /* optional amdgpu_vm we do this update for */
  72         struct amdgpu_vm *vm;
  73         /* address where to copy page table entries from */
  74         uint64_t src;
  75         /* indirect buffer to fill with commands */
  76         struct amdgpu_ib *ib;
  77         /* Function which actually does the update */
  78         void (*func)(struct amdgpu_pte_update_params *params,
  79                      struct amdgpu_bo *bo, uint64_t pe,
  80                      uint64_t addr, unsigned count, uint32_t incr,
  81                      uint64_t flags);
  82         /* The next two are used during VM update by CPU
  83          *  DMA addresses to use for mapping
  84          *  Kernel pointer of PD/PT BO that needs to be updated
  85          */
  86         dma_addr_t *pages_addr;
  87         void *kptr;
  88 };
  89
  90 /* Helper to disable partial resident texture feature from a fence callback */
  91 struct amdgpu_prt_cb {
  92         struct amdgpu_device *adev;
  93         struct dma_fence_cb cb;
  94 };
  95
  96 /**
  97  * amdgpu_vm_level_shift - return the addr shift for each level
  98  *
  99  * @adev: amdgpu_device pointer
 100  *
 101  * Returns the number of bits the pfn needs to be right shifted for a level.
 102  */
 103 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
 104                                       unsigned level)
 105 {
 106         unsigned shift = 0xff;
 107
 108         switch (level) {
 109         case AMDGPU_VM_PDB2:
 110         case AMDGPU_VM_PDB1:
 111         case AMDGPU_VM_PDB0:
 112                 shift = 9 * (AMDGPU_VM_PDB0 - level) +
 113                         adev->vm_manager.block_size;
 114                 break;
 115         case AMDGPU_VM_PTB:
 116                 shift = 0;
 117                 break;
 118         default:
 119                 dev_err(adev->dev, "the level%d isn't supported.\n", level);
 120         }
 121
 122         return shift;
 123 }
 124
 125 /**
 126  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
 127  *
 128  * @adev: amdgpu_device pointer
 129  *
 130  * Calculate the number of entries in a page directory or page table.
 131  */
 132 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
 133                                       unsigned level)
 134 {
 135         unsigned shift = amdgpu_vm_level_shift(adev,
 136                                                adev->vm_manager.root_level);
 137
 138         if (level == adev->vm_manager.root_level)
 139                 /* For the root directory */
 140                 return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
 141         else if (level != AMDGPU_VM_PTB)
 142                 /* Everything in between */
 143                 return 512;
 144         else
 145                 /* For the page tables on the leaves */
 146                 return AMDGPU_VM_PTE_COUNT(adev);
 147 }
 148
 149 /**
 150  * amdgpu_vm_bo_size - returns the size of the BOs in bytes
 151  *
 152  * @adev: amdgpu_device pointer
 153  *
 154  * Calculate the size of the BO for a page directory or page table in bytes.
 155  */
 156 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
 157 {
 158         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
 159 }
 160
 161 /**
 162  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 163  *
 164  * @vm: vm providing the BOs
 165  * @validated: head of validation list
 166  * @entry: entry to add
 167  *
 168  * Add the page directory to the list of BOs to
 169  * validate for command submission.
 170  */
 171 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 172                          struct list_head *validated,
 173                          struct amdgpu_bo_list_entry *entry)
 174 {
 175         entry->robj = vm->root.base.bo;
 176         entry->priority = 0;
 177         entry->tv.bo = &entry->robj->tbo;
 178         entry->tv.shared = true;
 179         entry->user_pages = NULL;
 180         list_add(&entry->tv.head, validated);
 181 }
 182
 183 /**
 184  * amdgpu_vm_validate_pt_bos - validate the page table BOs
 185  *
 186  * @adev: amdgpu device pointer
 187  * @vm: vm providing the BOs
 188  * @validate: callback to do the validation
 189  * @param: parameter for the validation callback
 190  *
 191  * Validate the page table BOs on command submission if neccessary.
 192  */
 193 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 194                               int (*validate)(void *p, struct amdgpu_bo *bo),
 195                               void *param)
 196 {
 197         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 198         int r;
 199
 200         spin_lock(&vm->status_lock);
 201         while (!list_empty(&vm->evicted)) {
 202                 struct amdgpu_vm_bo_base *bo_base;
 203                 struct amdgpu_bo *bo;
 204
 205                 bo_base = list_first_entry(&vm->evicted,
 206                                            struct amdgpu_vm_bo_base,
 207                                            vm_status);
 208                 spin_unlock(&vm->status_lock);
 209
 210                 bo = bo_base->bo;
 211                 BUG_ON(!bo);
 212                 if (bo->parent) {
 213                         r = validate(param, bo);
 214                         if (r)
 215                                 return r;
 216
 217                         spin_lock(&glob->lru_lock);
 218                         ttm_bo_move_to_lru_tail(&bo->tbo);
 219                         if (bo->shadow)
 220                                 ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
 221                         spin_unlock(&glob->lru_lock);
 222                 }
 223
 224                 if (bo->tbo.type == ttm_bo_type_kernel &&
 225                     vm->use_cpu_for_update) {
 226                         r = amdgpu_bo_kmap(bo, NULL);
 227                         if (r)
 228                                 return r;
 229                 }
 230
 231                 spin_lock(&vm->status_lock);
 232                 if (bo->tbo.type != ttm_bo_type_kernel)
 233                         list_move(&bo_base->vm_status, &vm->moved);
 234                 else
 235                         list_move(&bo_base->vm_status, &vm->relocated);
 236         }
 237         spin_unlock(&vm->status_lock);
 238
 239         return 0;
 240 }
 241
 242 /**
 243  * amdgpu_vm_ready - check VM is ready for updates
 244  *
 245  * @vm: VM to check
 246  *
 247  * Check if all VM PDs/PTs are ready for updates
 248  */
 249 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 250 {
 251         bool ready;
 252
 253         spin_lock(&vm->status_lock);
 254         ready = list_empty(&vm->evicted);
 255         spin_unlock(&vm->status_lock);
 256
 257         return ready;
 258 }
 259
 260 /**
 261  * amdgpu_vm_alloc_levels - allocate the PD/PT levels
 262  *
 263  * @adev: amdgpu_device pointer
 264  * @vm: requested vm
 265  * @saddr: start of the address range
 266  * @eaddr: end of the address range
 267  *
 268  * Make sure the page directories and page tables are allocated
 269  */
 270 static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 271                                   struct amdgpu_vm *vm,
 272                                   struct amdgpu_vm_pt *parent,
 273                                   uint64_t saddr, uint64_t eaddr,
 274                                   unsigned level)
 275 {
 276         unsigned shift = amdgpu_vm_level_shift(adev, level);
 277         unsigned pt_idx, from, to;
 278         int r;
 279         u64 flags;
 280         uint64_t init_value = 0;
 281
 282         if (!parent->entries) {
 283                 unsigned num_entries = amdgpu_vm_num_entries(adev, level);
 284
 285                 parent->entries = kvmalloc_array(num_entries,
 286                                                    sizeof(struct amdgpu_vm_pt),
 287                                                    GFP_KERNEL | __GFP_ZERO);
 288                 if (!parent->entries)
 289                         return -ENOMEM;
 290                 memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
 291         }
 292
 293         from = saddr >> shift;
 294         to = eaddr >> shift;
 295         if (from >= amdgpu_vm_num_entries(adev, level) ||
 296             to >= amdgpu_vm_num_entries(adev, level))
 297                 return -EINVAL;
 298
 299         ++level;
 300         saddr = saddr & ((1 << shift) - 1);
 301         eaddr = eaddr & ((1 << shift) - 1);
 302
 303         flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
 304                         AMDGPU_GEM_CREATE_VRAM_CLEARED;
 305         if (vm->use_cpu_for_update)
 306                 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 307         else
 308                 flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
 309                                 AMDGPU_GEM_CREATE_SHADOW);
 310
 311         if (vm->pte_support_ats) {
 312                 init_value = AMDGPU_PTE_DEFAULT_ATC;
 313                 if (level != AMDGPU_VM_PTB)
 314                         init_value |= AMDGPU_PDE_PTE;
 315
 316         }
 317
 318         /* walk over the address space and allocate the page tables */
 319         for (pt_idx = from; pt_idx <= to; ++pt_idx) {
 320                 struct reservation_object *resv = vm->root.base.bo->tbo.resv;
 321                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 322                 struct amdgpu_bo *pt;
 323
 324                 if (!entry->base.bo) {
 325                         r = amdgpu_bo_create(adev,
 326                                              amdgpu_vm_bo_size(adev, level),
 327                                              AMDGPU_GPU_PAGE_SIZE, true,
 328                                              AMDGPU_GEM_DOMAIN_VRAM,
 329                                              flags,
 330                                              NULL, resv, init_value, &pt);
 331                         if (r)
 332                                 return r;
 333
 334                         if (vm->use_cpu_for_update) {
 335                                 r = amdgpu_bo_kmap(pt, NULL);
 336                                 if (r) {
 337                                         amdgpu_bo_unref(&pt);
 338                                         return r;
 339                                 }
 340                         }
 341
 342                         /* Keep a reference to the root directory to avoid
 343                         * freeing them up in the wrong order.
 344                         */
 345                         pt->parent = amdgpu_bo_ref(parent->base.bo);
 346
 347                         entry->base.vm = vm;
 348                         entry->base.bo = pt;
 349                         list_add_tail(&entry->base.bo_list, &pt->va);
 350                         spin_lock(&vm->status_lock);
 351                         list_add(&entry->base.vm_status, &vm->relocated);
 352                         spin_unlock(&vm->status_lock);
 353                 }
 354
 355                 if (level < AMDGPU_VM_PTB) {
 356                         uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
 357                         uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
 358                                 ((1 << shift) - 1);
 359                         r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
 360                                                    sub_eaddr, level);
 361                         if (r)
 362                                 return r;
 363                 }
 364         }
 365
 366         return 0;
 367 }
 368
 369 /**
 370  * amdgpu_vm_alloc_pts - Allocate page tables.
 371  *
 372  * @adev: amdgpu_device pointer
 373  * @vm: VM to allocate page tables for
 374  * @saddr: Start address which needs to be allocated
 375  * @size: Size from start address we need.
 376  *
 377  * Make sure the page tables are allocated.
 378  */
 379 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 380                         struct amdgpu_vm *vm,
 381                         uint64_t saddr, uint64_t size)
 382 {
 383         uint64_t last_pfn;
 384         uint64_t eaddr;
 385
 386         /* validate the parameters */
 387         if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
 388                 return -EINVAL;
 389
 390         eaddr = saddr + size - 1;
 391         last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
 392         if (last_pfn >= adev->vm_manager.max_pfn) {
 393                 dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
 394                         last_pfn, adev->vm_manager.max_pfn);
 395                 return -EINVAL;
 396         }
 397
 398         saddr /= AMDGPU_GPU_PAGE_SIZE;
 399         eaddr /= AMDGPU_GPU_PAGE_SIZE;
 400
 401         return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr,
 402                                       adev->vm_manager.root_level);
 403 }
 404
 405 /**
 406  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 407  *
 408  * @adev: amdgpu_device pointer
 409  */
 410 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 411 {
 412         const struct amdgpu_ip_block *ip_block;
 413         bool has_compute_vm_bug;
 414         struct amdgpu_ring *ring;
 415         int i;
 416
 417         has_compute_vm_bug = false;
 418
 419         ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 420         if (ip_block) {
 421                 /* Compute has a VM bug for GFX version < 7.
 422                    Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
 423                 if (ip_block->version->major <= 7)
 424                         has_compute_vm_bug = true;
 425                 else if (ip_block->version->major == 8)
 426                         if (adev->gfx.mec_fw_version < 673)
 427                                 has_compute_vm_bug = true;
 428         }
 429
 430         for (i = 0; i < adev->num_rings; i++) {
 431                 ring = adev->rings[i];
 432                 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
 433                         /* only compute rings */
 434                         ring->has_compute_vm_bug = has_compute_vm_bug;
 435                 else
 436                         ring->has_compute_vm_bug = false;
 437         }
 438 }
 439
 440 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 441                                   struct amdgpu_job *job)
 442 {
 443         struct amdgpu_device *adev = ring->adev;
 444         unsigned vmhub = ring->funcs->vmhub;
 445         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 446         struct amdgpu_vmid *id;
 447         bool gds_switch_needed;
 448         bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
 449
 450         if (job->vmid == 0)
 451                 return false;
 452         id = &id_mgr->ids[job->vmid];
 453         gds_switch_needed = ring->funcs->emit_gds_switch && (
 454                 id->gds_base != job->gds_base ||
 455                 id->gds_size != job->gds_size ||
 456                 id->gws_base != job->gws_base ||
 457                 id->gws_size != job->gws_size ||
 458                 id->oa_base != job->oa_base ||
 459                 id->oa_size != job->oa_size);
 460
 461         if (amdgpu_vmid_had_gpu_reset(adev, id))
 462                 return true;
 463
 464         return vm_flush_needed || gds_switch_needed;
 465 }
 466
 467 static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
 468 {
 469         return (adev->gmc.real_vram_size == adev->gmc.visible_vram_size);
 470 }
 471
 472 /**
 473  * amdgpu_vm_flush - hardware flush the vm
 474  *
 475  * @ring: ring to use for flush
 476  * @vmid: vmid number to use
 477  * @pd_addr: address of the page directory
 478  *
 479  * Emit a VM flush when it is necessary.
 480  */
 481 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
 482 {
 483         struct amdgpu_device *adev = ring->adev;
 484         unsigned vmhub = ring->funcs->vmhub;
 485         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 486         struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
 487         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 488                 id->gds_base != job->gds_base ||
 489                 id->gds_size != job->gds_size ||
 490                 id->gws_base != job->gws_base ||
 491                 id->gws_size != job->gws_size ||
 492                 id->oa_base != job->oa_base ||
 493                 id->oa_size != job->oa_size);
 494         bool vm_flush_needed = job->vm_needs_flush;
 495         unsigned patch_offset = 0;
 496         int r;
 497
 498         if (amdgpu_vmid_had_gpu_reset(adev, id)) {
 499                 gds_switch_needed = true;
 500                 vm_flush_needed = true;
 501         }
 502
 503         if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
 504                 return 0;
 505
 506         if (ring->funcs->init_cond_exec)
 507                 patch_offset = amdgpu_ring_init_cond_exec(ring);
 508
 509         if (need_pipe_sync)
 510                 amdgpu_ring_emit_pipeline_sync(ring);
 511
 512         if (ring->funcs->emit_vm_flush && vm_flush_needed) {
 513                 struct dma_fence *fence;
 514
 515                 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
 516                 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->pasid,
 517                                           job->vm_pd_addr);
 518
 519                 r = amdgpu_fence_emit(ring, &fence);
 520                 if (r)
 521                         return r;
 522
 523                 mutex_lock(&id_mgr->lock);
 524                 dma_fence_put(id->last_flush);
 525                 id->last_flush = fence;
 526                 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
 527                 mutex_unlock(&id_mgr->lock);
 528         }
 529
 530         if (ring->funcs->emit_gds_switch && gds_switch_needed) {
 531                 id->gds_base = job->gds_base;
 532                 id->gds_size = job->gds_size;
 533                 id->gws_base = job->gws_base;
 534                 id->gws_size = job->gws_size;
 535                 id->oa_base = job->oa_base;
 536                 id->oa_size = job->oa_size;
 537                 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 538                                             job->gds_size, job->gws_base,
 539                                             job->gws_size, job->oa_base,
 540                                             job->oa_size);
 541         }
 542
 543         if (ring->funcs->patch_cond_exec)
 544                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
 545
 546         /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
 547         if (ring->funcs->emit_switch_buffer) {
 548                 amdgpu_ring_emit_switch_buffer(ring);
 549                 amdgpu_ring_emit_switch_buffer(ring);
 550         }
 551         return 0;
 552 }
 553
 554 /**
 555  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 556  *
 557  * @vm: requested vm
 558  * @bo: requested buffer object
 559  *
 560  * Find @bo inside the requested vm.
 561  * Search inside the @bos vm list for the requested vm
 562  * Returns the found bo_va or NULL if none is found
 563  *
 564  * Object has to be reserved!
 565  */
 566 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 567                                        struct amdgpu_bo *bo)
 568 {
 569         struct amdgpu_bo_va *bo_va;
 570
 571         list_for_each_entry(bo_va, &bo->va, base.bo_list) {
 572                 if (bo_va->base.vm == vm) {
 573                         return bo_va;
 574                 }
 575         }
 576         return NULL;
 577 }
 578
 579 /**
 580  * amdgpu_vm_do_set_ptes - helper to call the right asic function
 581  *
 582  * @params: see amdgpu_pte_update_params definition
 583  * @bo: PD/PT to update
 584  * @pe: addr of the page entry
 585  * @addr: dst addr to write into pe
 586  * @count: number of page entries to update
 587  * @incr: increase next addr by incr bytes
 588  * @flags: hw access flags
 589  *
 590  * Traces the parameters and calls the right asic functions
 591  * to setup the page table using the DMA.
 592  */
 593 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
 594                                   struct amdgpu_bo *bo,
 595                                   uint64_t pe, uint64_t addr,
 596                                   unsigned count, uint32_t incr,
 597                                   uint64_t flags)
 598 {
 599         pe += amdgpu_bo_gpu_offset(bo);
 600         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 601
 602         if (count < 3) {
 603                 amdgpu_vm_write_pte(params->adev, params->ib, pe,
 604                                     addr | flags, count, incr);
 605
 606         } else {
 607                 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
 608                                       count, incr, flags);
 609         }
 610 }
 611
 612 /**
 613  * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 614  *
 615  * @params: see amdgpu_pte_update_params definition
 616  * @bo: PD/PT to update
 617  * @pe: addr of the page entry
 618  * @addr: dst addr to write into pe
 619  * @count: number of page entries to update
 620  * @incr: increase next addr by incr bytes
 621  * @flags: hw access flags
 622  *
 623  * Traces the parameters and calls the DMA function to copy the PTEs.
 624  */
 625 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 626                                    struct amdgpu_bo *bo,
 627                                    uint64_t pe, uint64_t addr,
 628                                    unsigned count, uint32_t incr,
 629                                    uint64_t flags)
 630 {
 631         uint64_t src = (params->src + (addr >> 12) * 8);
 632
 633         pe += amdgpu_bo_gpu_offset(bo);
 634         trace_amdgpu_vm_copy_ptes(pe, src, count);
 635
 636         amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
 637 }
 638
 639 /**
 640  * amdgpu_vm_map_gart - Resolve gart mapping of addr
 641  *
 642  * @pages_addr: optional DMA address to use for lookup
 643  * @addr: the unmapped addr
 644  *
 645  * Look up the physical address of the page that the pte resolves
 646  * to and return the pointer for the page table entry.
 647  */
 648 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 649 {
 650         uint64_t result;
 651
 652         /* page table offset */
 653         result = pages_addr[addr >> PAGE_SHIFT];
 654
 655         /* in case cpu page size != gpu page size*/
 656         result |= addr & (~PAGE_MASK);
 657
 658         result &= 0xFFFFFFFFFFFFF000ULL;
 659
 660         return result;
 661 }
 662
 663 /**
 664  * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
 665  *
 666  * @params: see amdgpu_pte_update_params definition
 667  * @bo: PD/PT to update
 668  * @pe: kmap addr of the page entry
 669  * @addr: dst addr to write into pe
 670  * @count: number of page entries to update
 671  * @incr: increase next addr by incr bytes
 672  * @flags: hw access flags
 673  *
 674  * Write count number of PT/PD entries directly.
 675  */
 676 static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
 677                                    struct amdgpu_bo *bo,
 678                                    uint64_t pe, uint64_t addr,
 679                                    unsigned count, uint32_t incr,
 680                                    uint64_t flags)
 681 {
 682         unsigned int i;
 683         uint64_t value;
 684
 685         pe += (unsigned long)amdgpu_bo_kptr(bo);
 686
 687         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
 688
 689         for (i = 0; i < count; i++) {
 690                 value = params->pages_addr ?
 691                         amdgpu_vm_map_gart(params->pages_addr, addr) :
 692                         addr;
 693                 amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
 694                                        i, value, flags);
 695                 addr += incr;
 696         }
 697 }
 698
 699 static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 700                              void *owner)
 701 {
 702         struct amdgpu_sync sync;
 703         int r;
 704
 705         amdgpu_sync_create(&sync);
 706         amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
 707         r = amdgpu_sync_wait(&sync, true);
 708         amdgpu_sync_free(&sync);
 709
 710         return r;
 711 }
 712
 713 /*
 714  * amdgpu_vm_update_pde - update a single level in the hierarchy
 715  *
 716  * @param: parameters for the update
 717  * @vm: requested vm
 718  * @parent: parent directory
 719  * @entry: entry to update
 720  *
 721  * Makes sure the requested entry in parent is up to date.
 722  */
 723 static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
 724                                  struct amdgpu_vm *vm,
 725                                  struct amdgpu_vm_pt *parent,
 726                                  struct amdgpu_vm_pt *entry)
 727 {
 728         struct amdgpu_bo *bo = parent->base.bo, *pbo;
 729         uint64_t pde, pt, flags;
 730         unsigned level;
 731
 732         /* Don't update huge pages here */
 733         if (entry->huge)
 734                 return;
 735
 736         for (level = 0, pbo = bo->parent; pbo; ++level)
 737                 pbo = pbo->parent;
 738
 739         level += params->adev->vm_manager.root_level;
 740         pt = amdgpu_bo_gpu_offset(entry->base.bo);
 741         flags = AMDGPU_PTE_VALID;
 742         amdgpu_gmc_get_vm_pde(params->adev, level, &pt, &flags);
 743         pde = (entry - parent->entries) * 8;
 744         if (bo->shadow)
 745                 params->func(params, bo->shadow, pde, pt, 1, 0, flags);
 746         params->func(params, bo, pde, pt, 1, 0, flags);
 747 }
 748
 749 /*
 750  * amdgpu_vm_invalidate_level - mark all PD levels as invalid
 751  *
 752  * @parent: parent PD
 753  *
 754  * Mark all PD level as invalid after an error.
 755  */
 756 static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
 757                                        struct amdgpu_vm *vm,
 758                                        struct amdgpu_vm_pt *parent,
 759                                        unsigned level)
 760 {
 761         unsigned pt_idx, num_entries;
 762
 763         /*
 764          * Recurse into the subdirectories. This recursion is harmless because
 765          * we only have a maximum of 5 layers.
 766          */
 767         num_entries = amdgpu_vm_num_entries(adev, level);
 768         for (pt_idx = 0; pt_idx < num_entries; ++pt_idx) {
 769                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 770
 771                 if (!entry->base.bo)
 772                         continue;
 773
 774                 spin_lock(&vm->status_lock);
 775                 if (list_empty(&entry->base.vm_status))
 776                         list_add(&entry->base.vm_status, &vm->relocated);
 777                 spin_unlock(&vm->status_lock);
 778                 amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
 779         }
 780 }
 781
 782 /*
 783  * amdgpu_vm_update_directories - make sure that all directories are valid
 784  *
 785  * @adev: amdgpu_device pointer
 786  * @vm: requested vm
 787  *
 788  * Makes sure all directories are up to date.
 789  * Returns 0 for success, error for failure.
 790  */
 791 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 792                                  struct amdgpu_vm *vm)
 793 {
 794         struct amdgpu_pte_update_params params;
 795         struct amdgpu_job *job;
 796         unsigned ndw = 0;
 797         int r = 0;
 798
 799         if (list_empty(&vm->relocated))
 800                 return 0;
 801
 802 restart:
 803         memset(&params, 0, sizeof(params));
 804         params.adev = adev;
 805
 806         if (vm->use_cpu_for_update) {
 807                 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
 808                 if (unlikely(r))
 809                         return r;
 810
 811                 params.func = amdgpu_vm_cpu_set_ptes;
 812         } else {
 813                 ndw = 512 * 8;
 814                 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 815                 if (r)
 816                         return r;
 817
 818                 params.ib = &job->ibs[0];
 819                 params.func = amdgpu_vm_do_set_ptes;
 820         }
 821
 822         spin_lock(&vm->status_lock);
 823         while (!list_empty(&vm->relocated)) {
 824                 struct amdgpu_vm_bo_base *bo_base, *parent;
 825                 struct amdgpu_vm_pt *pt, *entry;
 826                 struct amdgpu_bo *bo;
 827
 828                 bo_base = list_first_entry(&vm->relocated,
 829                                            struct amdgpu_vm_bo_base,
 830                                            vm_status);
 831                 list_del_init(&bo_base->vm_status);
 832                 spin_unlock(&vm->status_lock);
 833
 834                 bo = bo_base->bo->parent;
 835                 if (!bo) {
 836                         spin_lock(&vm->status_lock);
 837                         continue;
 838                 }
 839
 840                 parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base,
 841                                           bo_list);
 842                 pt = container_of(parent, struct amdgpu_vm_pt, base);
 843                 entry = container_of(bo_base, struct amdgpu_vm_pt, base);
 844
 845                 amdgpu_vm_update_pde(&params, vm, pt, entry);
 846
 847                 spin_lock(&vm->status_lock);
 848                 if (!vm->use_cpu_for_update &&
 849                     (ndw - params.ib->length_dw) < 32)
 850                         break;
 851         }
 852         spin_unlock(&vm->status_lock);
 853
 854         if (vm->use_cpu_for_update) {
 855                 /* Flush HDP */
 856                 mb();
 857                 amdgpu_asic_flush_hdp(adev, NULL);
 858         } else if (params.ib->length_dw == 0) {
 859                 amdgpu_job_free(job);
 860         } else {
 861                 struct amdgpu_bo *root = vm->root.base.bo;
 862                 struct amdgpu_ring *ring;
 863                 struct dma_fence *fence;
 864
 865                 ring = container_of(vm->entity.sched, struct amdgpu_ring,
 866                                     sched);
 867
 868                 amdgpu_ring_pad_ib(ring, params.ib);
 869                 amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
 870                                  AMDGPU_FENCE_OWNER_VM, false);
 871                 if (root->shadow)
 872                         amdgpu_sync_resv(adev, &job->sync,
 873                                          root->shadow->tbo.resv,
 874                                          AMDGPU_FENCE_OWNER_VM, false);
 875
 876                 WARN_ON(params.ib->length_dw > ndw);
 877                 r = amdgpu_job_submit(job, ring, &vm->entity,
 878                                       AMDGPU_FENCE_OWNER_VM, &fence);
 879                 if (r)
 880                         goto error;
 881
 882                 amdgpu_bo_fence(root, fence, true);
 883                 dma_fence_put(vm->last_update);
 884                 vm->last_update = fence;
 885         }
 886
 887         if (!list_empty(&vm->relocated))
 888                 goto restart;
 889
 890         return 0;
 891
 892 error:
 893         amdgpu_vm_invalidate_level(adev, vm, &vm->root,
 894                                    adev->vm_manager.root_level);
 895         amdgpu_job_free(job);
 896         return r;
 897 }
 898
 899 /**
 900  * amdgpu_vm_find_entry - find the entry for an address
 901  *
 902  * @p: see amdgpu_pte_update_params definition
 903  * @addr: virtual address in question
 904  * @entry: resulting entry or NULL
 905  * @parent: parent entry
 906  *
 907  * Find the vm_pt entry and it's parent for the given address.
 908  */
 909 void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
 910                          struct amdgpu_vm_pt **entry,
 911                          struct amdgpu_vm_pt **parent)
 912 {
 913         unsigned level = p->adev->vm_manager.root_level;
 914
 915         *parent = NULL;
 916         *entry = &p->vm->root;
 917         while ((*entry)->entries) {
 918                 unsigned shift = amdgpu_vm_level_shift(p->adev, level++);
 919
 920                 *parent = *entry;
 921                 *entry = &(*entry)->entries[addr >> shift];
 922                 addr &= (1ULL << shift) - 1;
 923         }
 924
 925         if (level != AMDGPU_VM_PTB)
 926                 *entry = NULL;
 927 }
 928
 929 /**
 930  * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
 931  *
 932  * @p: see amdgpu_pte_update_params definition
 933  * @entry: vm_pt entry to check
 934  * @parent: parent entry
 935  * @nptes: number of PTEs updated with this operation
 936  * @dst: destination address where the PTEs should point to
 937  * @flags: access flags fro the PTEs
 938  *
 939  * Check if we can update the PD with a huge page.
 940  */
 941 static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
 942                                         struct amdgpu_vm_pt *entry,
 943                                         struct amdgpu_vm_pt *parent,
 944                                         unsigned nptes, uint64_t dst,
 945                                         uint64_t flags)
 946 {
 947         uint64_t pde;
 948
 949         /* In the case of a mixed PT the PDE must point to it*/
 950         if (p->adev->asic_type >= CHIP_VEGA10 && !p->src &&
 951             nptes == AMDGPU_VM_PTE_COUNT(p->adev)) {
 952                 /* Set the huge page flag to stop scanning at this PDE */
 953                 flags |= AMDGPU_PDE_PTE;
 954         }
 955
 956         if (!(flags & AMDGPU_PDE_PTE)) {
 957                 if (entry->huge) {
 958                         /* Add the entry to the relocated list to update it. */
 959                         entry->huge = false;
 960                         spin_lock(&p->vm->status_lock);
 961                         list_move(&entry->base.vm_status, &p->vm->relocated);
 962                         spin_unlock(&p->vm->status_lock);
 963                 }
 964                 return;
 965         }
 966
 967         entry->huge = true;
 968         amdgpu_gmc_get_vm_pde(p->adev, AMDGPU_VM_PDB0, &dst, &flags);
 969
 970         pde = (entry - parent->entries) * 8;
 971         if (parent->base.bo->shadow)
 972                 p->func(p, parent->base.bo->shadow, pde, dst, 1, 0, flags);
 973         p->func(p, parent->base.bo, pde, dst, 1, 0, flags);
 974 }
 975
 976 /**
 977  * amdgpu_vm_update_ptes - make sure that page tables are valid
 978  *
 979  * @params: see amdgpu_pte_update_params definition
 980  * @vm: requested vm
 981  * @start: start of GPU address range
 982  * @end: end of GPU address range
 983  * @dst: destination address to map to, the next dst inside the function
 984  * @flags: mapping flags
 985  *
 986  * Update the page tables in the range @start - @end.
 987  * Returns 0 for success, -EINVAL for failure.
 988  */
 989 static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 990                                   uint64_t start, uint64_t end,
 991                                   uint64_t dst, uint64_t flags)
 992 {
 993         struct amdgpu_device *adev = params->adev;
 994         const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
 995
 996         uint64_t addr, pe_start;
 997         struct amdgpu_bo *pt;
 998         unsigned nptes;
 999
1000         /* walk over the address space and update the page tables */
1001         for (addr = start; addr < end; addr += nptes,
1002              dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
1003                 struct amdgpu_vm_pt *entry, *parent;
1004
1005                 amdgpu_vm_get_entry(params, addr, &entry, &parent);
1006                 if (!entry)
1007                         return -ENOENT;
1008
1009                 if ((addr & ~mask) == (end & ~mask))
1010                         nptes = end - addr;
1011                 else
1012                         nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1013
1014                 amdgpu_vm_handle_huge_pages(params, entry, parent,
1015                                             nptes, dst, flags);
1016                 /* We don't need to update PTEs for huge pages */
1017                 if (entry->huge)
1018                         continue;
1019
1020                 pt = entry->base.bo;
1021                 pe_start = (addr & mask) * 8;
1022                 if (pt->shadow)
1023                         params->func(params, pt->shadow, pe_start, dst, nptes,
1024                                      AMDGPU_GPU_PAGE_SIZE, flags);
1025                 params->func(params, pt, pe_start, dst, nptes,
1026                              AMDGPU_GPU_PAGE_SIZE, flags);
1027         }
1028
1029         return 0;
1030 }
1031
1032 /*
1033  * amdgpu_vm_frag_ptes - add fragment information to PTEs
1034  *
1035  * @params: see amdgpu_pte_update_params definition
1036  * @vm: requested vm
1037  * @start: first PTE to handle
1038  * @end: last PTE to handle
1039  * @dst: addr those PTEs should point to
1040  * @flags: hw mapping flags
1041  * Returns 0 for success, -EINVAL for failure.
1042  */
1043 static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params  *params,
1044                                 uint64_t start, uint64_t end,
1045                                 uint64_t dst, uint64_t flags)
1046 {
1047         /**
1048          * The MC L1 TLB supports variable sized pages, based on a fragment
1049          * field in the PTE. When this field is set to a non-zero value, page
1050          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1051          * flags are considered valid for all PTEs within the fragment range
1052          * and corresponding mappings are assumed to be physically contiguous.
1053          *
1054          * The L1 TLB can store a single PTE for the whole fragment,
1055          * significantly increasing the space available for translation
1056          * caching. This leads to large improvements in throughput when the
1057          * TLB is under pressure.
1058          *
1059          * The L2 TLB distributes small and large fragments into two
1060          * asymmetric partitions. The large fragment cache is significantly
1061          * larger. Thus, we try to use large fragments wherever possible.
1062          * Userspace can support this by aligning virtual base address and
1063          * allocation size to the fragment size.
1064          */
1065         unsigned max_frag = params->adev->vm_manager.fragment_size;
1066         int r;
1067
1068         /* system pages are non continuously */
1069         if (params->src || !(flags & AMDGPU_PTE_VALID))
1070                 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
1071
1072         while (start != end) {
1073                 uint64_t frag_flags, frag_end;
1074                 unsigned frag;
1075
1076                 /* This intentionally wraps around if no bit is set */
1077                 frag = min((unsigned)ffs(start) - 1,
1078                            (unsigned)fls64(end - start) - 1);
1079                 if (frag >= max_frag) {
1080                         frag_flags = AMDGPU_PTE_FRAG(max_frag);
1081                         frag_end = end & ~((1ULL << max_frag) - 1);
1082                 } else {
1083                         frag_flags = AMDGPU_PTE_FRAG(frag);
1084                         frag_end = start + (1 << frag);
1085                 }
1086
1087                 r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
1088                                           flags | frag_flags);
1089                 if (r)
1090                         return r;
1091
1092                 dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
1093                 start = frag_end;
1094         }
1095
1096         return 0;
1097 }
1098
1099 /**
1100  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1101  *
1102  * @adev: amdgpu_device pointer
1103  * @exclusive: fence we need to sync to
1104  * @pages_addr: DMA addresses to use for mapping
1105  * @vm: requested vm
1106  * @start: start of mapped range
1107  * @last: last mapped entry
1108  * @flags: flags for the entries
1109  * @addr: addr to set the area to
1110  * @fence: optional resulting fence
1111  *
1112  * Fill in the page table entries between @start and @last.
1113  * Returns 0 for success, -EINVAL for failure.
1114  */
1115 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1116                                        struct dma_fence *exclusive,
1117                                        dma_addr_t *pages_addr,
1118                                        struct amdgpu_vm *vm,
1119                                        uint64_t start, uint64_t last,
1120                                        uint64_t flags, uint64_t addr,
1121                                        struct dma_fence **fence)
1122 {
1123         struct amdgpu_ring *ring;
1124         void *owner = AMDGPU_FENCE_OWNER_VM;
1125         unsigned nptes, ncmds, ndw;
1126         struct amdgpu_job *job;
1127         struct amdgpu_pte_update_params params;
1128         struct dma_fence *f = NULL;
1129         int r;
1130
1131         memset(&params, 0, sizeof(params));
1132         params.adev = adev;
1133         params.vm = vm;
1134
1135         /* sync to everything on unmapping */
1136         if (!(flags & AMDGPU_PTE_VALID))
1137                 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
1138
1139         if (vm->use_cpu_for_update) {
1140                 /* params.src is used as flag to indicate system Memory */
1141                 if (pages_addr)
1142                         params.src = ~0;
1143
1144                 /* Wait for PT BOs to be free. PTs share the same resv. object
1145                  * as the root PD BO
1146                  */
1147                 r = amdgpu_vm_wait_pd(adev, vm, owner);
1148                 if (unlikely(r))
1149                         return r;
1150
1151                 params.func = amdgpu_vm_cpu_set_ptes;
1152                 params.pages_addr = pages_addr;
1153                 return amdgpu_vm_frag_ptes(&params, start, last + 1,
1154                                            addr, flags);
1155         }
1156
1157         ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
1158
1159         nptes = last - start + 1;
1160
1161         /*
1162          * reserve space for two commands every (1 << BLOCK_SIZE)
1163          *  entries or 2k dwords (whatever is smaller)
1164          *
1165          * The second command is for the shadow pagetables.
1166          */
1167         if (vm->root.base.bo->shadow)
1168                 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
1169         else
1170                 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
1171
1172         /* padding, etc. */
1173         ndw = 64;
1174
1175         if (pages_addr) {
1176                 /* copy commands needed */
1177                 ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
1178
1179                 /* and also PTEs */
1180                 ndw += nptes * 2;
1181
1182                 params.func = amdgpu_vm_do_copy_ptes;
1183
1184         } else {
1185                 /* set page commands needed */
1186                 ndw += ncmds * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw;
1187
1188                 /* extra commands for begin/end fragments */
1189                 ndw += 2 * adev->vm_manager.vm_pte_funcs->set_pte_pde_num_dw
1190                                 * adev->vm_manager.fragment_size;
1191
1192                 params.func = amdgpu_vm_do_set_ptes;
1193         }
1194
1195         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1196         if (r)
1197                 return r;
1198
1199         params.ib = &job->ibs[0];
1200
1201         if (pages_addr) {
1202                 uint64_t *pte;
1203                 unsigned i;
1204
1205                 /* Put the PTEs at the end of the IB. */
1206                 i = ndw - nptes * 2;
1207                 pte= (uint64_t *)&(job->ibs->ptr[i]);
1208                 params.src = job->ibs->gpu_addr + i * 4;
1209
1210                 for (i = 0; i < nptes; ++i) {
1211                         pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1212                                                     AMDGPU_GPU_PAGE_SIZE);
1213                         pte[i] |= flags;
1214                 }
1215                 addr = 0;
1216         }
1217
1218         r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
1219         if (r)
1220                 goto error_free;
1221
1222         r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
1223                              owner, false);
1224         if (r)
1225                 goto error_free;
1226
1227         r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
1228         if (r)
1229                 goto error_free;
1230
1231         r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
1232         if (r)
1233                 goto error_free;
1234
1235         amdgpu_ring_pad_ib(ring, params.ib);
1236         WARN_ON(params.ib->length_dw > ndw);
1237         r = amdgpu_job_submit(job, ring, &vm->entity,
1238                               AMDGPU_FENCE_OWNER_VM, &f);
1239         if (r)
1240                 goto error_free;
1241
1242         amdgpu_bo_fence(vm->root.base.bo, f, true);
1243         dma_fence_put(*fence);
1244         *fence = f;
1245         return 0;
1246
1247 error_free:
1248         amdgpu_job_free(job);
1249         return r;
1250 }
1251
1252 /**
1253  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1254  *
1255  * @adev: amdgpu_device pointer
1256  * @exclusive: fence we need to sync to
1257  * @pages_addr: DMA addresses to use for mapping
1258  * @vm: requested vm
1259  * @mapping: mapped range and flags to use for the update
1260  * @flags: HW flags for the mapping
1261  * @nodes: array of drm_mm_nodes with the MC addresses
1262  * @fence: optional resulting fence
1263  *
1264  * Split the mapping into smaller chunks so that each update fits
1265  * into a SDMA IB.
1266  * Returns 0 for success, -EINVAL for failure.
1267  */
1268 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1269                                       struct dma_fence *exclusive,
1270                                       dma_addr_t *pages_addr,
1271                                       struct amdgpu_vm *vm,
1272                                       struct amdgpu_bo_va_mapping *mapping,
1273                                       uint64_t flags,
1274                                       struct drm_mm_node *nodes,
1275                                       struct dma_fence **fence)
1276 {
1277         unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1278         uint64_t pfn, start = mapping->start;
1279         int r;
1280
1281         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1282          * but in case of something, we filter the flags in first place
1283          */
1284         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1285                 flags &= ~AMDGPU_PTE_READABLE;
1286         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1287                 flags &= ~AMDGPU_PTE_WRITEABLE;
1288
1289         flags &= ~AMDGPU_PTE_EXECUTABLE;
1290         flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1291
1292         flags &= ~AMDGPU_PTE_MTYPE_MASK;
1293         flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
1294
1295         if ((mapping->flags & AMDGPU_PTE_PRT) &&
1296             (adev->asic_type >= CHIP_VEGA10)) {
1297                 flags |= AMDGPU_PTE_PRT;
1298                 flags &= ~AMDGPU_PTE_VALID;
1299         }
1300
1301         trace_amdgpu_vm_bo_update(mapping);
1302
1303         pfn = mapping->offset >> PAGE_SHIFT;
1304         if (nodes) {
1305                 while (pfn >= nodes->size) {
1306                         pfn -= nodes->size;
1307                         ++nodes;
1308                 }
1309         }
1310
1311         do {
1312                 dma_addr_t *dma_addr = NULL;
1313                 uint64_t max_entries;
1314                 uint64_t addr, last;
1315
1316                 if (nodes) {
1317                         addr = nodes->start << PAGE_SHIFT;
1318                         max_entries = (nodes->size - pfn) *
1319                                 (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
1320                 } else {
1321                         addr = 0;
1322                         max_entries = S64_MAX;
1323                 }
1324
1325                 if (pages_addr) {
1326                         uint64_t count;
1327
1328                         max_entries = min(max_entries, 16ull * 1024ull);
1329                         for (count = 1; count < max_entries; ++count) {
1330                                 uint64_t idx = pfn + count;
1331
1332                                 if (pages_addr[idx] !=
1333                                     (pages_addr[idx - 1] + PAGE_SIZE))
1334                                         break;
1335                         }
1336
1337                         if (count < min_linear_pages) {
1338                                 addr = pfn << PAGE_SHIFT;
1339                                 dma_addr = pages_addr;
1340                         } else {
1341                                 addr = pages_addr[pfn];
1342                                 max_entries = count;
1343                         }
1344
1345                 } else if (flags & AMDGPU_PTE_VALID) {
1346                         addr += adev->vm_manager.vram_base_offset;
1347                         addr += pfn << PAGE_SHIFT;
1348                 }
1349
1350                 last = min((uint64_t)mapping->last, start + max_entries - 1);
1351                 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1352                                                 start, last, flags, addr,
1353                                                 fence);
1354                 if (r)
1355                         return r;
1356
1357                 pfn += last - start + 1;
1358                 if (nodes && nodes->size == pfn) {
1359                         pfn = 0;
1360                         ++nodes;
1361                 }
1362                 start = last + 1;
1363
1364         } while (unlikely(start != mapping->last + 1));
1365
1366         return 0;
1367 }
1368
1369 /**
1370  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1371  *
1372  * @adev: amdgpu_device pointer
1373  * @bo_va: requested BO and VM object
1374  * @clear: if true clear the entries
1375  *
1376  * Fill in the page table entries for @bo_va.
1377  * Returns 0 for success, -EINVAL for failure.
1378  */
1379 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1380                         struct amdgpu_bo_va *bo_va,
1381                         bool clear)
1382 {
1383         struct amdgpu_bo *bo = bo_va->base.bo;
1384         struct amdgpu_vm *vm = bo_va->base.vm;
1385         struct amdgpu_bo_va_mapping *mapping;
1386         dma_addr_t *pages_addr = NULL;
1387         struct ttm_mem_reg *mem;
1388         struct drm_mm_node *nodes;
1389         struct dma_fence *exclusive, **last_update;
1390         uint64_t flags;
1391         int r;
1392
1393         if (clear || !bo_va->base.bo) {
1394                 mem = NULL;
1395                 nodes = NULL;
1396                 exclusive = NULL;
1397         } else {
1398                 struct ttm_dma_tt *ttm;
1399
1400                 mem = &bo_va->base.bo->tbo.mem;
1401                 nodes = mem->mm_node;
1402                 if (mem->mem_type == TTM_PL_TT) {
1403                         ttm = container_of(bo_va->base.bo->tbo.ttm,
1404                                            struct ttm_dma_tt, ttm);
1405                         pages_addr = ttm->dma_address;
1406                 }
1407                 exclusive = reservation_object_get_excl(bo->tbo.resv);
1408         }
1409
1410         if (bo)
1411                 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1412         else
1413                 flags = 0x0;
1414
1415         if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
1416                 last_update = &vm->last_update;
1417         else
1418                 last_update = &bo_va->last_pt_update;
1419
1420         if (!clear && bo_va->base.moved) {
1421                 bo_va->base.moved = false;
1422                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1423
1424         } else if (bo_va->cleared != clear) {
1425                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1426         }
1427
1428         list_for_each_entry(mapping, &bo_va->invalids, list) {
1429                 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1430                                                mapping, flags, nodes,
1431                                                last_update);
1432                 if (r)
1433                         return r;
1434         }
1435
1436         if (vm->use_cpu_for_update) {
1437                 /* Flush HDP */
1438                 mb();
1439                 amdgpu_asic_flush_hdp(adev, NULL);
1440         }
1441
1442         spin_lock(&vm->status_lock);
1443         list_del_init(&bo_va->base.vm_status);
1444         spin_unlock(&vm->status_lock);
1445
1446         list_splice_init(&bo_va->invalids, &bo_va->valids);
1447         bo_va->cleared = clear;
1448
1449         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1450                 list_for_each_entry(mapping, &bo_va->valids, list)
1451                         trace_amdgpu_vm_bo_mapping(mapping);
1452         }
1453
1454         return 0;
1455 }
1456
1457 /**
1458  * amdgpu_vm_update_prt_state - update the global PRT state
1459  */
1460 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1461 {
1462         unsigned long flags;
1463         bool enable;
1464
1465         spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1466         enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1467         adev->gmc.gmc_funcs->set_prt(adev, enable);
1468         spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1469 }
1470
1471 /**
1472  * amdgpu_vm_prt_get - add a PRT user
1473  */
1474 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1475 {
1476         if (!adev->gmc.gmc_funcs->set_prt)
1477                 return;
1478
1479         if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1480                 amdgpu_vm_update_prt_state(adev);
1481 }
1482
1483 /**
1484  * amdgpu_vm_prt_put - drop a PRT user
1485  */
1486 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1487 {
1488         if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1489                 amdgpu_vm_update_prt_state(adev);
1490 }
1491
1492 /**
1493  * amdgpu_vm_prt_cb - callback for updating the PRT status
1494  */
1495 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1496 {
1497         struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1498
1499         amdgpu_vm_prt_put(cb->adev);
1500         kfree(cb);
1501 }
1502
1503 /**
1504  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1505  */
1506 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1507                                  struct dma_fence *fence)
1508 {
1509         struct amdgpu_prt_cb *cb;
1510
1511         if (!adev->gmc.gmc_funcs->set_prt)
1512                 return;
1513
1514         cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1515         if (!cb) {
1516                 /* Last resort when we are OOM */
1517                 if (fence)
1518                         dma_fence_wait(fence, false);
1519
1520                 amdgpu_vm_prt_put(adev);
1521         } else {
1522                 cb->adev = adev;
1523                 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1524                                                      amdgpu_vm_prt_cb))
1525                         amdgpu_vm_prt_cb(fence, &cb->cb);
1526         }
1527 }
1528
1529 /**
1530  * amdgpu_vm_free_mapping - free a mapping
1531  *
1532  * @adev: amdgpu_device pointer
1533  * @vm: requested vm
1534  * @mapping: mapping to be freed
1535  * @fence: fence of the unmap operation
1536  *
1537  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1538  */
1539 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1540                                    struct amdgpu_vm *vm,
1541                                    struct amdgpu_bo_va_mapping *mapping,
1542                                    struct dma_fence *fence)
1543 {
1544         if (mapping->flags & AMDGPU_PTE_PRT)
1545                 amdgpu_vm_add_prt_cb(adev, fence);
1546         kfree(mapping);
1547 }
1548
1549 /**
1550  * amdgpu_vm_prt_fini - finish all prt mappings
1551  *
1552  * @adev: amdgpu_device pointer
1553  * @vm: requested vm
1554  *
1555  * Register a cleanup callback to disable PRT support after VM dies.
1556  */
1557 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1558 {
1559         struct reservation_object *resv = vm->root.base.bo->tbo.resv;
1560         struct dma_fence *excl, **shared;
1561         unsigned i, shared_count;
1562         int r;
1563
1564         r = reservation_object_get_fences_rcu(resv, &excl,
1565                                               &shared_count, &shared);
1566         if (r) {
1567                 /* Not enough memory to grab the fence list, as last resort
1568                  * block for all the fences to complete.
1569                  */
1570                 reservation_object_wait_timeout_rcu(resv, true, false,
1571                                                     MAX_SCHEDULE_TIMEOUT);
1572                 return;
1573         }
1574
1575         /* Add a callback for each fence in the reservation object */
1576         amdgpu_vm_prt_get(adev);
1577         amdgpu_vm_add_prt_cb(adev, excl);
1578
1579         for (i = 0; i < shared_count; ++i) {
1580                 amdgpu_vm_prt_get(adev);
1581                 amdgpu_vm_add_prt_cb(adev, shared[i]);
1582         }
1583
1584         kfree(shared);
1585 }
1586
1587 /**
1588  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1589  *
1590  * @adev: amdgpu_device pointer
1591  * @vm: requested vm
1592  * @fence: optional resulting fence (unchanged if no work needed to be done
1593  * or if an error occurred)
1594  *
1595  * Make sure all freed BOs are cleared in the PT.
1596  * Returns 0 for success.
1597  *
1598  * PTs have to be reserved and mutex must be locked!
1599  */
1600 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1601                           struct amdgpu_vm *vm,
1602                           struct dma_fence **fence)
1603 {
1604         struct amdgpu_bo_va_mapping *mapping;
1605         struct dma_fence *f = NULL;
1606         int r;
1607         uint64_t init_pte_value = 0;
1608
1609         while (!list_empty(&vm->freed)) {
1610                 mapping = list_first_entry(&vm->freed,
1611                         struct amdgpu_bo_va_mapping, list);
1612                 list_del(&mapping->list);
1613
1614                 if (vm->pte_support_ats)
1615                         init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1616
1617                 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
1618                                                 mapping->start, mapping->last,
1619                                                 init_pte_value, 0, &f);
1620                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1621                 if (r) {
1622                         dma_fence_put(f);
1623                         return r;
1624                 }
1625         }
1626
1627         if (fence && f) {
1628                 dma_fence_put(*fence);
1629                 *fence = f;
1630         } else {
1631                 dma_fence_put(f);
1632         }
1633
1634         return 0;
1635
1636 }
1637
1638 /**
1639  * amdgpu_vm_handle_moved - handle moved BOs in the PT
1640  *
1641  * @adev: amdgpu_device pointer
1642  * @vm: requested vm
1643  * @sync: sync object to add fences to
1644  *
1645  * Make sure all BOs which are moved are updated in the PTs.
1646  * Returns 0 for success.
1647  *
1648  * PTs have to be reserved!
1649  */
1650 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1651                            struct amdgpu_vm *vm)
1652 {
1653         bool clear;
1654         int r = 0;
1655
1656         spin_lock(&vm->status_lock);
1657         while (!list_empty(&vm->moved)) {
1658                 struct amdgpu_bo_va *bo_va;
1659                 struct reservation_object *resv;
1660
1661                 bo_va = list_first_entry(&vm->moved,
1662                         struct amdgpu_bo_va, base.vm_status);
1663                 spin_unlock(&vm->status_lock);
1664
1665                 resv = bo_va->base.bo->tbo.resv;
1666
1667                 /* Per VM BOs never need to bo cleared in the page tables */
1668                 if (resv == vm->root.base.bo->tbo.resv)
1669                         clear = false;
1670                 /* Try to reserve the BO to avoid clearing its ptes */
1671                 else if (!amdgpu_vm_debug && reservation_object_trylock(resv))
1672                         clear = false;
1673                 /* Somebody else is using the BO right now */
1674                 else
1675                         clear = true;
1676
1677                 r = amdgpu_vm_bo_update(adev, bo_va, clear);
1678                 if (r)
1679                         return r;
1680
1681                 if (!clear && resv != vm->root.base.bo->tbo.resv)
1682                         reservation_object_unlock(resv);
1683
1684                 spin_lock(&vm->status_lock);
1685         }
1686         spin_unlock(&vm->status_lock);
1687
1688         return r;
1689 }
1690
1691 /**
1692  * amdgpu_vm_bo_add - add a bo to a specific vm
1693  *
1694  * @adev: amdgpu_device pointer
1695  * @vm: requested vm
1696  * @bo: amdgpu buffer object
1697  *
1698  * Add @bo into the requested vm.
1699  * Add @bo to the list of bos associated with the vm
1700  * Returns newly added bo_va or NULL for failure
1701  *
1702  * Object has to be reserved!
1703  */
1704 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1705                                       struct amdgpu_vm *vm,
1706                                       struct amdgpu_bo *bo)
1707 {
1708         struct amdgpu_bo_va *bo_va;
1709
1710         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1711         if (bo_va == NULL) {
1712                 return NULL;
1713         }
1714         bo_va->base.vm = vm;
1715         bo_va->base.bo = bo;
1716         INIT_LIST_HEAD(&bo_va->base.bo_list);
1717         INIT_LIST_HEAD(&bo_va->base.vm_status);
1718
1719         bo_va->ref_count = 1;
1720         INIT_LIST_HEAD(&bo_va->valids);
1721         INIT_LIST_HEAD(&bo_va->invalids);
1722
1723         if (!bo)
1724                 return bo_va;
1725
1726         list_add_tail(&bo_va->base.bo_list, &bo->va);
1727
1728         if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
1729                 return bo_va;
1730
1731         if (bo->preferred_domains &
1732             amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
1733                 return bo_va;
1734
1735         /*
1736          * We checked all the prerequisites, but it looks like this per VM BO
1737          * is currently evicted. add the BO to the evicted list to make sure it
1738          * is validated on next VM use to avoid fault.
1739          * */
1740         spin_lock(&vm->status_lock);
1741         list_move_tail(&bo_va->base.vm_status, &vm->evicted);
1742         spin_unlock(&vm->status_lock);
1743
1744         return bo_va;
1745 }
1746
1747
1748 /**
1749  * amdgpu_vm_bo_insert_mapping - insert a new mapping
1750  *
1751  * @adev: amdgpu_device pointer
1752  * @bo_va: bo_va to store the address
1753  * @mapping: the mapping to insert
1754  *
1755  * Insert a new mapping into all structures.
1756  */
1757 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1758                                     struct amdgpu_bo_va *bo_va,
1759                                     struct amdgpu_bo_va_mapping *mapping)
1760 {
1761         struct amdgpu_vm *vm = bo_va->base.vm;
1762         struct amdgpu_bo *bo = bo_va->base.bo;
1763
1764         mapping->bo_va = bo_va;
1765         list_add(&mapping->list, &bo_va->invalids);
1766         amdgpu_vm_it_insert(mapping, &vm->va);
1767
1768         if (mapping->flags & AMDGPU_PTE_PRT)
1769                 amdgpu_vm_prt_get(adev);
1770
1771         if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
1772                 spin_lock(&vm->status_lock);
1773                 if (list_empty(&bo_va->base.vm_status))
1774                         list_add(&bo_va->base.vm_status, &vm->moved);
1775                 spin_unlock(&vm->status_lock);
1776         }
1777         trace_amdgpu_vm_bo_map(bo_va, mapping);
1778 }
1779
1780 /**
1781  * amdgpu_vm_bo_map - map bo inside a vm
1782  *
1783  * @adev: amdgpu_device pointer
1784  * @bo_va: bo_va to store the address
1785  * @saddr: where to map the BO
1786  * @offset: requested offset in the BO
1787  * @flags: attributes of pages (read/write/valid/etc.)
1788  *
1789  * Add a mapping of the BO at the specefied addr into the VM.
1790  * Returns 0 for success, error for failure.
1791  *
1792  * Object has to be reserved and unreserved outside!
1793  */
1794 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1795                      struct amdgpu_bo_va *bo_va,
1796                      uint64_t saddr, uint64_t offset,
1797                      uint64_t size, uint64_t flags)
1798 {
1799         struct amdgpu_bo_va_mapping *mapping, *tmp;
1800         struct amdgpu_bo *bo = bo_va->base.bo;
1801         struct amdgpu_vm *vm = bo_va->base.vm;
1802         uint64_t eaddr;
1803
1804         /* validate the parameters */
1805         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1806             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1807                 return -EINVAL;
1808
1809         /* make sure object fit at this offset */
1810         eaddr = saddr + size - 1;
1811         if (saddr >= eaddr ||
1812             (bo && offset + size > amdgpu_bo_size(bo)))
1813                 return -EINVAL;
1814
1815         saddr /= AMDGPU_GPU_PAGE_SIZE;
1816         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1817
1818         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1819         if (tmp) {
1820                 /* bo and tmp overlap, invalid addr */
1821                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1822                         "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1823                         tmp->start, tmp->last + 1);
1824                 return -EINVAL;
1825         }
1826
1827         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1828         if (!mapping)
1829                 return -ENOMEM;
1830
1831         mapping->start = saddr;
1832         mapping->last = eaddr;
1833         mapping->offset = offset;
1834         mapping->flags = flags;
1835
1836         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1837
1838         return 0;
1839 }
1840
1841 /**
1842  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1843  *
1844  * @adev: amdgpu_device pointer
1845  * @bo_va: bo_va to store the address
1846  * @saddr: where to map the BO
1847  * @offset: requested offset in the BO
1848  * @flags: attributes of pages (read/write/valid/etc.)
1849  *
1850  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1851  * mappings as we do so.
1852  * Returns 0 for success, error for failure.
1853  *
1854  * Object has to be reserved and unreserved outside!
1855  */
1856 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1857                              struct amdgpu_bo_va *bo_va,
1858                              uint64_t saddr, uint64_t offset,
1859                              uint64_t size, uint64_t flags)
1860 {
1861         struct amdgpu_bo_va_mapping *mapping;
1862         struct amdgpu_bo *bo = bo_va->base.bo;
1863         uint64_t eaddr;
1864         int r;
1865
1866         /* validate the parameters */
1867         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1868             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1869                 return -EINVAL;
1870
1871         /* make sure object fit at this offset */
1872         eaddr = saddr + size - 1;
1873         if (saddr >= eaddr ||
1874             (bo && offset + size > amdgpu_bo_size(bo)))
1875                 return -EINVAL;
1876
1877         /* Allocate all the needed memory */
1878         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1879         if (!mapping)
1880                 return -ENOMEM;
1881
1882         r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1883         if (r) {
1884                 kfree(mapping);
1885                 return r;
1886         }
1887
1888         saddr /= AMDGPU_GPU_PAGE_SIZE;
1889         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1890
1891         mapping->start = saddr;
1892         mapping->last = eaddr;
1893         mapping->offset = offset;
1894         mapping->flags = flags;
1895
1896         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1897
1898         return 0;
1899 }
1900
1901 /**
1902  * amdgpu_vm_bo_unmap - remove bo mapping from vm
1903  *
1904  * @adev: amdgpu_device pointer
1905  * @bo_va: bo_va to remove the address from
1906  * @saddr: where to the BO is mapped
1907  *
1908  * Remove a mapping of the BO at the specefied addr from the VM.
1909  * Returns 0 for success, error for failure.
1910  *
1911  * Object has to be reserved and unreserved outside!
1912  */
1913 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1914                        struct amdgpu_bo_va *bo_va,
1915                        uint64_t saddr)
1916 {
1917         struct amdgpu_bo_va_mapping *mapping;
1918         struct amdgpu_vm *vm = bo_va->base.vm;
1919         bool valid = true;
1920
1921         saddr /= AMDGPU_GPU_PAGE_SIZE;
1922
1923         list_for_each_entry(mapping, &bo_va->valids, list) {
1924                 if (mapping->start == saddr)
1925                         break;
1926         }
1927
1928         if (&mapping->list == &bo_va->valids) {
1929                 valid = false;
1930
1931                 list_for_each_entry(mapping, &bo_va->invalids, list) {
1932                         if (mapping->start == saddr)
1933                                 break;
1934                 }
1935
1936                 if (&mapping->list == &bo_va->invalids)
1937                         return -ENOENT;
1938         }
1939
1940         list_del(&mapping->list);
1941         amdgpu_vm_it_remove(mapping, &vm->va);
1942         mapping->bo_va = NULL;
1943         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1944
1945         if (valid)
1946                 list_add(&mapping->list, &vm->freed);
1947         else
1948                 amdgpu_vm_free_mapping(adev, vm, mapping,
1949                                        bo_va->last_pt_update);
1950
1951         return 0;
1952 }
1953
1954 /**
1955  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
1956  *
1957  * @adev: amdgpu_device pointer
1958  * @vm: VM structure to use
1959  * @saddr: start of the range
1960  * @size: size of the range
1961  *
1962  * Remove all mappings in a range, split them as appropriate.
1963  * Returns 0 for success, error for failure.
1964  */
1965 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
1966                                 struct amdgpu_vm *vm,
1967                                 uint64_t saddr, uint64_t size)
1968 {
1969         struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
1970         LIST_HEAD(removed);
1971         uint64_t eaddr;
1972
1973         eaddr = saddr + size - 1;
1974         saddr /= AMDGPU_GPU_PAGE_SIZE;
1975         eaddr /= AMDGPU_GPU_PAGE_SIZE;
1976
1977         /* Allocate all the needed memory */
1978         before = kzalloc(sizeof(*before), GFP_KERNEL);
1979         if (!before)
1980                 return -ENOMEM;
1981         INIT_LIST_HEAD(&before->list);
1982
1983         after = kzalloc(sizeof(*after), GFP_KERNEL);
1984         if (!after) {
1985                 kfree(before);
1986                 return -ENOMEM;
1987         }
1988         INIT_LIST_HEAD(&after->list);
1989
1990         /* Now gather all removed mappings */
1991         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1992         while (tmp) {
1993                 /* Remember mapping split at the start */
1994                 if (tmp->start < saddr) {
1995                         before->start = tmp->start;
1996                         before->last = saddr - 1;
1997                         before->offset = tmp->offset;
1998                         before->flags = tmp->flags;
1999                         list_add(&before->list, &tmp->list);
2000                 }
2001
2002                 /* Remember mapping split at the end */
2003                 if (tmp->last > eaddr) {
2004                         after->start = eaddr + 1;
2005                         after->last = tmp->last;
2006                         after->offset = tmp->offset;
2007                         after->offset += after->start - tmp->start;
2008                         after->flags = tmp->flags;
2009                         list_add(&after->list, &tmp->list);
2010                 }
2011
2012                 list_del(&tmp->list);
2013                 list_add(&tmp->list, &removed);
2014
2015                 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2016         }
2017
2018         /* And free them up */
2019         list_for_each_entry_safe(tmp, next, &removed, list) {
2020                 amdgpu_vm_it_remove(tmp, &vm->va);
2021                 list_del(&tmp->list);
2022
2023                 if (tmp->start < saddr)
2024                     tmp->start = saddr;
2025                 if (tmp->last > eaddr)
2026                     tmp->last = eaddr;
2027
2028                 tmp->bo_va = NULL;
2029                 list_add(&tmp->list, &vm->freed);
2030                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2031         }
2032
2033         /* Insert partial mapping before the range */
2034         if (!list_empty(&before->list)) {
2035                 amdgpu_vm_it_insert(before, &vm->va);
2036                 if (before->flags & AMDGPU_PTE_PRT)
2037                         amdgpu_vm_prt_get(adev);
2038         } else {
2039                 kfree(before);
2040         }
2041
2042         /* Insert partial mapping after the range */
2043         if (!list_empty(&after->list)) {
2044                 amdgpu_vm_it_insert(after, &vm->va);
2045                 if (after->flags & AMDGPU_PTE_PRT)
2046                         amdgpu_vm_prt_get(adev);
2047         } else {
2048                 kfree(after);
2049         }
2050
2051         return 0;
2052 }
2053
2054 /**
2055  * amdgpu_vm_bo_lookup_mapping - find mapping by address
2056  *
2057  * @vm: the requested VM
2058  *
2059  * Find a mapping by it's address.
2060  */
2061 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2062                                                          uint64_t addr)
2063 {
2064         return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2065 }
2066
2067 /**
2068  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2069  *
2070  * @adev: amdgpu_device pointer
2071  * @bo_va: requested bo_va
2072  *
2073  * Remove @bo_va->bo from the requested vm.
2074  *
2075  * Object have to be reserved!
2076  */
2077 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2078                       struct amdgpu_bo_va *bo_va)
2079 {
2080         struct amdgpu_bo_va_mapping *mapping, *next;
2081         struct amdgpu_vm *vm = bo_va->base.vm;
2082
2083         list_del(&bo_va->base.bo_list);
2084
2085         spin_lock(&vm->status_lock);
2086         list_del(&bo_va->base.vm_status);
2087         spin_unlock(&vm->status_lock);
2088
2089         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2090                 list_del(&mapping->list);
2091                 amdgpu_vm_it_remove(mapping, &vm->va);
2092                 mapping->bo_va = NULL;
2093                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2094                 list_add(&mapping->list, &vm->freed);
2095         }
2096         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2097                 list_del(&mapping->list);
2098                 amdgpu_vm_it_remove(mapping, &vm->va);
2099                 amdgpu_vm_free_mapping(adev, vm, mapping,
2100                                        bo_va->last_pt_update);
2101         }
2102
2103         dma_fence_put(bo_va->last_pt_update);
2104         kfree(bo_va);
2105 }
2106
2107 /**
2108  * amdgpu_vm_bo_invalidate - mark the bo as invalid
2109  *
2110  * @adev: amdgpu_device pointer
2111  * @vm: requested vm
2112  * @bo: amdgpu buffer object
2113  *
2114  * Mark @bo as invalid.
2115  */
2116 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2117                              struct amdgpu_bo *bo, bool evicted)
2118 {
2119         struct amdgpu_vm_bo_base *bo_base;
2120
2121         list_for_each_entry(bo_base, &bo->va, bo_list) {
2122                 struct amdgpu_vm *vm = bo_base->vm;
2123
2124                 bo_base->moved = true;
2125                 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2126                         spin_lock(&bo_base->vm->status_lock);
2127                         if (bo->tbo.type == ttm_bo_type_kernel)
2128                                 list_move(&bo_base->vm_status, &vm->evicted);
2129                         else
2130                                 list_move_tail(&bo_base->vm_status,
2131                                                &vm->evicted);
2132                         spin_unlock(&bo_base->vm->status_lock);
2133                         continue;
2134                 }
2135
2136                 if (bo->tbo.type == ttm_bo_type_kernel) {
2137                         spin_lock(&bo_base->vm->status_lock);
2138                         if (list_empty(&bo_base->vm_status))
2139                                 list_add(&bo_base->vm_status, &vm->relocated);
2140                         spin_unlock(&bo_base->vm->status_lock);
2141                         continue;
2142                 }
2143
2144                 spin_lock(&bo_base->vm->status_lock);
2145                 if (list_empty(&bo_base->vm_status))
2146                         list_add(&bo_base->vm_status, &vm->moved);
2147                 spin_unlock(&bo_base->vm->status_lock);
2148         }
2149 }
2150
2151 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2152 {
2153         /* Total bits covered by PD + PTs */
2154         unsigned bits = ilog2(vm_size) + 18;
2155
2156         /* Make sure the PD is 4K in size up to 8GB address space.
2157            Above that split equal between PD and PTs */
2158         if (vm_size <= 8)
2159                 return (bits - 9);
2160         else
2161                 return ((bits + 3) / 2);
2162 }
2163
2164 /**
2165  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2166  *
2167  * @adev: amdgpu_device pointer
2168  * @vm_size: the default vm size if it's set auto
2169  */
2170 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
2171                            uint32_t fragment_size_default, unsigned max_level,
2172                            unsigned max_bits)
2173 {
2174         uint64_t tmp;
2175
2176         /* adjust vm size first */
2177         if (amdgpu_vm_size != -1) {
2178                 unsigned max_size = 1 << (max_bits - 30);
2179
2180                 vm_size = amdgpu_vm_size;
2181                 if (vm_size > max_size) {
2182                         dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2183                                  amdgpu_vm_size, max_size);
2184                         vm_size = max_size;
2185                 }
2186         }
2187
2188         adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2189
2190         tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2191         if (amdgpu_vm_block_size != -1)
2192                 tmp >>= amdgpu_vm_block_size - 9;
2193         tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2194         adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2195         switch (adev->vm_manager.num_level) {
2196         case 3:
2197                 adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2198                 break;
2199         case 2:
2200                 adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2201                 break;
2202         case 1:
2203                 adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2204                 break;
2205         default:
2206                 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2207         }
2208         /* block size depends on vm size and hw setup*/
2209         if (amdgpu_vm_block_size != -1)
2210                 adev->vm_manager.block_size =
2211                         min((unsigned)amdgpu_vm_block_size, max_bits
2212                             - AMDGPU_GPU_PAGE_SHIFT
2213                             - 9 * adev->vm_manager.num_level);
2214         else if (adev->vm_manager.num_level > 1)
2215                 adev->vm_manager.block_size = 9;
2216         else
2217                 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2218
2219         if (amdgpu_vm_fragment_size == -1)
2220                 adev->vm_manager.fragment_size = fragment_size_default;
2221         else
2222                 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2223
2224         DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2225                  vm_size, adev->vm_manager.num_level + 1,
2226                  adev->vm_manager.block_size,
2227                  adev->vm_manager.fragment_size);
2228 }
2229
2230 /**
2231  * amdgpu_vm_init - initialize a vm instance
2232  *
2233  * @adev: amdgpu_device pointer
2234  * @vm: requested vm
2235  * @vm_context: Indicates if it GFX or Compute context
2236  *
2237  * Init @vm fields.
2238  */
2239 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2240                    int vm_context, unsigned int pasid)
2241 {
2242         const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
2243                 AMDGPU_VM_PTE_COUNT(adev) * 8);
2244         uint64_t init_pde_value = 0, flags;
2245         unsigned ring_instance;
2246         struct amdgpu_ring *ring;
2247         struct drm_sched_rq *rq;
2248         unsigned long size;
2249         int r, i;
2250
2251         vm->va = RB_ROOT_CACHED;
2252         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2253                 vm->reserved_vmid[i] = NULL;
2254         spin_lock_init(&vm->status_lock);
2255         INIT_LIST_HEAD(&vm->evicted);
2256         INIT_LIST_HEAD(&vm->relocated);
2257         INIT_LIST_HEAD(&vm->moved);
2258         INIT_LIST_HEAD(&vm->freed);
2259
2260         /* create scheduler entity for page table updates */
2261
2262         ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
2263         ring_instance %= adev->vm_manager.vm_pte_num_rings;
2264         ring = adev->vm_manager.vm_pte_rings[ring_instance];
2265         rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
2266         r = drm_sched_entity_init(&ring->sched, &vm->entity,
2267                                   rq, amdgpu_sched_jobs, NULL);
2268         if (r)
2269                 return r;
2270
2271         vm->pte_support_ats = false;
2272
2273         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2274                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2275                                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2276
2277                 if (adev->asic_type == CHIP_RAVEN) {
2278                         vm->pte_support_ats = true;
2279                         init_pde_value = AMDGPU_PTE_DEFAULT_ATC
2280                                         | AMDGPU_PDE_PTE;
2281
2282                 }
2283         } else
2284                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2285                                                 AMDGPU_VM_USE_CPU_FOR_GFX);
2286         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2287                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2288         WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
2289                   "CPU update of VM recommended only for large BAR system\n");
2290         vm->last_update = NULL;
2291
2292         flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
2293                         AMDGPU_GEM_CREATE_VRAM_CLEARED;
2294         if (vm->use_cpu_for_update)
2295                 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
2296         else
2297                 flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
2298                                 AMDGPU_GEM_CREATE_SHADOW);
2299
2300         size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
2301         r = amdgpu_bo_create(adev, size, align, true, AMDGPU_GEM_DOMAIN_VRAM,
2302                              flags, NULL, NULL, init_pde_value,
2303                              &vm->root.base.bo);
2304         if (r)
2305                 goto error_free_sched_entity;
2306
2307         r = amdgpu_bo_reserve(vm->root.base.bo, true);
2308         if (r)
2309                 goto error_free_root;
2310
2311         vm->root.base.vm = vm;
2312         list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
2313         list_add_tail(&vm->root.base.vm_status, &vm->evicted);
2314         amdgpu_bo_unreserve(vm->root.base.bo);
2315
2316         if (pasid) {
2317                 unsigned long flags;
2318
2319                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2320                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2321                               GFP_ATOMIC);
2322                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2323                 if (r < 0)
2324                         goto error_free_root;
2325
2326                 vm->pasid = pasid;
2327         }
2328
2329         INIT_KFIFO(vm->faults);
2330         vm->fault_credit = 16;
2331
2332         return 0;
2333
2334 error_free_root:
2335         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2336         amdgpu_bo_unref(&vm->root.base.bo);
2337         vm->root.base.bo = NULL;
2338
2339 error_free_sched_entity:
2340         drm_sched_entity_fini(&ring->sched, &vm->entity);
2341
2342         return r;
2343 }
2344
2345 /**
2346  * amdgpu_vm_free_levels - free PD/PT levels
2347  *
2348  * @adev: amdgpu device structure
2349  * @parent: PD/PT starting level to free
2350  * @level: level of parent structure
2351  *
2352  * Free the page directory or page table level and all sub levels.
2353  */
2354 static void amdgpu_vm_free_levels(struct amdgpu_device *adev,
2355                                   struct amdgpu_vm_pt *parent,
2356                                   unsigned level)
2357 {
2358         unsigned i, num_entries = amdgpu_vm_num_entries(adev, level);
2359
2360         if (parent->base.bo) {
2361                 list_del(&parent->base.bo_list);
2362                 list_del(&parent->base.vm_status);
2363                 amdgpu_bo_unref(&parent->base.bo->shadow);
2364                 amdgpu_bo_unref(&parent->base.bo);
2365         }
2366
2367         if (parent->entries)
2368                 for (i = 0; i < num_entries; i++)
2369                         amdgpu_vm_free_levels(adev, &parent->entries[i],
2370                                               level + 1);
2371
2372         kvfree(parent->entries);
2373 }
2374
2375 /**
2376  * amdgpu_vm_fini - tear down a vm instance
2377  *
2378  * @adev: amdgpu_device pointer
2379  * @vm: requested vm
2380  *
2381  * Tear down @vm.
2382  * Unbind the VM and remove all bos from the vm bo list
2383  */
2384 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2385 {
2386         struct amdgpu_bo_va_mapping *mapping, *tmp;
2387         bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2388         struct amdgpu_bo *root;
2389         u64 fault;
2390         int i, r;
2391
2392         /* Clear pending page faults from IH when the VM is destroyed */
2393         while (kfifo_get(&vm->faults, &fault))
2394                 amdgpu_ih_clear_fault(adev, fault);
2395
2396         if (vm->pasid) {
2397                 unsigned long flags;
2398
2399                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2400                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2401                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2402         }
2403
2404         drm_sched_entity_fini(vm->entity.sched, &vm->entity);
2405
2406         if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2407                 dev_err(adev->dev, "still active bo inside vm\n");
2408         }
2409         rbtree_postorder_for_each_entry_safe(mapping, tmp,
2410                                              &vm->va.rb_root, rb) {
2411                 list_del(&mapping->list);
2412                 amdgpu_vm_it_remove(mapping, &vm->va);
2413                 kfree(mapping);
2414         }
2415         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2416                 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2417                         amdgpu_vm_prt_fini(adev, vm);
2418                         prt_fini_needed = false;
2419                 }
2420
2421                 list_del(&mapping->list);
2422                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2423         }
2424
2425         root = amdgpu_bo_ref(vm->root.base.bo);
2426         r = amdgpu_bo_reserve(root, true);
2427         if (r) {
2428                 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
2429         } else {
2430                 amdgpu_vm_free_levels(adev, &vm->root,
2431                                       adev->vm_manager.root_level);
2432                 amdgpu_bo_unreserve(root);
2433         }
2434         amdgpu_bo_unref(&root);
2435         dma_fence_put(vm->last_update);
2436         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2437                 amdgpu_vmid_free_reserved(adev, vm, i);
2438 }
2439
2440 /**
2441  * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
2442  *
2443  * @adev: amdgpu_device pointer
2444  * @pasid: PASID do identify the VM
2445  *
2446  * This function is expected to be called in interrupt context. Returns
2447  * true if there was fault credit, false otherwise
2448  */
2449 bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
2450                                   unsigned int pasid)
2451 {
2452         struct amdgpu_vm *vm;
2453
2454         spin_lock(&adev->vm_manager.pasid_lock);
2455         vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
2456         if (!vm) {
2457                 /* VM not found, can't track fault credit */
2458                 spin_unlock(&adev->vm_manager.pasid_lock);
2459                 return true;
2460         }
2461
2462         /* No lock needed. only accessed by IRQ handler */
2463         if (!vm->fault_credit) {
2464                 /* Too many faults in this VM */
2465                 spin_unlock(&adev->vm_manager.pasid_lock);
2466                 return false;
2467         }
2468
2469         vm->fault_credit--;
2470         spin_unlock(&adev->vm_manager.pasid_lock);
2471         return true;
2472 }
2473
2474 /**
2475  * amdgpu_vm_manager_init - init the VM manager
2476  *
2477  * @adev: amdgpu_device pointer
2478  *
2479  * Initialize the VM manager structures
2480  */
2481 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2482 {
2483         unsigned i;
2484
2485         amdgpu_vmid_mgr_init(adev);
2486
2487         adev->vm_manager.fence_context =
2488                 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2489         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
2490                 adev->vm_manager.seqno[i] = 0;
2491
2492         atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
2493         spin_lock_init(&adev->vm_manager.prt_lock);
2494         atomic_set(&adev->vm_manager.num_prt_users, 0);
2495
2496         /* If not overridden by the user, by default, only in large BAR systems
2497          * Compute VM tables will be updated by CPU
2498          */
2499 #ifdef CONFIG_X86_64
2500         if (amdgpu_vm_update_mode == -1) {
2501                 if (amdgpu_vm_is_large_bar(adev))
2502                         adev->vm_manager.vm_update_mode =
2503                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2504                 else
2505                         adev->vm_manager.vm_update_mode = 0;
2506         } else
2507                 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2508 #else
2509         adev->vm_manager.vm_update_mode = 0;
2510 #endif
2511
2512         idr_init(&adev->vm_manager.pasid_idr);
2513         spin_lock_init(&adev->vm_manager.pasid_lock);
2514 }
2515
2516 /**
2517  * amdgpu_vm_manager_fini - cleanup VM manager
2518  *
2519  * @adev: amdgpu_device pointer
2520  *
2521  * Cleanup the VM manager and free resources.
2522  */
2523 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2524 {
2525         WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
2526         idr_destroy(&adev->vm_manager.pasid_idr);
2527
2528         amdgpu_vmid_mgr_fini(adev);
2529 }
2530
2531 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2532 {
2533         union drm_amdgpu_vm *args = data;
2534         struct amdgpu_device *adev = dev->dev_private;
2535         struct amdgpu_fpriv *fpriv = filp->driver_priv;
2536         int r;
2537
2538         switch (args->in.op) {
2539         case AMDGPU_VM_OP_RESERVE_VMID:
2540                 /* current, we only have requirement to reserve vmid from gfxhub */
2541                 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
2542                 if (r)
2543                         return r;
2544                 break;
2545         case AMDGPU_VM_OP_UNRESERVE_VMID:
2546                 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
2547                 break;
2548         default:
2549                 return -EINVAL;
2550         }
2551
2552         return 0;
2553 }