]> Git Repo - linux.git/blob - drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
drm/amdgpu: add amdgpu_vm_update_func
[linux.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_vm.c
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/dma-fence-array.h>
29 #include <linux/interval_tree_generic.h>
30 #include <linux/idr.h>
31 #include <drm/drmP.h>
32 #include <drm/amdgpu_drm.h>
33 #include "amdgpu.h"
34 #include "amdgpu_trace.h"
35 #include "amdgpu_amdkfd.h"
36 #include "amdgpu_gmc.h"
37
38 /**
39  * DOC: GPUVM
40  *
41  * GPUVM is similar to the legacy gart on older asics, however
42  * rather than there being a single global gart table
43  * for the entire GPU, there are multiple VM page tables active
44  * at any given time.  The VM page tables can contain a mix
45  * vram pages and system memory pages and system memory pages
46  * can be mapped as snooped (cached system pages) or unsnooped
47  * (uncached system pages).
48  * Each VM has an ID associated with it and there is a page table
49  * associated with each VMID.  When execting a command buffer,
50  * the kernel tells the the ring what VMID to use for that command
51  * buffer.  VMIDs are allocated dynamically as commands are submitted.
52  * The userspace drivers maintain their own address space and the kernel
53  * sets up their pages tables accordingly when they submit their
54  * command buffers and a VMID is assigned.
55  * Cayman/Trinity support up to 8 active VMs at any given time;
56  * SI supports 16.
57  */
58
59 #define START(node) ((node)->start)
60 #define LAST(node) ((node)->last)
61
62 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
63                      START, LAST, static, amdgpu_vm_it)
64
65 #undef START
66 #undef LAST
67
68 /**
69  * struct amdgpu_pte_update_params - Local structure
70  *
71  * Encapsulate some VM table update parameters to reduce
72  * the number of function parameters
73  *
74  */
75 struct amdgpu_pte_update_params {
76
77         /**
78          * @adev: amdgpu device we do this update for
79          */
80         struct amdgpu_device *adev;
81
82         /**
83          * @vm: optional amdgpu_vm we do this update for
84          */
85         struct amdgpu_vm *vm;
86
87         /**
88          * @src: address where to copy page table entries from
89          */
90         uint64_t src;
91
92         /**
93          * @ib: indirect buffer to fill with commands
94          */
95         struct amdgpu_ib *ib;
96
97         /**
98          * @func: Function which actually does the update
99          */
100         void (*func)(struct amdgpu_pte_update_params *params,
101                      struct amdgpu_bo *bo, uint64_t pe,
102                      uint64_t addr, unsigned count, uint32_t incr,
103                      uint64_t flags);
104         /**
105          * @pages_addr:
106          *
107          * DMA addresses to use for mapping, used during VM update by CPU
108          */
109         dma_addr_t *pages_addr;
110
111         /**
112          * @kptr:
113          *
114          * Kernel pointer of PD/PT BO that needs to be updated,
115          * used during VM update by CPU
116          */
117         void *kptr;
118 };
119
120 /**
121  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
122  */
123 struct amdgpu_prt_cb {
124
125         /**
126          * @adev: amdgpu device
127          */
128         struct amdgpu_device *adev;
129
130         /**
131          * @cb: callback
132          */
133         struct dma_fence_cb cb;
134 };
135
136 /**
137  * amdgpu_vm_level_shift - return the addr shift for each level
138  *
139  * @adev: amdgpu_device pointer
140  * @level: VMPT level
141  *
142  * Returns:
143  * The number of bits the pfn needs to be right shifted for a level.
144  */
145 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
146                                       unsigned level)
147 {
148         unsigned shift = 0xff;
149
150         switch (level) {
151         case AMDGPU_VM_PDB2:
152         case AMDGPU_VM_PDB1:
153         case AMDGPU_VM_PDB0:
154                 shift = 9 * (AMDGPU_VM_PDB0 - level) +
155                         adev->vm_manager.block_size;
156                 break;
157         case AMDGPU_VM_PTB:
158                 shift = 0;
159                 break;
160         default:
161                 dev_err(adev->dev, "the level%d isn't supported.\n", level);
162         }
163
164         return shift;
165 }
166
167 /**
168  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
169  *
170  * @adev: amdgpu_device pointer
171  * @level: VMPT level
172  *
173  * Returns:
174  * The number of entries in a page directory or page table.
175  */
176 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
177                                       unsigned level)
178 {
179         unsigned shift = amdgpu_vm_level_shift(adev,
180                                                adev->vm_manager.root_level);
181
182         if (level == adev->vm_manager.root_level)
183                 /* For the root directory */
184                 return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
185         else if (level != AMDGPU_VM_PTB)
186                 /* Everything in between */
187                 return 512;
188         else
189                 /* For the page tables on the leaves */
190                 return AMDGPU_VM_PTE_COUNT(adev);
191 }
192
193 /**
194  * amdgpu_vm_bo_size - returns the size of the BOs in bytes
195  *
196  * @adev: amdgpu_device pointer
197  * @level: VMPT level
198  *
199  * Returns:
200  * The size of the BO for a page directory or page table in bytes.
201  */
202 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
203 {
204         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
205 }
206
207 /**
208  * amdgpu_vm_bo_evicted - vm_bo is evicted
209  *
210  * @vm_bo: vm_bo which is evicted
211  *
212  * State for PDs/PTs and per VM BOs which are not at the location they should
213  * be.
214  */
215 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
216 {
217         struct amdgpu_vm *vm = vm_bo->vm;
218         struct amdgpu_bo *bo = vm_bo->bo;
219
220         vm_bo->moved = true;
221         if (bo->tbo.type == ttm_bo_type_kernel)
222                 list_move(&vm_bo->vm_status, &vm->evicted);
223         else
224                 list_move_tail(&vm_bo->vm_status, &vm->evicted);
225 }
226
227 /**
228  * amdgpu_vm_bo_relocated - vm_bo is reloacted
229  *
230  * @vm_bo: vm_bo which is relocated
231  *
232  * State for PDs/PTs which needs to update their parent PD.
233  */
234 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
235 {
236         list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
237 }
238
239 /**
240  * amdgpu_vm_bo_moved - vm_bo is moved
241  *
242  * @vm_bo: vm_bo which is moved
243  *
244  * State for per VM BOs which are moved, but that change is not yet reflected
245  * in the page tables.
246  */
247 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
248 {
249         list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
250 }
251
252 /**
253  * amdgpu_vm_bo_idle - vm_bo is idle
254  *
255  * @vm_bo: vm_bo which is now idle
256  *
257  * State for PDs/PTs and per VM BOs which have gone through the state machine
258  * and are now idle.
259  */
260 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
261 {
262         list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
263         vm_bo->moved = false;
264 }
265
266 /**
267  * amdgpu_vm_bo_invalidated - vm_bo is invalidated
268  *
269  * @vm_bo: vm_bo which is now invalidated
270  *
271  * State for normal BOs which are invalidated and that change not yet reflected
272  * in the PTs.
273  */
274 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
275 {
276         spin_lock(&vm_bo->vm->invalidated_lock);
277         list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
278         spin_unlock(&vm_bo->vm->invalidated_lock);
279 }
280
281 /**
282  * amdgpu_vm_bo_done - vm_bo is done
283  *
284  * @vm_bo: vm_bo which is now done
285  *
286  * State for normal BOs which are invalidated and that change has been updated
287  * in the PTs.
288  */
289 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
290 {
291         spin_lock(&vm_bo->vm->invalidated_lock);
292         list_del_init(&vm_bo->vm_status);
293         spin_unlock(&vm_bo->vm->invalidated_lock);
294 }
295
296 /**
297  * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
298  *
299  * @base: base structure for tracking BO usage in a VM
300  * @vm: vm to which bo is to be added
301  * @bo: amdgpu buffer object
302  *
303  * Initialize a bo_va_base structure and add it to the appropriate lists
304  *
305  */
306 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
307                                    struct amdgpu_vm *vm,
308                                    struct amdgpu_bo *bo)
309 {
310         base->vm = vm;
311         base->bo = bo;
312         INIT_LIST_HEAD(&base->bo_list);
313         INIT_LIST_HEAD(&base->vm_status);
314
315         if (!bo)
316                 return;
317         list_add_tail(&base->bo_list, &bo->va);
318
319         if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
320                 return;
321
322         vm->bulk_moveable = false;
323         if (bo->tbo.type == ttm_bo_type_kernel)
324                 amdgpu_vm_bo_relocated(base);
325         else
326                 amdgpu_vm_bo_idle(base);
327
328         if (bo->preferred_domains &
329             amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
330                 return;
331
332         /*
333          * we checked all the prerequisites, but it looks like this per vm bo
334          * is currently evicted. add the bo to the evicted list to make sure it
335          * is validated on next vm use to avoid fault.
336          * */
337         amdgpu_vm_bo_evicted(base);
338 }
339
340 /**
341  * amdgpu_vm_pt_parent - get the parent page directory
342  *
343  * @pt: child page table
344  *
345  * Helper to get the parent entry for the child page table. NULL if we are at
346  * the root page directory.
347  */
348 static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt)
349 {
350         struct amdgpu_bo *parent = pt->base.bo->parent;
351
352         if (!parent)
353                 return NULL;
354
355         return list_first_entry(&parent->va, struct amdgpu_vm_pt, base.bo_list);
356 }
357
358 /**
359  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
360  *
361  * @vm: vm providing the BOs
362  * @validated: head of validation list
363  * @entry: entry to add
364  *
365  * Add the page directory to the list of BOs to
366  * validate for command submission.
367  */
368 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
369                          struct list_head *validated,
370                          struct amdgpu_bo_list_entry *entry)
371 {
372         entry->robj = vm->root.base.bo;
373         entry->priority = 0;
374         entry->tv.bo = &entry->robj->tbo;
375         entry->tv.shared = true;
376         entry->user_pages = NULL;
377         list_add(&entry->tv.head, validated);
378 }
379
380 /**
381  * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
382  *
383  * @adev: amdgpu device pointer
384  * @vm: vm providing the BOs
385  *
386  * Move all BOs to the end of LRU and remember their positions to put them
387  * together.
388  */
389 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
390                                 struct amdgpu_vm *vm)
391 {
392         struct ttm_bo_global *glob = adev->mman.bdev.glob;
393         struct amdgpu_vm_bo_base *bo_base;
394
395         if (vm->bulk_moveable) {
396                 spin_lock(&glob->lru_lock);
397                 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
398                 spin_unlock(&glob->lru_lock);
399                 return;
400         }
401
402         memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
403
404         spin_lock(&glob->lru_lock);
405         list_for_each_entry(bo_base, &vm->idle, vm_status) {
406                 struct amdgpu_bo *bo = bo_base->bo;
407
408                 if (!bo->parent)
409                         continue;
410
411                 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);
412                 if (bo->shadow)
413                         ttm_bo_move_to_lru_tail(&bo->shadow->tbo,
414                                                 &vm->lru_bulk_move);
415         }
416         spin_unlock(&glob->lru_lock);
417
418         vm->bulk_moveable = true;
419 }
420
421 /**
422  * amdgpu_vm_validate_pt_bos - validate the page table BOs
423  *
424  * @adev: amdgpu device pointer
425  * @vm: vm providing the BOs
426  * @validate: callback to do the validation
427  * @param: parameter for the validation callback
428  *
429  * Validate the page table BOs on command submission if neccessary.
430  *
431  * Returns:
432  * Validation result.
433  */
434 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
435                               int (*validate)(void *p, struct amdgpu_bo *bo),
436                               void *param)
437 {
438         struct amdgpu_vm_bo_base *bo_base, *tmp;
439         int r = 0;
440
441         vm->bulk_moveable &= list_empty(&vm->evicted);
442
443         list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
444                 struct amdgpu_bo *bo = bo_base->bo;
445
446                 r = validate(param, bo);
447                 if (r)
448                         break;
449
450                 if (bo->tbo.type != ttm_bo_type_kernel) {
451                         amdgpu_vm_bo_moved(bo_base);
452                 } else {
453                         if (vm->use_cpu_for_update)
454                                 r = amdgpu_bo_kmap(bo, NULL);
455                         else
456                                 r = amdgpu_ttm_alloc_gart(&bo->tbo);
457                         if (r)
458                                 break;
459                         if (bo->shadow) {
460                                 r = amdgpu_ttm_alloc_gart(&bo->shadow->tbo);
461                                 if (r)
462                                         break;
463                         }
464                         amdgpu_vm_bo_relocated(bo_base);
465                 }
466         }
467
468         return r;
469 }
470
471 /**
472  * amdgpu_vm_ready - check VM is ready for updates
473  *
474  * @vm: VM to check
475  *
476  * Check if all VM PDs/PTs are ready for updates
477  *
478  * Returns:
479  * True if eviction list is empty.
480  */
481 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
482 {
483         return list_empty(&vm->evicted);
484 }
485
486 /**
487  * amdgpu_vm_clear_bo - initially clear the PDs/PTs
488  *
489  * @adev: amdgpu_device pointer
490  * @vm: VM to clear BO from
491  * @bo: BO to clear
492  * @level: level this BO is at
493  * @pte_support_ats: indicate ATS support from PTE
494  *
495  * Root PD needs to be reserved when calling this.
496  *
497  * Returns:
498  * 0 on success, errno otherwise.
499  */
500 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
501                               struct amdgpu_vm *vm, struct amdgpu_bo *bo,
502                               unsigned level, bool pte_support_ats)
503 {
504         struct ttm_operation_ctx ctx = { true, false };
505         struct dma_fence *fence = NULL;
506         unsigned entries, ats_entries;
507         struct amdgpu_ring *ring;
508         struct amdgpu_job *job;
509         uint64_t addr;
510         int r;
511
512         entries = amdgpu_bo_size(bo) / 8;
513
514         if (pte_support_ats) {
515                 if (level == adev->vm_manager.root_level) {
516                         ats_entries = amdgpu_vm_level_shift(adev, level);
517                         ats_entries += AMDGPU_GPU_PAGE_SHIFT;
518                         ats_entries = AMDGPU_GMC_HOLE_START >> ats_entries;
519                         ats_entries = min(ats_entries, entries);
520                         entries -= ats_entries;
521                 } else {
522                         ats_entries = entries;
523                         entries = 0;
524                 }
525         } else {
526                 ats_entries = 0;
527         }
528
529         ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
530
531         r = reservation_object_reserve_shared(bo->tbo.resv);
532         if (r)
533                 return r;
534
535         r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
536         if (r)
537                 goto error;
538
539         r = amdgpu_ttm_alloc_gart(&bo->tbo);
540         if (r)
541                 return r;
542
543         r = amdgpu_job_alloc_with_ib(adev, 64, &job);
544         if (r)
545                 goto error;
546
547         addr = amdgpu_bo_gpu_offset(bo);
548         if (ats_entries) {
549                 uint64_t ats_value;
550
551                 ats_value = AMDGPU_PTE_DEFAULT_ATC;
552                 if (level != AMDGPU_VM_PTB)
553                         ats_value |= AMDGPU_PDE_PTE;
554
555                 amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
556                                       ats_entries, 0, ats_value);
557                 addr += ats_entries * 8;
558         }
559
560         if (entries)
561                 amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
562                                       entries, 0, 0);
563
564         amdgpu_ring_pad_ib(ring, &job->ibs[0]);
565
566         WARN_ON(job->ibs[0].length_dw > 64);
567         r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv,
568                              AMDGPU_FENCE_OWNER_UNDEFINED, false);
569         if (r)
570                 goto error_free;
571
572         r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_UNDEFINED,
573                               &fence);
574         if (r)
575                 goto error_free;
576
577         amdgpu_bo_fence(bo, fence, true);
578         dma_fence_put(fence);
579
580         if (bo->shadow)
581                 return amdgpu_vm_clear_bo(adev, vm, bo->shadow,
582                                           level, pte_support_ats);
583
584         return 0;
585
586 error_free:
587         amdgpu_job_free(job);
588
589 error:
590         return r;
591 }
592
593 /**
594  * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation
595  *
596  * @adev: amdgpu_device pointer
597  * @vm: requesting vm
598  * @bp: resulting BO allocation parameters
599  */
600 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
601                                int level, struct amdgpu_bo_param *bp)
602 {
603         memset(bp, 0, sizeof(*bp));
604
605         bp->size = amdgpu_vm_bo_size(adev, level);
606         bp->byte_align = AMDGPU_GPU_PAGE_SIZE;
607         bp->domain = AMDGPU_GEM_DOMAIN_VRAM;
608         if (bp->size <= PAGE_SIZE && adev->asic_type >= CHIP_VEGA10 &&
609             adev->flags & AMD_IS_APU)
610                 bp->domain |= AMDGPU_GEM_DOMAIN_GTT;
611         bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);
612         bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
613                 AMDGPU_GEM_CREATE_CPU_GTT_USWC;
614         if (vm->use_cpu_for_update)
615                 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
616         else if (!vm->root.base.bo || vm->root.base.bo->shadow)
617                 bp->flags |= AMDGPU_GEM_CREATE_SHADOW;
618         bp->type = ttm_bo_type_kernel;
619         if (vm->root.base.bo)
620                 bp->resv = vm->root.base.bo->tbo.resv;
621 }
622
623 /**
624  * amdgpu_vm_alloc_levels - allocate the PD/PT levels
625  *
626  * @adev: amdgpu_device pointer
627  * @vm: requested vm
628  * @parent: parent PT
629  * @saddr: start of the address range
630  * @eaddr: end of the address range
631  * @level: VMPT level
632  * @ats: indicate ATS support from PTE
633  *
634  * Make sure the page directories and page tables are allocated
635  *
636  * Returns:
637  * 0 on success, errno otherwise.
638  */
639 static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
640                                   struct amdgpu_vm *vm,
641                                   struct amdgpu_vm_pt *parent,
642                                   uint64_t saddr, uint64_t eaddr,
643                                   unsigned level, bool ats)
644 {
645         unsigned shift = amdgpu_vm_level_shift(adev, level);
646         struct amdgpu_bo_param bp;
647         unsigned pt_idx, from, to;
648         int r;
649
650         if (!parent->entries) {
651                 unsigned num_entries = amdgpu_vm_num_entries(adev, level);
652
653                 parent->entries = kvmalloc_array(num_entries,
654                                                    sizeof(struct amdgpu_vm_pt),
655                                                    GFP_KERNEL | __GFP_ZERO);
656                 if (!parent->entries)
657                         return -ENOMEM;
658         }
659
660         from = saddr >> shift;
661         to = eaddr >> shift;
662         if (from >= amdgpu_vm_num_entries(adev, level) ||
663             to >= amdgpu_vm_num_entries(adev, level))
664                 return -EINVAL;
665
666         ++level;
667         saddr = saddr & ((1 << shift) - 1);
668         eaddr = eaddr & ((1 << shift) - 1);
669
670         amdgpu_vm_bo_param(adev, vm, level, &bp);
671
672         /* walk over the address space and allocate the page tables */
673         for (pt_idx = from; pt_idx <= to; ++pt_idx) {
674                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
675                 struct amdgpu_bo *pt;
676
677                 if (!entry->base.bo) {
678                         r = amdgpu_bo_create(adev, &bp, &pt);
679                         if (r)
680                                 return r;
681
682                         r = amdgpu_vm_clear_bo(adev, vm, pt, level, ats);
683                         if (r) {
684                                 amdgpu_bo_unref(&pt->shadow);
685                                 amdgpu_bo_unref(&pt);
686                                 return r;
687                         }
688
689                         if (vm->use_cpu_for_update) {
690                                 r = amdgpu_bo_kmap(pt, NULL);
691                                 if (r) {
692                                         amdgpu_bo_unref(&pt->shadow);
693                                         amdgpu_bo_unref(&pt);
694                                         return r;
695                                 }
696                         }
697
698                         /* Keep a reference to the root directory to avoid
699                         * freeing them up in the wrong order.
700                         */
701                         pt->parent = amdgpu_bo_ref(parent->base.bo);
702
703                         amdgpu_vm_bo_base_init(&entry->base, vm, pt);
704                 }
705
706                 if (level < AMDGPU_VM_PTB) {
707                         uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
708                         uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
709                                 ((1 << shift) - 1);
710                         r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
711                                                    sub_eaddr, level, ats);
712                         if (r)
713                                 return r;
714                 }
715         }
716
717         return 0;
718 }
719
720 /**
721  * amdgpu_vm_alloc_pts - Allocate page tables.
722  *
723  * @adev: amdgpu_device pointer
724  * @vm: VM to allocate page tables for
725  * @saddr: Start address which needs to be allocated
726  * @size: Size from start address we need.
727  *
728  * Make sure the page tables are allocated.
729  *
730  * Returns:
731  * 0 on success, errno otherwise.
732  */
733 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
734                         struct amdgpu_vm *vm,
735                         uint64_t saddr, uint64_t size)
736 {
737         uint64_t eaddr;
738         bool ats = false;
739
740         /* validate the parameters */
741         if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
742                 return -EINVAL;
743
744         eaddr = saddr + size - 1;
745
746         if (vm->pte_support_ats)
747                 ats = saddr < AMDGPU_GMC_HOLE_START;
748
749         saddr /= AMDGPU_GPU_PAGE_SIZE;
750         eaddr /= AMDGPU_GPU_PAGE_SIZE;
751
752         if (eaddr >= adev->vm_manager.max_pfn) {
753                 dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
754                         eaddr, adev->vm_manager.max_pfn);
755                 return -EINVAL;
756         }
757
758         return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr,
759                                       adev->vm_manager.root_level, ats);
760 }
761
762 /**
763  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
764  *
765  * @adev: amdgpu_device pointer
766  */
767 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
768 {
769         const struct amdgpu_ip_block *ip_block;
770         bool has_compute_vm_bug;
771         struct amdgpu_ring *ring;
772         int i;
773
774         has_compute_vm_bug = false;
775
776         ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
777         if (ip_block) {
778                 /* Compute has a VM bug for GFX version < 7.
779                    Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
780                 if (ip_block->version->major <= 7)
781                         has_compute_vm_bug = true;
782                 else if (ip_block->version->major == 8)
783                         if (adev->gfx.mec_fw_version < 673)
784                                 has_compute_vm_bug = true;
785         }
786
787         for (i = 0; i < adev->num_rings; i++) {
788                 ring = adev->rings[i];
789                 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
790                         /* only compute rings */
791                         ring->has_compute_vm_bug = has_compute_vm_bug;
792                 else
793                         ring->has_compute_vm_bug = false;
794         }
795 }
796
797 /**
798  * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
799  *
800  * @ring: ring on which the job will be submitted
801  * @job: job to submit
802  *
803  * Returns:
804  * True if sync is needed.
805  */
806 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
807                                   struct amdgpu_job *job)
808 {
809         struct amdgpu_device *adev = ring->adev;
810         unsigned vmhub = ring->funcs->vmhub;
811         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
812         struct amdgpu_vmid *id;
813         bool gds_switch_needed;
814         bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
815
816         if (job->vmid == 0)
817                 return false;
818         id = &id_mgr->ids[job->vmid];
819         gds_switch_needed = ring->funcs->emit_gds_switch && (
820                 id->gds_base != job->gds_base ||
821                 id->gds_size != job->gds_size ||
822                 id->gws_base != job->gws_base ||
823                 id->gws_size != job->gws_size ||
824                 id->oa_base != job->oa_base ||
825                 id->oa_size != job->oa_size);
826
827         if (amdgpu_vmid_had_gpu_reset(adev, id))
828                 return true;
829
830         return vm_flush_needed || gds_switch_needed;
831 }
832
833 /**
834  * amdgpu_vm_flush - hardware flush the vm
835  *
836  * @ring: ring to use for flush
837  * @job:  related job
838  * @need_pipe_sync: is pipe sync needed
839  *
840  * Emit a VM flush when it is necessary.
841  *
842  * Returns:
843  * 0 on success, errno otherwise.
844  */
845 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
846 {
847         struct amdgpu_device *adev = ring->adev;
848         unsigned vmhub = ring->funcs->vmhub;
849         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
850         struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
851         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
852                 id->gds_base != job->gds_base ||
853                 id->gds_size != job->gds_size ||
854                 id->gws_base != job->gws_base ||
855                 id->gws_size != job->gws_size ||
856                 id->oa_base != job->oa_base ||
857                 id->oa_size != job->oa_size);
858         bool vm_flush_needed = job->vm_needs_flush;
859         bool pasid_mapping_needed = id->pasid != job->pasid ||
860                 !id->pasid_mapping ||
861                 !dma_fence_is_signaled(id->pasid_mapping);
862         struct dma_fence *fence = NULL;
863         unsigned patch_offset = 0;
864         int r;
865
866         if (amdgpu_vmid_had_gpu_reset(adev, id)) {
867                 gds_switch_needed = true;
868                 vm_flush_needed = true;
869                 pasid_mapping_needed = true;
870         }
871
872         gds_switch_needed &= !!ring->funcs->emit_gds_switch;
873         vm_flush_needed &= !!ring->funcs->emit_vm_flush;
874         pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
875                 ring->funcs->emit_wreg;
876
877         if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
878                 return 0;
879
880         if (ring->funcs->init_cond_exec)
881                 patch_offset = amdgpu_ring_init_cond_exec(ring);
882
883         if (need_pipe_sync)
884                 amdgpu_ring_emit_pipeline_sync(ring);
885
886         if (vm_flush_needed) {
887                 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
888                 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
889         }
890
891         if (pasid_mapping_needed)
892                 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
893
894         if (vm_flush_needed || pasid_mapping_needed) {
895                 r = amdgpu_fence_emit(ring, &fence, 0);
896                 if (r)
897                         return r;
898         }
899
900         if (vm_flush_needed) {
901                 mutex_lock(&id_mgr->lock);
902                 dma_fence_put(id->last_flush);
903                 id->last_flush = dma_fence_get(fence);
904                 id->current_gpu_reset_count =
905                         atomic_read(&adev->gpu_reset_counter);
906                 mutex_unlock(&id_mgr->lock);
907         }
908
909         if (pasid_mapping_needed) {
910                 id->pasid = job->pasid;
911                 dma_fence_put(id->pasid_mapping);
912                 id->pasid_mapping = dma_fence_get(fence);
913         }
914         dma_fence_put(fence);
915
916         if (ring->funcs->emit_gds_switch && gds_switch_needed) {
917                 id->gds_base = job->gds_base;
918                 id->gds_size = job->gds_size;
919                 id->gws_base = job->gws_base;
920                 id->gws_size = job->gws_size;
921                 id->oa_base = job->oa_base;
922                 id->oa_size = job->oa_size;
923                 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
924                                             job->gds_size, job->gws_base,
925                                             job->gws_size, job->oa_base,
926                                             job->oa_size);
927         }
928
929         if (ring->funcs->patch_cond_exec)
930                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
931
932         /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
933         if (ring->funcs->emit_switch_buffer) {
934                 amdgpu_ring_emit_switch_buffer(ring);
935                 amdgpu_ring_emit_switch_buffer(ring);
936         }
937         return 0;
938 }
939
940 /**
941  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
942  *
943  * @vm: requested vm
944  * @bo: requested buffer object
945  *
946  * Find @bo inside the requested vm.
947  * Search inside the @bos vm list for the requested vm
948  * Returns the found bo_va or NULL if none is found
949  *
950  * Object has to be reserved!
951  *
952  * Returns:
953  * Found bo_va or NULL.
954  */
955 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
956                                        struct amdgpu_bo *bo)
957 {
958         struct amdgpu_bo_va *bo_va;
959
960         list_for_each_entry(bo_va, &bo->va, base.bo_list) {
961                 if (bo_va->base.vm == vm) {
962                         return bo_va;
963                 }
964         }
965         return NULL;
966 }
967
968 /**
969  * amdgpu_vm_do_set_ptes - helper to call the right asic function
970  *
971  * @params: see amdgpu_pte_update_params definition
972  * @bo: PD/PT to update
973  * @pe: addr of the page entry
974  * @addr: dst addr to write into pe
975  * @count: number of page entries to update
976  * @incr: increase next addr by incr bytes
977  * @flags: hw access flags
978  *
979  * Traces the parameters and calls the right asic functions
980  * to setup the page table using the DMA.
981  */
982 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
983                                   struct amdgpu_bo *bo,
984                                   uint64_t pe, uint64_t addr,
985                                   unsigned count, uint32_t incr,
986                                   uint64_t flags)
987 {
988         pe += amdgpu_bo_gpu_offset(bo);
989         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
990
991         if (count < 3) {
992                 amdgpu_vm_write_pte(params->adev, params->ib, pe,
993                                     addr | flags, count, incr);
994
995         } else {
996                 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
997                                       count, incr, flags);
998         }
999 }
1000
1001 /**
1002  * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
1003  *
1004  * @params: see amdgpu_pte_update_params definition
1005  * @bo: PD/PT to update
1006  * @pe: addr of the page entry
1007  * @addr: dst addr to write into pe
1008  * @count: number of page entries to update
1009  * @incr: increase next addr by incr bytes
1010  * @flags: hw access flags
1011  *
1012  * Traces the parameters and calls the DMA function to copy the PTEs.
1013  */
1014 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
1015                                    struct amdgpu_bo *bo,
1016                                    uint64_t pe, uint64_t addr,
1017                                    unsigned count, uint32_t incr,
1018                                    uint64_t flags)
1019 {
1020         uint64_t src = (params->src + (addr >> 12) * 8);
1021
1022         pe += amdgpu_bo_gpu_offset(bo);
1023         trace_amdgpu_vm_copy_ptes(pe, src, count);
1024
1025         amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
1026 }
1027
1028 /**
1029  * amdgpu_vm_map_gart - Resolve gart mapping of addr
1030  *
1031  * @pages_addr: optional DMA address to use for lookup
1032  * @addr: the unmapped addr
1033  *
1034  * Look up the physical address of the page that the pte resolves
1035  * to.
1036  *
1037  * Returns:
1038  * The pointer for the page table entry.
1039  */
1040 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
1041 {
1042         uint64_t result;
1043
1044         /* page table offset */
1045         result = pages_addr[addr >> PAGE_SHIFT];
1046
1047         /* in case cpu page size != gpu page size*/
1048         result |= addr & (~PAGE_MASK);
1049
1050         result &= 0xFFFFFFFFFFFFF000ULL;
1051
1052         return result;
1053 }
1054
1055 /**
1056  * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
1057  *
1058  * @params: see amdgpu_pte_update_params definition
1059  * @bo: PD/PT to update
1060  * @pe: kmap addr of the page entry
1061  * @addr: dst addr to write into pe
1062  * @count: number of page entries to update
1063  * @incr: increase next addr by incr bytes
1064  * @flags: hw access flags
1065  *
1066  * Write count number of PT/PD entries directly.
1067  */
1068 static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
1069                                    struct amdgpu_bo *bo,
1070                                    uint64_t pe, uint64_t addr,
1071                                    unsigned count, uint32_t incr,
1072                                    uint64_t flags)
1073 {
1074         unsigned int i;
1075         uint64_t value;
1076
1077         pe += (unsigned long)amdgpu_bo_kptr(bo);
1078
1079         trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
1080
1081         for (i = 0; i < count; i++) {
1082                 value = params->pages_addr ?
1083                         amdgpu_vm_map_gart(params->pages_addr, addr) :
1084                         addr;
1085                 amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
1086                                        i, value, flags);
1087                 addr += incr;
1088         }
1089 }
1090
1091
1092 /**
1093  * amdgpu_vm_wait_pd - Wait for PT BOs to be free.
1094  *
1095  * @adev: amdgpu_device pointer
1096  * @vm: related vm
1097  * @owner: fence owner
1098  *
1099  * Returns:
1100  * 0 on success, errno otherwise.
1101  */
1102 static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1103                              void *owner)
1104 {
1105         struct amdgpu_sync sync;
1106         int r;
1107
1108         amdgpu_sync_create(&sync);
1109         amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
1110         r = amdgpu_sync_wait(&sync, true);
1111         amdgpu_sync_free(&sync);
1112
1113         return r;
1114 }
1115
1116 /**
1117  * amdgpu_vm_update_func - helper to call update function
1118  *
1119  * Calls the update function for both the given BO as well as its shadow.
1120  */
1121 static void amdgpu_vm_update_func(struct amdgpu_pte_update_params *params,
1122                                   struct amdgpu_bo *bo,
1123                                   uint64_t pe, uint64_t addr,
1124                                   unsigned count, uint32_t incr,
1125                                   uint64_t flags)
1126 {
1127         if (bo->shadow)
1128                 params->func(params, bo->shadow, pe, addr, count, incr, flags);
1129         params->func(params, bo, pe, addr, count, incr, flags);
1130 }
1131
1132 /*
1133  * amdgpu_vm_update_pde - update a single level in the hierarchy
1134  *
1135  * @param: parameters for the update
1136  * @vm: requested vm
1137  * @parent: parent directory
1138  * @entry: entry to update
1139  *
1140  * Makes sure the requested entry in parent is up to date.
1141  */
1142 static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
1143                                  struct amdgpu_vm *vm,
1144                                  struct amdgpu_vm_pt *parent,
1145                                  struct amdgpu_vm_pt *entry)
1146 {
1147         struct amdgpu_bo *bo = parent->base.bo, *pbo;
1148         uint64_t pde, pt, flags;
1149         unsigned level;
1150
1151         /* Don't update huge pages here */
1152         if (entry->huge)
1153                 return;
1154
1155         for (level = 0, pbo = bo->parent; pbo; ++level)
1156                 pbo = pbo->parent;
1157
1158         level += params->adev->vm_manager.root_level;
1159         amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
1160         pde = (entry - parent->entries) * 8;
1161         amdgpu_vm_update_func(params, bo, pde, pt, 1, 0, flags);
1162 }
1163
1164 /*
1165  * amdgpu_vm_invalidate_level - mark all PD levels as invalid
1166  *
1167  * @adev: amdgpu_device pointer
1168  * @vm: related vm
1169  * @parent: parent PD
1170  * @level: VMPT level
1171  *
1172  * Mark all PD level as invalid after an error.
1173  */
1174 static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
1175                                        struct amdgpu_vm *vm,
1176                                        struct amdgpu_vm_pt *parent,
1177                                        unsigned level)
1178 {
1179         unsigned pt_idx, num_entries;
1180
1181         /*
1182          * Recurse into the subdirectories. This recursion is harmless because
1183          * we only have a maximum of 5 layers.
1184          */
1185         num_entries = amdgpu_vm_num_entries(adev, level);
1186         for (pt_idx = 0; pt_idx < num_entries; ++pt_idx) {
1187                 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
1188
1189                 if (!entry->base.bo)
1190                         continue;
1191
1192                 if (!entry->base.moved)
1193                         amdgpu_vm_bo_relocated(&entry->base);
1194                 amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
1195         }
1196 }
1197
1198 /*
1199  * amdgpu_vm_update_directories - make sure that all directories are valid
1200  *
1201  * @adev: amdgpu_device pointer
1202  * @vm: requested vm
1203  *
1204  * Makes sure all directories are up to date.
1205  *
1206  * Returns:
1207  * 0 for success, error for failure.
1208  */
1209 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
1210                                  struct amdgpu_vm *vm)
1211 {
1212         struct amdgpu_pte_update_params params;
1213         struct amdgpu_job *job;
1214         unsigned ndw = 0;
1215         int r = 0;
1216
1217         if (list_empty(&vm->relocated))
1218                 return 0;
1219
1220 restart:
1221         memset(&params, 0, sizeof(params));
1222         params.adev = adev;
1223
1224         if (vm->use_cpu_for_update) {
1225                 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
1226                 if (unlikely(r))
1227                         return r;
1228
1229                 params.func = amdgpu_vm_cpu_set_ptes;
1230         } else {
1231                 ndw = 512 * 8;
1232                 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1233                 if (r)
1234                         return r;
1235
1236                 params.ib = &job->ibs[0];
1237                 params.func = amdgpu_vm_do_set_ptes;
1238         }
1239
1240         while (!list_empty(&vm->relocated)) {
1241                 struct amdgpu_vm_pt *pt, *entry;
1242
1243                 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
1244                                          base.vm_status);
1245                 amdgpu_vm_bo_idle(&entry->base);
1246
1247                 pt = amdgpu_vm_pt_parent(entry);
1248                 if (!pt)
1249                         continue;
1250
1251                 amdgpu_vm_update_pde(&params, vm, pt, entry);
1252
1253                 if (!vm->use_cpu_for_update &&
1254                     (ndw - params.ib->length_dw) < 32)
1255                         break;
1256         }
1257
1258         if (vm->use_cpu_for_update) {
1259                 /* Flush HDP */
1260                 mb();
1261                 amdgpu_asic_flush_hdp(adev, NULL);
1262         } else if (params.ib->length_dw == 0) {
1263                 amdgpu_job_free(job);
1264         } else {
1265                 struct amdgpu_bo *root = vm->root.base.bo;
1266                 struct amdgpu_ring *ring;
1267                 struct dma_fence *fence;
1268
1269                 ring = container_of(vm->entity.rq->sched, struct amdgpu_ring,
1270                                     sched);
1271
1272                 amdgpu_ring_pad_ib(ring, params.ib);
1273                 amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
1274                                  AMDGPU_FENCE_OWNER_VM, false);
1275                 WARN_ON(params.ib->length_dw > ndw);
1276                 r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM,
1277                                       &fence);
1278                 if (r)
1279                         goto error;
1280
1281                 amdgpu_bo_fence(root, fence, true);
1282                 dma_fence_put(vm->last_update);
1283                 vm->last_update = fence;
1284         }
1285
1286         if (!list_empty(&vm->relocated))
1287                 goto restart;
1288
1289         return 0;
1290
1291 error:
1292         amdgpu_vm_invalidate_level(adev, vm, &vm->root,
1293                                    adev->vm_manager.root_level);
1294         amdgpu_job_free(job);
1295         return r;
1296 }
1297
1298 /**
1299  * amdgpu_vm_find_entry - find the entry for an address
1300  *
1301  * @p: see amdgpu_pte_update_params definition
1302  * @addr: virtual address in question
1303  * @entry: resulting entry or NULL
1304  * @parent: parent entry
1305  *
1306  * Find the vm_pt entry and it's parent for the given address.
1307  */
1308 void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
1309                          struct amdgpu_vm_pt **entry,
1310                          struct amdgpu_vm_pt **parent)
1311 {
1312         unsigned level = p->adev->vm_manager.root_level;
1313
1314         *parent = NULL;
1315         *entry = &p->vm->root;
1316         while ((*entry)->entries) {
1317                 unsigned shift = amdgpu_vm_level_shift(p->adev, level++);
1318
1319                 *parent = *entry;
1320                 *entry = &(*entry)->entries[addr >> shift];
1321                 addr &= (1ULL << shift) - 1;
1322         }
1323
1324         if (level != AMDGPU_VM_PTB)
1325                 *entry = NULL;
1326 }
1327
1328 /**
1329  * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
1330  *
1331  * @p: see amdgpu_pte_update_params definition
1332  * @entry: vm_pt entry to check
1333  * @parent: parent entry
1334  * @nptes: number of PTEs updated with this operation
1335  * @dst: destination address where the PTEs should point to
1336  * @flags: access flags fro the PTEs
1337  *
1338  * Check if we can update the PD with a huge page.
1339  */
1340 static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
1341                                         struct amdgpu_vm_pt *entry,
1342                                         struct amdgpu_vm_pt *parent,
1343                                         unsigned nptes, uint64_t dst,
1344                                         uint64_t flags)
1345 {
1346         uint64_t pde;
1347
1348         /* In the case of a mixed PT the PDE must point to it*/
1349         if (p->adev->asic_type >= CHIP_VEGA10 && !p->src &&
1350             nptes == AMDGPU_VM_PTE_COUNT(p->adev)) {
1351                 /* Set the huge page flag to stop scanning at this PDE */
1352                 flags |= AMDGPU_PDE_PTE;
1353         }
1354
1355         if (!(flags & AMDGPU_PDE_PTE)) {
1356                 if (entry->huge) {
1357                         /* Add the entry to the relocated list to update it. */
1358                         entry->huge = false;
1359                         amdgpu_vm_bo_relocated(&entry->base);
1360                 }
1361                 return;
1362         }
1363
1364         entry->huge = true;
1365         amdgpu_gmc_get_vm_pde(p->adev, AMDGPU_VM_PDB0, &dst, &flags);
1366
1367         pde = (entry - parent->entries) * 8;
1368         amdgpu_vm_update_func(p, parent->base.bo, pde, dst, 1, 0, flags);
1369 }
1370
1371 /**
1372  * amdgpu_vm_update_ptes - make sure that page tables are valid
1373  *
1374  * @params: see amdgpu_pte_update_params definition
1375  * @start: start of GPU address range
1376  * @end: end of GPU address range
1377  * @dst: destination address to map to, the next dst inside the function
1378  * @flags: mapping flags
1379  *
1380  * Update the page tables in the range @start - @end.
1381  *
1382  * Returns:
1383  * 0 for success, -EINVAL for failure.
1384  */
1385 static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
1386                                   uint64_t start, uint64_t end,
1387                                   uint64_t dst, uint64_t flags)
1388 {
1389         struct amdgpu_device *adev = params->adev;
1390         const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
1391
1392         uint64_t addr, pe_start;
1393         struct amdgpu_bo *pt;
1394         unsigned nptes;
1395
1396         /* walk over the address space and update the page tables */
1397         for (addr = start; addr < end; addr += nptes,
1398              dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
1399                 struct amdgpu_vm_pt *entry, *parent;
1400
1401                 amdgpu_vm_get_entry(params, addr, &entry, &parent);
1402                 if (!entry)
1403                         return -ENOENT;
1404
1405                 if ((addr & ~mask) == (end & ~mask))
1406                         nptes = end - addr;
1407                 else
1408                         nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
1409
1410                 amdgpu_vm_handle_huge_pages(params, entry, parent,
1411                                             nptes, dst, flags);
1412                 /* We don't need to update PTEs for huge pages */
1413                 if (entry->huge)
1414                         continue;
1415
1416                 pt = entry->base.bo;
1417                 pe_start = (addr & mask) * 8;
1418                 amdgpu_vm_update_func(params, pt, pe_start, dst, nptes,
1419                                       AMDGPU_GPU_PAGE_SIZE, flags);
1420
1421         }
1422
1423         return 0;
1424 }
1425
1426 /*
1427  * amdgpu_vm_frag_ptes - add fragment information to PTEs
1428  *
1429  * @params: see amdgpu_pte_update_params definition
1430  * @vm: requested vm
1431  * @start: first PTE to handle
1432  * @end: last PTE to handle
1433  * @dst: addr those PTEs should point to
1434  * @flags: hw mapping flags
1435  *
1436  * Returns:
1437  * 0 for success, -EINVAL for failure.
1438  */
1439 static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params  *params,
1440                                 uint64_t start, uint64_t end,
1441                                 uint64_t dst, uint64_t flags)
1442 {
1443         /**
1444          * The MC L1 TLB supports variable sized pages, based on a fragment
1445          * field in the PTE. When this field is set to a non-zero value, page
1446          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1447          * flags are considered valid for all PTEs within the fragment range
1448          * and corresponding mappings are assumed to be physically contiguous.
1449          *
1450          * The L1 TLB can store a single PTE for the whole fragment,
1451          * significantly increasing the space available for translation
1452          * caching. This leads to large improvements in throughput when the
1453          * TLB is under pressure.
1454          *
1455          * The L2 TLB distributes small and large fragments into two
1456          * asymmetric partitions. The large fragment cache is significantly
1457          * larger. Thus, we try to use large fragments wherever possible.
1458          * Userspace can support this by aligning virtual base address and
1459          * allocation size to the fragment size.
1460          */
1461         unsigned max_frag = params->adev->vm_manager.fragment_size;
1462         int r;
1463
1464         /* system pages are non continuously */
1465         if (params->src || !(flags & AMDGPU_PTE_VALID))
1466                 return amdgpu_vm_update_ptes(params, start, end, dst, flags);
1467
1468         while (start != end) {
1469                 uint64_t frag_flags, frag_end;
1470                 unsigned frag;
1471
1472                 /* This intentionally wraps around if no bit is set */
1473                 frag = min((unsigned)ffs(start) - 1,
1474                            (unsigned)fls64(end - start) - 1);
1475                 if (frag >= max_frag) {
1476                         frag_flags = AMDGPU_PTE_FRAG(max_frag);
1477                         frag_end = end & ~((1ULL << max_frag) - 1);
1478                 } else {
1479                         frag_flags = AMDGPU_PTE_FRAG(frag);
1480                         frag_end = start + (1 << frag);
1481                 }
1482
1483                 r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
1484                                           flags | frag_flags);
1485                 if (r)
1486                         return r;
1487
1488                 dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
1489                 start = frag_end;
1490         }
1491
1492         return 0;
1493 }
1494
1495 /**
1496  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1497  *
1498  * @adev: amdgpu_device pointer
1499  * @exclusive: fence we need to sync to
1500  * @pages_addr: DMA addresses to use for mapping
1501  * @vm: requested vm
1502  * @start: start of mapped range
1503  * @last: last mapped entry
1504  * @flags: flags for the entries
1505  * @addr: addr to set the area to
1506  * @fence: optional resulting fence
1507  *
1508  * Fill in the page table entries between @start and @last.
1509  *
1510  * Returns:
1511  * 0 for success, -EINVAL for failure.
1512  */
1513 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1514                                        struct dma_fence *exclusive,
1515                                        dma_addr_t *pages_addr,
1516                                        struct amdgpu_vm *vm,
1517                                        uint64_t start, uint64_t last,
1518                                        uint64_t flags, uint64_t addr,
1519                                        struct dma_fence **fence)
1520 {
1521         struct amdgpu_ring *ring;
1522         void *owner = AMDGPU_FENCE_OWNER_VM;
1523         unsigned nptes, ncmds, ndw;
1524         struct amdgpu_job *job;
1525         struct amdgpu_pte_update_params params;
1526         struct dma_fence *f = NULL;
1527         int r;
1528
1529         memset(&params, 0, sizeof(params));
1530         params.adev = adev;
1531         params.vm = vm;
1532
1533         /* sync to everything on unmapping */
1534         if (!(flags & AMDGPU_PTE_VALID))
1535                 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
1536
1537         if (vm->use_cpu_for_update) {
1538                 /* params.src is used as flag to indicate system Memory */
1539                 if (pages_addr)
1540                         params.src = ~0;
1541
1542                 /* Wait for PT BOs to be free. PTs share the same resv. object
1543                  * as the root PD BO
1544                  */
1545                 r = amdgpu_vm_wait_pd(adev, vm, owner);
1546                 if (unlikely(r))
1547                         return r;
1548
1549                 params.func = amdgpu_vm_cpu_set_ptes;
1550                 params.pages_addr = pages_addr;
1551                 return amdgpu_vm_frag_ptes(&params, start, last + 1,
1552                                            addr, flags);
1553         }
1554
1555         ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
1556
1557         nptes = last - start + 1;
1558
1559         /*
1560          * reserve space for two commands every (1 << BLOCK_SIZE)
1561          *  entries or 2k dwords (whatever is smaller)
1562          *
1563          * The second command is for the shadow pagetables.
1564          */
1565         if (vm->root.base.bo->shadow)
1566                 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
1567         else
1568                 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
1569
1570         /* padding, etc. */
1571         ndw = 64;
1572
1573         if (pages_addr) {
1574                 /* copy commands needed */
1575                 ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
1576
1577                 /* and also PTEs */
1578                 ndw += nptes * 2;
1579
1580                 params.func = amdgpu_vm_do_copy_ptes;
1581
1582         } else {
1583                 /* set page commands needed */
1584                 ndw += ncmds * 10;
1585
1586                 /* extra commands for begin/end fragments */
1587                 if (vm->root.base.bo->shadow)
1588                         ndw += 2 * 10 * adev->vm_manager.fragment_size * 2;
1589                 else
1590                         ndw += 2 * 10 * adev->vm_manager.fragment_size;
1591
1592                 params.func = amdgpu_vm_do_set_ptes;
1593         }
1594
1595         r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1596         if (r)
1597                 return r;
1598
1599         params.ib = &job->ibs[0];
1600
1601         if (pages_addr) {
1602                 uint64_t *pte;
1603                 unsigned i;
1604
1605                 /* Put the PTEs at the end of the IB. */
1606                 i = ndw - nptes * 2;
1607                 pte= (uint64_t *)&(job->ibs->ptr[i]);
1608                 params.src = job->ibs->gpu_addr + i * 4;
1609
1610                 for (i = 0; i < nptes; ++i) {
1611                         pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1612                                                     AMDGPU_GPU_PAGE_SIZE);
1613                         pte[i] |= flags;
1614                 }
1615                 addr = 0;
1616         }
1617
1618         r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
1619         if (r)
1620                 goto error_free;
1621
1622         r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
1623                              owner, false);
1624         if (r)
1625                 goto error_free;
1626
1627         r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
1628         if (r)
1629                 goto error_free;
1630
1631         r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
1632         if (r)
1633                 goto error_free;
1634
1635         amdgpu_ring_pad_ib(ring, params.ib);
1636         WARN_ON(params.ib->length_dw > ndw);
1637         r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM, &f);
1638         if (r)
1639                 goto error_free;
1640
1641         amdgpu_bo_fence(vm->root.base.bo, f, true);
1642         dma_fence_put(*fence);
1643         *fence = f;
1644         return 0;
1645
1646 error_free:
1647         amdgpu_job_free(job);
1648         return r;
1649 }
1650
1651 /**
1652  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1653  *
1654  * @adev: amdgpu_device pointer
1655  * @exclusive: fence we need to sync to
1656  * @pages_addr: DMA addresses to use for mapping
1657  * @vm: requested vm
1658  * @mapping: mapped range and flags to use for the update
1659  * @flags: HW flags for the mapping
1660  * @nodes: array of drm_mm_nodes with the MC addresses
1661  * @fence: optional resulting fence
1662  *
1663  * Split the mapping into smaller chunks so that each update fits
1664  * into a SDMA IB.
1665  *
1666  * Returns:
1667  * 0 for success, -EINVAL for failure.
1668  */
1669 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1670                                       struct dma_fence *exclusive,
1671                                       dma_addr_t *pages_addr,
1672                                       struct amdgpu_vm *vm,
1673                                       struct amdgpu_bo_va_mapping *mapping,
1674                                       uint64_t flags,
1675                                       struct drm_mm_node *nodes,
1676                                       struct dma_fence **fence)
1677 {
1678         unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1679         uint64_t pfn, start = mapping->start;
1680         int r;
1681
1682         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1683          * but in case of something, we filter the flags in first place
1684          */
1685         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1686                 flags &= ~AMDGPU_PTE_READABLE;
1687         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1688                 flags &= ~AMDGPU_PTE_WRITEABLE;
1689
1690         flags &= ~AMDGPU_PTE_EXECUTABLE;
1691         flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1692
1693         flags &= ~AMDGPU_PTE_MTYPE_MASK;
1694         flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
1695
1696         if ((mapping->flags & AMDGPU_PTE_PRT) &&
1697             (adev->asic_type >= CHIP_VEGA10)) {
1698                 flags |= AMDGPU_PTE_PRT;
1699                 flags &= ~AMDGPU_PTE_VALID;
1700         }
1701
1702         trace_amdgpu_vm_bo_update(mapping);
1703
1704         pfn = mapping->offset >> PAGE_SHIFT;
1705         if (nodes) {
1706                 while (pfn >= nodes->size) {
1707                         pfn -= nodes->size;
1708                         ++nodes;
1709                 }
1710         }
1711
1712         do {
1713                 dma_addr_t *dma_addr = NULL;
1714                 uint64_t max_entries;
1715                 uint64_t addr, last;
1716
1717                 if (nodes) {
1718                         addr = nodes->start << PAGE_SHIFT;
1719                         max_entries = (nodes->size - pfn) *
1720                                 AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1721                 } else {
1722                         addr = 0;
1723                         max_entries = S64_MAX;
1724                 }
1725
1726                 if (pages_addr) {
1727                         uint64_t count;
1728
1729                         max_entries = min(max_entries, 16ull * 1024ull);
1730                         for (count = 1;
1731                              count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1732                              ++count) {
1733                                 uint64_t idx = pfn + count;
1734
1735                                 if (pages_addr[idx] !=
1736                                     (pages_addr[idx - 1] + PAGE_SIZE))
1737                                         break;
1738                         }
1739
1740                         if (count < min_linear_pages) {
1741                                 addr = pfn << PAGE_SHIFT;
1742                                 dma_addr = pages_addr;
1743                         } else {
1744                                 addr = pages_addr[pfn];
1745                                 max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1746                         }
1747
1748                 } else if (flags & AMDGPU_PTE_VALID) {
1749                         addr += adev->vm_manager.vram_base_offset;
1750                         addr += pfn << PAGE_SHIFT;
1751                 }
1752
1753                 last = min((uint64_t)mapping->last, start + max_entries - 1);
1754                 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1755                                                 start, last, flags, addr,
1756                                                 fence);
1757                 if (r)
1758                         return r;
1759
1760                 pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1761                 if (nodes && nodes->size == pfn) {
1762                         pfn = 0;
1763                         ++nodes;
1764                 }
1765                 start = last + 1;
1766
1767         } while (unlikely(start != mapping->last + 1));
1768
1769         return 0;
1770 }
1771
1772 /**
1773  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1774  *
1775  * @adev: amdgpu_device pointer
1776  * @bo_va: requested BO and VM object
1777  * @clear: if true clear the entries
1778  *
1779  * Fill in the page table entries for @bo_va.
1780  *
1781  * Returns:
1782  * 0 for success, -EINVAL for failure.
1783  */
1784 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1785                         struct amdgpu_bo_va *bo_va,
1786                         bool clear)
1787 {
1788         struct amdgpu_bo *bo = bo_va->base.bo;
1789         struct amdgpu_vm *vm = bo_va->base.vm;
1790         struct amdgpu_bo_va_mapping *mapping;
1791         dma_addr_t *pages_addr = NULL;
1792         struct ttm_mem_reg *mem;
1793         struct drm_mm_node *nodes;
1794         struct dma_fence *exclusive, **last_update;
1795         uint64_t flags;
1796         int r;
1797
1798         if (clear || !bo) {
1799                 mem = NULL;
1800                 nodes = NULL;
1801                 exclusive = NULL;
1802         } else {
1803                 struct ttm_dma_tt *ttm;
1804
1805                 mem = &bo->tbo.mem;
1806                 nodes = mem->mm_node;
1807                 if (mem->mem_type == TTM_PL_TT) {
1808                         ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
1809                         pages_addr = ttm->dma_address;
1810                 }
1811                 exclusive = reservation_object_get_excl(bo->tbo.resv);
1812         }
1813
1814         if (bo)
1815                 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1816         else
1817                 flags = 0x0;
1818
1819         if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
1820                 last_update = &vm->last_update;
1821         else
1822                 last_update = &bo_va->last_pt_update;
1823
1824         if (!clear && bo_va->base.moved) {
1825                 bo_va->base.moved = false;
1826                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1827
1828         } else if (bo_va->cleared != clear) {
1829                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1830         }
1831
1832         list_for_each_entry(mapping, &bo_va->invalids, list) {
1833                 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1834                                                mapping, flags, nodes,
1835                                                last_update);
1836                 if (r)
1837                         return r;
1838         }
1839
1840         if (vm->use_cpu_for_update) {
1841                 /* Flush HDP */
1842                 mb();
1843                 amdgpu_asic_flush_hdp(adev, NULL);
1844         }
1845
1846         /* If the BO is not in its preferred location add it back to
1847          * the evicted list so that it gets validated again on the
1848          * next command submission.
1849          */
1850         if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
1851                 uint32_t mem_type = bo->tbo.mem.mem_type;
1852
1853                 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
1854                         amdgpu_vm_bo_evicted(&bo_va->base);
1855                 else
1856                         amdgpu_vm_bo_idle(&bo_va->base);
1857         } else {
1858                 amdgpu_vm_bo_done(&bo_va->base);
1859         }
1860
1861         list_splice_init(&bo_va->invalids, &bo_va->valids);
1862         bo_va->cleared = clear;
1863
1864         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1865                 list_for_each_entry(mapping, &bo_va->valids, list)
1866                         trace_amdgpu_vm_bo_mapping(mapping);
1867         }
1868
1869         return 0;
1870 }
1871
1872 /**
1873  * amdgpu_vm_update_prt_state - update the global PRT state
1874  *
1875  * @adev: amdgpu_device pointer
1876  */
1877 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1878 {
1879         unsigned long flags;
1880         bool enable;
1881
1882         spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1883         enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1884         adev->gmc.gmc_funcs->set_prt(adev, enable);
1885         spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1886 }
1887
1888 /**
1889  * amdgpu_vm_prt_get - add a PRT user
1890  *
1891  * @adev: amdgpu_device pointer
1892  */
1893 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1894 {
1895         if (!adev->gmc.gmc_funcs->set_prt)
1896                 return;
1897
1898         if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1899                 amdgpu_vm_update_prt_state(adev);
1900 }
1901
1902 /**
1903  * amdgpu_vm_prt_put - drop a PRT user
1904  *
1905  * @adev: amdgpu_device pointer
1906  */
1907 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1908 {
1909         if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1910                 amdgpu_vm_update_prt_state(adev);
1911 }
1912
1913 /**
1914  * amdgpu_vm_prt_cb - callback for updating the PRT status
1915  *
1916  * @fence: fence for the callback
1917  * @_cb: the callback function
1918  */
1919 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1920 {
1921         struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1922
1923         amdgpu_vm_prt_put(cb->adev);
1924         kfree(cb);
1925 }
1926
1927 /**
1928  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1929  *
1930  * @adev: amdgpu_device pointer
1931  * @fence: fence for the callback
1932  */
1933 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1934                                  struct dma_fence *fence)
1935 {
1936         struct amdgpu_prt_cb *cb;
1937
1938         if (!adev->gmc.gmc_funcs->set_prt)
1939                 return;
1940
1941         cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1942         if (!cb) {
1943                 /* Last resort when we are OOM */
1944                 if (fence)
1945                         dma_fence_wait(fence, false);
1946
1947                 amdgpu_vm_prt_put(adev);
1948         } else {
1949                 cb->adev = adev;
1950                 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1951                                                      amdgpu_vm_prt_cb))
1952                         amdgpu_vm_prt_cb(fence, &cb->cb);
1953         }
1954 }
1955
1956 /**
1957  * amdgpu_vm_free_mapping - free a mapping
1958  *
1959  * @adev: amdgpu_device pointer
1960  * @vm: requested vm
1961  * @mapping: mapping to be freed
1962  * @fence: fence of the unmap operation
1963  *
1964  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1965  */
1966 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1967                                    struct amdgpu_vm *vm,
1968                                    struct amdgpu_bo_va_mapping *mapping,
1969                                    struct dma_fence *fence)
1970 {
1971         if (mapping->flags & AMDGPU_PTE_PRT)
1972                 amdgpu_vm_add_prt_cb(adev, fence);
1973         kfree(mapping);
1974 }
1975
1976 /**
1977  * amdgpu_vm_prt_fini - finish all prt mappings
1978  *
1979  * @adev: amdgpu_device pointer
1980  * @vm: requested vm
1981  *
1982  * Register a cleanup callback to disable PRT support after VM dies.
1983  */
1984 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1985 {
1986         struct reservation_object *resv = vm->root.base.bo->tbo.resv;
1987         struct dma_fence *excl, **shared;
1988         unsigned i, shared_count;
1989         int r;
1990
1991         r = reservation_object_get_fences_rcu(resv, &excl,
1992                                               &shared_count, &shared);
1993         if (r) {
1994                 /* Not enough memory to grab the fence list, as last resort
1995                  * block for all the fences to complete.
1996                  */
1997                 reservation_object_wait_timeout_rcu(resv, true, false,
1998                                                     MAX_SCHEDULE_TIMEOUT);
1999                 return;
2000         }
2001
2002         /* Add a callback for each fence in the reservation object */
2003         amdgpu_vm_prt_get(adev);
2004         amdgpu_vm_add_prt_cb(adev, excl);
2005
2006         for (i = 0; i < shared_count; ++i) {
2007                 amdgpu_vm_prt_get(adev);
2008                 amdgpu_vm_add_prt_cb(adev, shared[i]);
2009         }
2010
2011         kfree(shared);
2012 }
2013
2014 /**
2015  * amdgpu_vm_clear_freed - clear freed BOs in the PT
2016  *
2017  * @adev: amdgpu_device pointer
2018  * @vm: requested vm
2019  * @fence: optional resulting fence (unchanged if no work needed to be done
2020  * or if an error occurred)
2021  *
2022  * Make sure all freed BOs are cleared in the PT.
2023  * PTs have to be reserved and mutex must be locked!
2024  *
2025  * Returns:
2026  * 0 for success.
2027  *
2028  */
2029 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
2030                           struct amdgpu_vm *vm,
2031                           struct dma_fence **fence)
2032 {
2033         struct amdgpu_bo_va_mapping *mapping;
2034         uint64_t init_pte_value = 0;
2035         struct dma_fence *f = NULL;
2036         int r;
2037
2038         while (!list_empty(&vm->freed)) {
2039                 mapping = list_first_entry(&vm->freed,
2040                         struct amdgpu_bo_va_mapping, list);
2041                 list_del(&mapping->list);
2042
2043                 if (vm->pte_support_ats &&
2044                     mapping->start < AMDGPU_GMC_HOLE_START)
2045                         init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
2046
2047                 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
2048                                                 mapping->start, mapping->last,
2049                                                 init_pte_value, 0, &f);
2050                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
2051                 if (r) {
2052                         dma_fence_put(f);
2053                         return r;
2054                 }
2055         }
2056
2057         if (fence && f) {
2058                 dma_fence_put(*fence);
2059                 *fence = f;
2060         } else {
2061                 dma_fence_put(f);
2062         }
2063
2064         return 0;
2065
2066 }
2067
2068 /**
2069  * amdgpu_vm_handle_moved - handle moved BOs in the PT
2070  *
2071  * @adev: amdgpu_device pointer
2072  * @vm: requested vm
2073  *
2074  * Make sure all BOs which are moved are updated in the PTs.
2075  *
2076  * Returns:
2077  * 0 for success.
2078  *
2079  * PTs have to be reserved!
2080  */
2081 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
2082                            struct amdgpu_vm *vm)
2083 {
2084         struct amdgpu_bo_va *bo_va, *tmp;
2085         struct reservation_object *resv;
2086         bool clear;
2087         int r;
2088
2089         list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
2090                 /* Per VM BOs never need to bo cleared in the page tables */
2091                 r = amdgpu_vm_bo_update(adev, bo_va, false);
2092                 if (r)
2093                         return r;
2094         }
2095
2096         spin_lock(&vm->invalidated_lock);
2097         while (!list_empty(&vm->invalidated)) {
2098                 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
2099                                          base.vm_status);
2100                 resv = bo_va->base.bo->tbo.resv;
2101                 spin_unlock(&vm->invalidated_lock);
2102
2103                 /* Try to reserve the BO to avoid clearing its ptes */
2104                 if (!amdgpu_vm_debug && reservation_object_trylock(resv))
2105                         clear = false;
2106                 /* Somebody else is using the BO right now */
2107                 else
2108                         clear = true;
2109
2110                 r = amdgpu_vm_bo_update(adev, bo_va, clear);
2111                 if (r)
2112                         return r;
2113
2114                 if (!clear)
2115                         reservation_object_unlock(resv);
2116                 spin_lock(&vm->invalidated_lock);
2117         }
2118         spin_unlock(&vm->invalidated_lock);
2119
2120         return 0;
2121 }
2122
2123 /**
2124  * amdgpu_vm_bo_add - add a bo to a specific vm
2125  *
2126  * @adev: amdgpu_device pointer
2127  * @vm: requested vm
2128  * @bo: amdgpu buffer object
2129  *
2130  * Add @bo into the requested vm.
2131  * Add @bo to the list of bos associated with the vm
2132  *
2133  * Returns:
2134  * Newly added bo_va or NULL for failure
2135  *
2136  * Object has to be reserved!
2137  */
2138 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
2139                                       struct amdgpu_vm *vm,
2140                                       struct amdgpu_bo *bo)
2141 {
2142         struct amdgpu_bo_va *bo_va;
2143
2144         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
2145         if (bo_va == NULL) {
2146                 return NULL;
2147         }
2148         amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
2149
2150         bo_va->ref_count = 1;
2151         INIT_LIST_HEAD(&bo_va->valids);
2152         INIT_LIST_HEAD(&bo_va->invalids);
2153
2154         return bo_va;
2155 }
2156
2157
2158 /**
2159  * amdgpu_vm_bo_insert_mapping - insert a new mapping
2160  *
2161  * @adev: amdgpu_device pointer
2162  * @bo_va: bo_va to store the address
2163  * @mapping: the mapping to insert
2164  *
2165  * Insert a new mapping into all structures.
2166  */
2167 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
2168                                     struct amdgpu_bo_va *bo_va,
2169                                     struct amdgpu_bo_va_mapping *mapping)
2170 {
2171         struct amdgpu_vm *vm = bo_va->base.vm;
2172         struct amdgpu_bo *bo = bo_va->base.bo;
2173
2174         mapping->bo_va = bo_va;
2175         list_add(&mapping->list, &bo_va->invalids);
2176         amdgpu_vm_it_insert(mapping, &vm->va);
2177
2178         if (mapping->flags & AMDGPU_PTE_PRT)
2179                 amdgpu_vm_prt_get(adev);
2180
2181         if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
2182             !bo_va->base.moved) {
2183                 list_move(&bo_va->base.vm_status, &vm->moved);
2184         }
2185         trace_amdgpu_vm_bo_map(bo_va, mapping);
2186 }
2187
2188 /**
2189  * amdgpu_vm_bo_map - map bo inside a vm
2190  *
2191  * @adev: amdgpu_device pointer
2192  * @bo_va: bo_va to store the address
2193  * @saddr: where to map the BO
2194  * @offset: requested offset in the BO
2195  * @size: BO size in bytes
2196  * @flags: attributes of pages (read/write/valid/etc.)
2197  *
2198  * Add a mapping of the BO at the specefied addr into the VM.
2199  *
2200  * Returns:
2201  * 0 for success, error for failure.
2202  *
2203  * Object has to be reserved and unreserved outside!
2204  */
2205 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
2206                      struct amdgpu_bo_va *bo_va,
2207                      uint64_t saddr, uint64_t offset,
2208                      uint64_t size, uint64_t flags)
2209 {
2210         struct amdgpu_bo_va_mapping *mapping, *tmp;
2211         struct amdgpu_bo *bo = bo_va->base.bo;
2212         struct amdgpu_vm *vm = bo_va->base.vm;
2213         uint64_t eaddr;
2214
2215         /* validate the parameters */
2216         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2217             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2218                 return -EINVAL;
2219
2220         /* make sure object fit at this offset */
2221         eaddr = saddr + size - 1;
2222         if (saddr >= eaddr ||
2223             (bo && offset + size > amdgpu_bo_size(bo)))
2224                 return -EINVAL;
2225
2226         saddr /= AMDGPU_GPU_PAGE_SIZE;
2227         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2228
2229         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2230         if (tmp) {
2231                 /* bo and tmp overlap, invalid addr */
2232                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
2233                         "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
2234                         tmp->start, tmp->last + 1);
2235                 return -EINVAL;
2236         }
2237
2238         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2239         if (!mapping)
2240                 return -ENOMEM;
2241
2242         mapping->start = saddr;
2243         mapping->last = eaddr;
2244         mapping->offset = offset;
2245         mapping->flags = flags;
2246
2247         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2248
2249         return 0;
2250 }
2251
2252 /**
2253  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2254  *
2255  * @adev: amdgpu_device pointer
2256  * @bo_va: bo_va to store the address
2257  * @saddr: where to map the BO
2258  * @offset: requested offset in the BO
2259  * @size: BO size in bytes
2260  * @flags: attributes of pages (read/write/valid/etc.)
2261  *
2262  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2263  * mappings as we do so.
2264  *
2265  * Returns:
2266  * 0 for success, error for failure.
2267  *
2268  * Object has to be reserved and unreserved outside!
2269  */
2270 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2271                              struct amdgpu_bo_va *bo_va,
2272                              uint64_t saddr, uint64_t offset,
2273                              uint64_t size, uint64_t flags)
2274 {
2275         struct amdgpu_bo_va_mapping *mapping;
2276         struct amdgpu_bo *bo = bo_va->base.bo;
2277         uint64_t eaddr;
2278         int r;
2279
2280         /* validate the parameters */
2281         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2282             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2283                 return -EINVAL;
2284
2285         /* make sure object fit at this offset */
2286         eaddr = saddr + size - 1;
2287         if (saddr >= eaddr ||
2288             (bo && offset + size > amdgpu_bo_size(bo)))
2289                 return -EINVAL;
2290
2291         /* Allocate all the needed memory */
2292         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2293         if (!mapping)
2294                 return -ENOMEM;
2295
2296         r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
2297         if (r) {
2298                 kfree(mapping);
2299                 return r;
2300         }
2301
2302         saddr /= AMDGPU_GPU_PAGE_SIZE;
2303         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2304
2305         mapping->start = saddr;
2306         mapping->last = eaddr;
2307         mapping->offset = offset;
2308         mapping->flags = flags;
2309
2310         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2311
2312         return 0;
2313 }
2314
2315 /**
2316  * amdgpu_vm_bo_unmap - remove bo mapping from vm
2317  *
2318  * @adev: amdgpu_device pointer
2319  * @bo_va: bo_va to remove the address from
2320  * @saddr: where to the BO is mapped
2321  *
2322  * Remove a mapping of the BO at the specefied addr from the VM.
2323  *
2324  * Returns:
2325  * 0 for success, error for failure.
2326  *
2327  * Object has to be reserved and unreserved outside!
2328  */
2329 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2330                        struct amdgpu_bo_va *bo_va,
2331                        uint64_t saddr)
2332 {
2333         struct amdgpu_bo_va_mapping *mapping;
2334         struct amdgpu_vm *vm = bo_va->base.vm;
2335         bool valid = true;
2336
2337         saddr /= AMDGPU_GPU_PAGE_SIZE;
2338
2339         list_for_each_entry(mapping, &bo_va->valids, list) {
2340                 if (mapping->start == saddr)
2341                         break;
2342         }
2343
2344         if (&mapping->list == &bo_va->valids) {
2345                 valid = false;
2346
2347                 list_for_each_entry(mapping, &bo_va->invalids, list) {
2348                         if (mapping->start == saddr)
2349                                 break;
2350                 }
2351
2352                 if (&mapping->list == &bo_va->invalids)
2353                         return -ENOENT;
2354         }
2355
2356         list_del(&mapping->list);
2357         amdgpu_vm_it_remove(mapping, &vm->va);
2358         mapping->bo_va = NULL;
2359         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2360
2361         if (valid)
2362                 list_add(&mapping->list, &vm->freed);
2363         else
2364                 amdgpu_vm_free_mapping(adev, vm, mapping,
2365                                        bo_va->last_pt_update);
2366
2367         return 0;
2368 }
2369
2370 /**
2371  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2372  *
2373  * @adev: amdgpu_device pointer
2374  * @vm: VM structure to use
2375  * @saddr: start of the range
2376  * @size: size of the range
2377  *
2378  * Remove all mappings in a range, split them as appropriate.
2379  *
2380  * Returns:
2381  * 0 for success, error for failure.
2382  */
2383 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2384                                 struct amdgpu_vm *vm,
2385                                 uint64_t saddr, uint64_t size)
2386 {
2387         struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2388         LIST_HEAD(removed);
2389         uint64_t eaddr;
2390
2391         eaddr = saddr + size - 1;
2392         saddr /= AMDGPU_GPU_PAGE_SIZE;
2393         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2394
2395         /* Allocate all the needed memory */
2396         before = kzalloc(sizeof(*before), GFP_KERNEL);
2397         if (!before)
2398                 return -ENOMEM;
2399         INIT_LIST_HEAD(&before->list);
2400
2401         after = kzalloc(sizeof(*after), GFP_KERNEL);
2402         if (!after) {
2403                 kfree(before);
2404                 return -ENOMEM;
2405         }
2406         INIT_LIST_HEAD(&after->list);
2407
2408         /* Now gather all removed mappings */
2409         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2410         while (tmp) {
2411                 /* Remember mapping split at the start */
2412                 if (tmp->start < saddr) {
2413                         before->start = tmp->start;
2414                         before->last = saddr - 1;
2415                         before->offset = tmp->offset;
2416                         before->flags = tmp->flags;
2417                         before->bo_va = tmp->bo_va;
2418                         list_add(&before->list, &tmp->bo_va->invalids);
2419                 }
2420
2421                 /* Remember mapping split at the end */
2422                 if (tmp->last > eaddr) {
2423                         after->start = eaddr + 1;
2424                         after->last = tmp->last;
2425                         after->offset = tmp->offset;
2426                         after->offset += after->start - tmp->start;
2427                         after->flags = tmp->flags;
2428                         after->bo_va = tmp->bo_va;
2429                         list_add(&after->list, &tmp->bo_va->invalids);
2430                 }
2431
2432                 list_del(&tmp->list);
2433                 list_add(&tmp->list, &removed);
2434
2435                 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2436         }
2437
2438         /* And free them up */
2439         list_for_each_entry_safe(tmp, next, &removed, list) {
2440                 amdgpu_vm_it_remove(tmp, &vm->va);
2441                 list_del(&tmp->list);
2442
2443                 if (tmp->start < saddr)
2444                     tmp->start = saddr;
2445                 if (tmp->last > eaddr)
2446                     tmp->last = eaddr;
2447
2448                 tmp->bo_va = NULL;
2449                 list_add(&tmp->list, &vm->freed);
2450                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2451         }
2452
2453         /* Insert partial mapping before the range */
2454         if (!list_empty(&before->list)) {
2455                 amdgpu_vm_it_insert(before, &vm->va);
2456                 if (before->flags & AMDGPU_PTE_PRT)
2457                         amdgpu_vm_prt_get(adev);
2458         } else {
2459                 kfree(before);
2460         }
2461
2462         /* Insert partial mapping after the range */
2463         if (!list_empty(&after->list)) {
2464                 amdgpu_vm_it_insert(after, &vm->va);
2465                 if (after->flags & AMDGPU_PTE_PRT)
2466                         amdgpu_vm_prt_get(adev);
2467         } else {
2468                 kfree(after);
2469         }
2470
2471         return 0;
2472 }
2473
2474 /**
2475  * amdgpu_vm_bo_lookup_mapping - find mapping by address
2476  *
2477  * @vm: the requested VM
2478  * @addr: the address
2479  *
2480  * Find a mapping by it's address.
2481  *
2482  * Returns:
2483  * The amdgpu_bo_va_mapping matching for addr or NULL
2484  *
2485  */
2486 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2487                                                          uint64_t addr)
2488 {
2489         return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2490 }
2491
2492 /**
2493  * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2494  *
2495  * @vm: the requested vm
2496  * @ticket: CS ticket
2497  *
2498  * Trace all mappings of BOs reserved during a command submission.
2499  */
2500 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2501 {
2502         struct amdgpu_bo_va_mapping *mapping;
2503
2504         if (!trace_amdgpu_vm_bo_cs_enabled())
2505                 return;
2506
2507         for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2508              mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2509                 if (mapping->bo_va && mapping->bo_va->base.bo) {
2510                         struct amdgpu_bo *bo;
2511
2512                         bo = mapping->bo_va->base.bo;
2513                         if (READ_ONCE(bo->tbo.resv->lock.ctx) != ticket)
2514                                 continue;
2515                 }
2516
2517                 trace_amdgpu_vm_bo_cs(mapping);
2518         }
2519 }
2520
2521 /**
2522  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2523  *
2524  * @adev: amdgpu_device pointer
2525  * @bo_va: requested bo_va
2526  *
2527  * Remove @bo_va->bo from the requested vm.
2528  *
2529  * Object have to be reserved!
2530  */
2531 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2532                       struct amdgpu_bo_va *bo_va)
2533 {
2534         struct amdgpu_bo_va_mapping *mapping, *next;
2535         struct amdgpu_bo *bo = bo_va->base.bo;
2536         struct amdgpu_vm *vm = bo_va->base.vm;
2537
2538         if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv)
2539                 vm->bulk_moveable = false;
2540
2541         list_del(&bo_va->base.bo_list);
2542
2543         spin_lock(&vm->invalidated_lock);
2544         list_del(&bo_va->base.vm_status);
2545         spin_unlock(&vm->invalidated_lock);
2546
2547         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2548                 list_del(&mapping->list);
2549                 amdgpu_vm_it_remove(mapping, &vm->va);
2550                 mapping->bo_va = NULL;
2551                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2552                 list_add(&mapping->list, &vm->freed);
2553         }
2554         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2555                 list_del(&mapping->list);
2556                 amdgpu_vm_it_remove(mapping, &vm->va);
2557                 amdgpu_vm_free_mapping(adev, vm, mapping,
2558                                        bo_va->last_pt_update);
2559         }
2560
2561         dma_fence_put(bo_va->last_pt_update);
2562         kfree(bo_va);
2563 }
2564
2565 /**
2566  * amdgpu_vm_bo_invalidate - mark the bo as invalid
2567  *
2568  * @adev: amdgpu_device pointer
2569  * @bo: amdgpu buffer object
2570  * @evicted: is the BO evicted
2571  *
2572  * Mark @bo as invalid.
2573  */
2574 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2575                              struct amdgpu_bo *bo, bool evicted)
2576 {
2577         struct amdgpu_vm_bo_base *bo_base;
2578
2579         /* shadow bo doesn't have bo base, its validation needs its parent */
2580         if (bo->parent && bo->parent->shadow == bo)
2581                 bo = bo->parent;
2582
2583         list_for_each_entry(bo_base, &bo->va, bo_list) {
2584                 struct amdgpu_vm *vm = bo_base->vm;
2585
2586                 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2587                         amdgpu_vm_bo_evicted(bo_base);
2588                         continue;
2589                 }
2590
2591                 if (bo_base->moved)
2592                         continue;
2593                 bo_base->moved = true;
2594
2595                 if (bo->tbo.type == ttm_bo_type_kernel)
2596                         amdgpu_vm_bo_relocated(bo_base);
2597                 else if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
2598                         amdgpu_vm_bo_moved(bo_base);
2599                 else
2600                         amdgpu_vm_bo_invalidated(bo_base);
2601         }
2602 }
2603
2604 /**
2605  * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2606  *
2607  * @vm_size: VM size
2608  *
2609  * Returns:
2610  * VM page table as power of two
2611  */
2612 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2613 {
2614         /* Total bits covered by PD + PTs */
2615         unsigned bits = ilog2(vm_size) + 18;
2616
2617         /* Make sure the PD is 4K in size up to 8GB address space.
2618            Above that split equal between PD and PTs */
2619         if (vm_size <= 8)
2620                 return (bits - 9);
2621         else
2622                 return ((bits + 3) / 2);
2623 }
2624
2625 /**
2626  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2627  *
2628  * @adev: amdgpu_device pointer
2629  * @min_vm_size: the minimum vm size in GB if it's set auto
2630  * @fragment_size_default: Default PTE fragment size
2631  * @max_level: max VMPT level
2632  * @max_bits: max address space size in bits
2633  *
2634  */
2635 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2636                            uint32_t fragment_size_default, unsigned max_level,
2637                            unsigned max_bits)
2638 {
2639         unsigned int max_size = 1 << (max_bits - 30);
2640         unsigned int vm_size;
2641         uint64_t tmp;
2642
2643         /* adjust vm size first */
2644         if (amdgpu_vm_size != -1) {
2645                 vm_size = amdgpu_vm_size;
2646                 if (vm_size > max_size) {
2647                         dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2648                                  amdgpu_vm_size, max_size);
2649                         vm_size = max_size;
2650                 }
2651         } else {
2652                 struct sysinfo si;
2653                 unsigned int phys_ram_gb;
2654
2655                 /* Optimal VM size depends on the amount of physical
2656                  * RAM available. Underlying requirements and
2657                  * assumptions:
2658                  *
2659                  *  - Need to map system memory and VRAM from all GPUs
2660                  *     - VRAM from other GPUs not known here
2661                  *     - Assume VRAM <= system memory
2662                  *  - On GFX8 and older, VM space can be segmented for
2663                  *    different MTYPEs
2664                  *  - Need to allow room for fragmentation, guard pages etc.
2665                  *
2666                  * This adds up to a rough guess of system memory x3.
2667                  * Round up to power of two to maximize the available
2668                  * VM size with the given page table size.
2669                  */
2670                 si_meminfo(&si);
2671                 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2672                                (1 << 30) - 1) >> 30;
2673                 vm_size = roundup_pow_of_two(
2674                         min(max(phys_ram_gb * 3, min_vm_size), max_size));
2675         }
2676
2677         adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2678
2679         tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2680         if (amdgpu_vm_block_size != -1)
2681                 tmp >>= amdgpu_vm_block_size - 9;
2682         tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2683         adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2684         switch (adev->vm_manager.num_level) {
2685         case 3:
2686                 adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2687                 break;
2688         case 2:
2689                 adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2690                 break;
2691         case 1:
2692                 adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2693                 break;
2694         default:
2695                 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2696         }
2697         /* block size depends on vm size and hw setup*/
2698         if (amdgpu_vm_block_size != -1)
2699                 adev->vm_manager.block_size =
2700                         min((unsigned)amdgpu_vm_block_size, max_bits
2701                             - AMDGPU_GPU_PAGE_SHIFT
2702                             - 9 * adev->vm_manager.num_level);
2703         else if (adev->vm_manager.num_level > 1)
2704                 adev->vm_manager.block_size = 9;
2705         else
2706                 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2707
2708         if (amdgpu_vm_fragment_size == -1)
2709                 adev->vm_manager.fragment_size = fragment_size_default;
2710         else
2711                 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2712
2713         DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2714                  vm_size, adev->vm_manager.num_level + 1,
2715                  adev->vm_manager.block_size,
2716                  adev->vm_manager.fragment_size);
2717 }
2718
2719 /**
2720  * amdgpu_vm_init - initialize a vm instance
2721  *
2722  * @adev: amdgpu_device pointer
2723  * @vm: requested vm
2724  * @vm_context: Indicates if it GFX or Compute context
2725  * @pasid: Process address space identifier
2726  *
2727  * Init @vm fields.
2728  *
2729  * Returns:
2730  * 0 for success, error for failure.
2731  */
2732 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2733                    int vm_context, unsigned int pasid)
2734 {
2735         struct amdgpu_bo_param bp;
2736         struct amdgpu_bo *root;
2737         int r, i;
2738
2739         vm->va = RB_ROOT_CACHED;
2740         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2741                 vm->reserved_vmid[i] = NULL;
2742         INIT_LIST_HEAD(&vm->evicted);
2743         INIT_LIST_HEAD(&vm->relocated);
2744         INIT_LIST_HEAD(&vm->moved);
2745         INIT_LIST_HEAD(&vm->idle);
2746         INIT_LIST_HEAD(&vm->invalidated);
2747         spin_lock_init(&vm->invalidated_lock);
2748         INIT_LIST_HEAD(&vm->freed);
2749
2750         /* create scheduler entity for page table updates */
2751         r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs,
2752                                   adev->vm_manager.vm_pte_num_rqs, NULL);
2753         if (r)
2754                 return r;
2755
2756         vm->pte_support_ats = false;
2757
2758         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2759                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2760                                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2761
2762                 if (adev->asic_type == CHIP_RAVEN)
2763                         vm->pte_support_ats = true;
2764         } else {
2765                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2766                                                 AMDGPU_VM_USE_CPU_FOR_GFX);
2767         }
2768         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2769                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2770         WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2771                   "CPU update of VM recommended only for large BAR system\n");
2772         vm->last_update = NULL;
2773
2774         amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp);
2775         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
2776                 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
2777         r = amdgpu_bo_create(adev, &bp, &root);
2778         if (r)
2779                 goto error_free_sched_entity;
2780
2781         r = amdgpu_bo_reserve(root, true);
2782         if (r)
2783                 goto error_free_root;
2784
2785         r = amdgpu_vm_clear_bo(adev, vm, root,
2786                                adev->vm_manager.root_level,
2787                                vm->pte_support_ats);
2788         if (r)
2789                 goto error_unreserve;
2790
2791         amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
2792         amdgpu_bo_unreserve(vm->root.base.bo);
2793
2794         if (pasid) {
2795                 unsigned long flags;
2796
2797                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2798                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2799                               GFP_ATOMIC);
2800                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2801                 if (r < 0)
2802                         goto error_free_root;
2803
2804                 vm->pasid = pasid;
2805         }
2806
2807         INIT_KFIFO(vm->faults);
2808         vm->fault_credit = 16;
2809
2810         return 0;
2811
2812 error_unreserve:
2813         amdgpu_bo_unreserve(vm->root.base.bo);
2814
2815 error_free_root:
2816         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2817         amdgpu_bo_unref(&vm->root.base.bo);
2818         vm->root.base.bo = NULL;
2819
2820 error_free_sched_entity:
2821         drm_sched_entity_destroy(&vm->entity);
2822
2823         return r;
2824 }
2825
2826 /**
2827  * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2828  *
2829  * @adev: amdgpu_device pointer
2830  * @vm: requested vm
2831  *
2832  * This only works on GFX VMs that don't have any BOs added and no
2833  * page tables allocated yet.
2834  *
2835  * Changes the following VM parameters:
2836  * - use_cpu_for_update
2837  * - pte_supports_ats
2838  * - pasid (old PASID is released, because compute manages its own PASIDs)
2839  *
2840  * Reinitializes the page directory to reflect the changed ATS
2841  * setting.
2842  *
2843  * Returns:
2844  * 0 for success, -errno for errors.
2845  */
2846 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid)
2847 {
2848         bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2849         int r;
2850
2851         r = amdgpu_bo_reserve(vm->root.base.bo, true);
2852         if (r)
2853                 return r;
2854
2855         /* Sanity checks */
2856         if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) {
2857                 r = -EINVAL;
2858                 goto unreserve_bo;
2859         }
2860
2861         if (pasid) {
2862                 unsigned long flags;
2863
2864                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2865                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2866                               GFP_ATOMIC);
2867                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2868
2869                 if (r == -ENOSPC)
2870                         goto unreserve_bo;
2871                 r = 0;
2872         }
2873
2874         /* Check if PD needs to be reinitialized and do it before
2875          * changing any other state, in case it fails.
2876          */
2877         if (pte_support_ats != vm->pte_support_ats) {
2878                 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
2879                                adev->vm_manager.root_level,
2880                                pte_support_ats);
2881                 if (r)
2882                         goto free_idr;
2883         }
2884
2885         /* Update VM state */
2886         vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2887                                     AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2888         vm->pte_support_ats = pte_support_ats;
2889         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2890                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2891         WARN_ONCE((vm->use_cpu_for_update & !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2892                   "CPU update of VM recommended only for large BAR system\n");
2893
2894         if (vm->pasid) {
2895                 unsigned long flags;
2896
2897                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2898                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2899                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2900
2901                 /* Free the original amdgpu allocated pasid
2902                  * Will be replaced with kfd allocated pasid
2903                  */
2904                 amdgpu_pasid_free(vm->pasid);
2905                 vm->pasid = 0;
2906         }
2907
2908         /* Free the shadow bo for compute VM */
2909         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2910
2911         if (pasid)
2912                 vm->pasid = pasid;
2913
2914         goto unreserve_bo;
2915
2916 free_idr:
2917         if (pasid) {
2918                 unsigned long flags;
2919
2920                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2921                 idr_remove(&adev->vm_manager.pasid_idr, pasid);
2922                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2923         }
2924 unreserve_bo:
2925         amdgpu_bo_unreserve(vm->root.base.bo);
2926         return r;
2927 }
2928
2929 /**
2930  * amdgpu_vm_release_compute - release a compute vm
2931  * @adev: amdgpu_device pointer
2932  * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
2933  *
2934  * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2935  * pasid from vm. Compute should stop use of vm after this call.
2936  */
2937 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2938 {
2939         if (vm->pasid) {
2940                 unsigned long flags;
2941
2942                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2943                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2944                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2945         }
2946         vm->pasid = 0;
2947 }
2948
2949 /**
2950  * amdgpu_vm_free_levels - free PD/PT levels
2951  *
2952  * @adev: amdgpu device structure
2953  * @parent: PD/PT starting level to free
2954  * @level: level of parent structure
2955  *
2956  * Free the page directory or page table level and all sub levels.
2957  */
2958 static void amdgpu_vm_free_levels(struct amdgpu_device *adev,
2959                                   struct amdgpu_vm_pt *parent,
2960                                   unsigned level)
2961 {
2962         unsigned i, num_entries = amdgpu_vm_num_entries(adev, level);
2963
2964         if (parent->base.bo) {
2965                 list_del(&parent->base.bo_list);
2966                 list_del(&parent->base.vm_status);
2967                 amdgpu_bo_unref(&parent->base.bo->shadow);
2968                 amdgpu_bo_unref(&parent->base.bo);
2969         }
2970
2971         if (parent->entries)
2972                 for (i = 0; i < num_entries; i++)
2973                         amdgpu_vm_free_levels(adev, &parent->entries[i],
2974                                               level + 1);
2975
2976         kvfree(parent->entries);
2977 }
2978
2979 /**
2980  * amdgpu_vm_fini - tear down a vm instance
2981  *
2982  * @adev: amdgpu_device pointer
2983  * @vm: requested vm
2984  *
2985  * Tear down @vm.
2986  * Unbind the VM and remove all bos from the vm bo list
2987  */
2988 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2989 {
2990         struct amdgpu_bo_va_mapping *mapping, *tmp;
2991         bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2992         struct amdgpu_bo *root;
2993         u64 fault;
2994         int i, r;
2995
2996         amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2997
2998         /* Clear pending page faults from IH when the VM is destroyed */
2999         while (kfifo_get(&vm->faults, &fault))
3000                 amdgpu_ih_clear_fault(adev, fault);
3001
3002         if (vm->pasid) {
3003                 unsigned long flags;
3004
3005                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
3006                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
3007                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
3008         }
3009
3010         drm_sched_entity_destroy(&vm->entity);
3011
3012         if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
3013                 dev_err(adev->dev, "still active bo inside vm\n");
3014         }
3015         rbtree_postorder_for_each_entry_safe(mapping, tmp,
3016                                              &vm->va.rb_root, rb) {
3017                 list_del(&mapping->list);
3018                 amdgpu_vm_it_remove(mapping, &vm->va);
3019                 kfree(mapping);
3020         }
3021         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
3022                 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
3023                         amdgpu_vm_prt_fini(adev, vm);
3024                         prt_fini_needed = false;
3025                 }
3026
3027                 list_del(&mapping->list);
3028                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
3029         }
3030
3031         root = amdgpu_bo_ref(vm->root.base.bo);
3032         r = amdgpu_bo_reserve(root, true);
3033         if (r) {
3034                 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
3035         } else {
3036                 amdgpu_vm_free_levels(adev, &vm->root,
3037                                       adev->vm_manager.root_level);
3038                 amdgpu_bo_unreserve(root);
3039         }
3040         amdgpu_bo_unref(&root);
3041         dma_fence_put(vm->last_update);
3042         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
3043                 amdgpu_vmid_free_reserved(adev, vm, i);
3044 }
3045
3046 /**
3047  * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
3048  *
3049  * @adev: amdgpu_device pointer
3050  * @pasid: PASID do identify the VM
3051  *
3052  * This function is expected to be called in interrupt context.
3053  *
3054  * Returns:
3055  * True if there was fault credit, false otherwise
3056  */
3057 bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
3058                                   unsigned int pasid)
3059 {
3060         struct amdgpu_vm *vm;
3061
3062         spin_lock(&adev->vm_manager.pasid_lock);
3063         vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3064         if (!vm) {
3065                 /* VM not found, can't track fault credit */
3066                 spin_unlock(&adev->vm_manager.pasid_lock);
3067                 return true;
3068         }
3069
3070         /* No lock needed. only accessed by IRQ handler */
3071         if (!vm->fault_credit) {
3072                 /* Too many faults in this VM */
3073                 spin_unlock(&adev->vm_manager.pasid_lock);
3074                 return false;
3075         }
3076
3077         vm->fault_credit--;
3078         spin_unlock(&adev->vm_manager.pasid_lock);
3079         return true;
3080 }
3081
3082 /**
3083  * amdgpu_vm_manager_init - init the VM manager
3084  *
3085  * @adev: amdgpu_device pointer
3086  *
3087  * Initialize the VM manager structures
3088  */
3089 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
3090 {
3091         unsigned i;
3092
3093         amdgpu_vmid_mgr_init(adev);
3094
3095         adev->vm_manager.fence_context =
3096                 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3097         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
3098                 adev->vm_manager.seqno[i] = 0;
3099
3100         spin_lock_init(&adev->vm_manager.prt_lock);
3101         atomic_set(&adev->vm_manager.num_prt_users, 0);
3102
3103         /* If not overridden by the user, by default, only in large BAR systems
3104          * Compute VM tables will be updated by CPU
3105          */
3106 #ifdef CONFIG_X86_64
3107         if (amdgpu_vm_update_mode == -1) {
3108                 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
3109                         adev->vm_manager.vm_update_mode =
3110                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
3111                 else
3112                         adev->vm_manager.vm_update_mode = 0;
3113         } else
3114                 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
3115 #else
3116         adev->vm_manager.vm_update_mode = 0;
3117 #endif
3118
3119         idr_init(&adev->vm_manager.pasid_idr);
3120         spin_lock_init(&adev->vm_manager.pasid_lock);
3121 }
3122
3123 /**
3124  * amdgpu_vm_manager_fini - cleanup VM manager
3125  *
3126  * @adev: amdgpu_device pointer
3127  *
3128  * Cleanup the VM manager and free resources.
3129  */
3130 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
3131 {
3132         WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
3133         idr_destroy(&adev->vm_manager.pasid_idr);
3134
3135         amdgpu_vmid_mgr_fini(adev);
3136 }
3137
3138 /**
3139  * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
3140  *
3141  * @dev: drm device pointer
3142  * @data: drm_amdgpu_vm
3143  * @filp: drm file pointer
3144  *
3145  * Returns:
3146  * 0 for success, -errno for errors.
3147  */
3148 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
3149 {
3150         union drm_amdgpu_vm *args = data;
3151         struct amdgpu_device *adev = dev->dev_private;
3152         struct amdgpu_fpriv *fpriv = filp->driver_priv;
3153         int r;
3154
3155         switch (args->in.op) {
3156         case AMDGPU_VM_OP_RESERVE_VMID:
3157                 /* current, we only have requirement to reserve vmid from gfxhub */
3158                 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
3159                 if (r)
3160                         return r;
3161                 break;
3162         case AMDGPU_VM_OP_UNRESERVE_VMID:
3163                 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
3164                 break;
3165         default:
3166                 return -EINVAL;
3167         }
3168
3169         return 0;
3170 }
3171
3172 /**
3173  * amdgpu_vm_get_task_info - Extracts task info for a PASID.
3174  *
3175  * @adev: drm device pointer
3176  * @pasid: PASID identifier for VM
3177  * @task_info: task_info to fill.
3178  */
3179 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
3180                          struct amdgpu_task_info *task_info)
3181 {
3182         struct amdgpu_vm *vm;
3183
3184         spin_lock(&adev->vm_manager.pasid_lock);
3185
3186         vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3187         if (vm)
3188                 *task_info = vm->task_info;
3189
3190         spin_unlock(&adev->vm_manager.pasid_lock);
3191 }
3192
3193 /**
3194  * amdgpu_vm_set_task_info - Sets VMs task info.
3195  *
3196  * @vm: vm for which to set the info
3197  */
3198 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
3199 {
3200         if (!vm->task_info.pid) {
3201                 vm->task_info.pid = current->pid;
3202                 get_task_comm(vm->task_info.task_name, current);
3203
3204                 if (current->group_leader->mm == current->mm) {
3205                         vm->task_info.tgid = current->group_leader->pid;
3206                         get_task_comm(vm->task_info.process_name, current->group_leader);
3207                 }
3208         }
3209 }
This page took 0.224343 seconds and 4 git commands to generate.