Merge tag 'iommu-updates-v4.18' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_vm.c

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index f0deedcaf1c93e7b7585ff21f12c64f90f0d8d91..ccba88cc8c54227e6a20cd9b84a0234f8f0590e6 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -224,21 +224,16 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                              void *param)
 {
        struct ttm_bo_global *glob = adev->mman.bdev.glob;
-       int r;
-
-       while (!list_empty(&vm->evicted)) {
-               struct amdgpu_vm_bo_base *bo_base;
-               struct amdgpu_bo *bo;
+       struct amdgpu_vm_bo_base *bo_base, *tmp;
+       int r = 0;
 
-               bo_base = list_first_entry(&vm->evicted,
-                                          struct amdgpu_vm_bo_base,
-                                          vm_status);
+       list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
+               struct amdgpu_bo *bo = bo_base->bo;
 
-               bo = bo_base->bo;
                if (bo->parent) {
                        r = validate(param, bo);
                        if (r)
-                               return r;
+                               break;
 
                        spin_lock(&glob->lru_lock);
                        ttm_bo_move_to_lru_tail(&bo->tbo);
@@ -247,13 +242,6 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                        spin_unlock(&glob->lru_lock);
                }
 
-               if (bo->tbo.type == ttm_bo_type_kernel &&
-                   vm->use_cpu_for_update) {
-                       r = amdgpu_bo_kmap(bo, NULL);
-                       if (r)
-                               return r;
-               }
-
                if (bo->tbo.type != ttm_bo_type_kernel) {
                        spin_lock(&vm->moved_lock);
                        list_move(&bo_base->vm_status, &vm->moved);
@@ -263,7 +251,20 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                }
        }
 
-       return 0;
+       spin_lock(&glob->lru_lock);
+       list_for_each_entry(bo_base, &vm->idle, vm_status) {
+               struct amdgpu_bo *bo = bo_base->bo;
+
+               if (!bo->parent)
+                       continue;
+
+               ttm_bo_move_to_lru_tail(&bo->tbo);
+               if (bo->shadow)
+                       ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
+       }
+       spin_unlock(&glob->lru_lock);
+
+       return r;
 }
 
 /**
@@ -914,8 +915,8 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
                if (!entry->base.bo)
                        continue;
 
-               if (list_empty(&entry->base.vm_status))
-                       list_add(&entry->base.vm_status, &vm->relocated);
+               if (!entry->base.moved)
+                       list_move(&entry->base.vm_status, &vm->relocated);
                amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
        }
 }
@@ -945,6 +946,14 @@ restart:
        params.adev = adev;
 
        if (vm->use_cpu_for_update) {
+               struct amdgpu_vm_bo_base *bo_base;
+
+               list_for_each_entry(bo_base, &vm->relocated, vm_status) {
+                       r = amdgpu_bo_kmap(bo_base->bo, NULL);
+                       if (unlikely(r))
+                               return r;
+               }
+
                r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
                if (unlikely(r))
                        return r;
@@ -968,7 +977,8 @@ restart:
                bo_base = list_first_entry(&vm->relocated,
                                           struct amdgpu_vm_bo_base,
                                           vm_status);
-               list_del_init(&bo_base->vm_status);
+               bo_base->moved = false;
+               list_move(&bo_base->vm_status, &vm->idle);
 
                bo = bo_base->bo->parent;
                if (!bo)
@@ -1574,10 +1584,14 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         * the evicted list so that it gets validated again on the
         * next command submission.
         */
-       if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
-           !(bo->preferred_domains &
-           amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)))
-               list_add_tail(&bo_va->base.vm_status, &vm->evicted);
+       if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+               uint32_t mem_type = bo->tbo.mem.mem_type;
+
+               if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
+                       list_add_tail(&bo_va->base.vm_status, &vm->evicted);
+               else
+                       list_add(&bo_va->base.vm_status, &vm->idle);
+       }
 
        list_splice_init(&bo_va->invalids, &bo_va->valids);
        bo_va->cleared = clear;
@@ -1786,19 +1800,18 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
                           struct amdgpu_vm *vm)
 {
+       struct amdgpu_bo_va *bo_va, *tmp;
+       struct list_head moved;
        bool clear;
-       int r = 0;
+       int r;
 
+       INIT_LIST_HEAD(&moved);
        spin_lock(&vm->moved_lock);
-       while (!list_empty(&vm->moved)) {
-               struct amdgpu_bo_va *bo_va;
-               struct reservation_object *resv;
-
-               bo_va = list_first_entry(&vm->moved,
-                       struct amdgpu_bo_va, base.vm_status);
-               spin_unlock(&vm->moved_lock);
+       list_splice_init(&vm->moved, &moved);
+       spin_unlock(&vm->moved_lock);
 
-               resv = bo_va->base.bo->tbo.resv;
+       list_for_each_entry_safe(bo_va, tmp, &moved, base.vm_status) {
+               struct reservation_object *resv = bo_va->base.bo->tbo.resv;
 
                /* Per VM BOs never need to bo cleared in the page tables */
                if (resv == vm->root.base.bo->tbo.resv)
@@ -1811,17 +1824,19 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
                        clear = true;
 
                r = amdgpu_vm_bo_update(adev, bo_va, clear);
-               if (r)
+               if (r) {
+                       spin_lock(&vm->moved_lock);
+                       list_splice(&moved, &vm->moved);
+                       spin_unlock(&vm->moved_lock);
                        return r;
+               }
 
                if (!clear && resv != vm->root.base.bo->tbo.resv)
                        reservation_object_unlock(resv);
 
-               spin_lock(&vm->moved_lock);
        }
-       spin_unlock(&vm->moved_lock);
 
-       return r;
+       return 0;
 }
 
 /**
@@ -1880,10 +1895,10 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
        if (mapping->flags & AMDGPU_PTE_PRT)
                amdgpu_vm_prt_get(adev);
 
-       if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
+       if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
+           !bo_va->base.moved) {
                spin_lock(&vm->moved_lock);
-               if (list_empty(&bo_va->base.vm_status))
-                       list_add(&bo_va->base.vm_status, &vm->moved);
+               list_move(&bo_va->base.vm_status, &vm->moved);
                spin_unlock(&vm->moved_lock);
        }
        trace_amdgpu_vm_bo_map(bo_va, mapping);
@@ -2236,6 +2251,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
 
        list_for_each_entry(bo_base, &bo->va, bo_list) {
                struct amdgpu_vm *vm = bo_base->vm;
+               bool was_moved = bo_base->moved;
 
                bo_base->moved = true;
                if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
@@ -2247,16 +2263,16 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
                        continue;
                }
 
-               if (bo->tbo.type == ttm_bo_type_kernel) {
-                       if (list_empty(&bo_base->vm_status))
-                               list_add(&bo_base->vm_status, &vm->relocated);
+               if (was_moved)
                        continue;
-               }
 
-               spin_lock(&bo_base->vm->moved_lock);
-               if (list_empty(&bo_base->vm_status))
-                       list_add(&bo_base->vm_status, &vm->moved);
-               spin_unlock(&bo_base->vm->moved_lock);
+               if (bo->tbo.type == ttm_bo_type_kernel) {
+                       list_move(&bo_base->vm_status, &vm->relocated);
+               } else {
+                       spin_lock(&bo_base->vm->moved_lock);
+                       list_move(&bo_base->vm_status, &vm->moved);
+                       spin_unlock(&bo_base->vm->moved_lock);
+               }
        }
 }
 
@@ -2369,6 +2385,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
        INIT_LIST_HEAD(&vm->relocated);
        spin_lock_init(&vm->moved_lock);
        INIT_LIST_HEAD(&vm->moved);
+       INIT_LIST_HEAD(&vm->idle);
        INIT_LIST_HEAD(&vm->freed);
 
        /* create scheduler entity for page table updates */