struct dma_fence_cb cb;
};
-/**
+/*
* vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
* happens while holding this lock anywhere to prevent deadlocks when
* an MMU notifier runs in reclaim-FS context.
* @adev: amdgpu_device pointer
* @vm: VM to clear BO from
* @bo: BO to clear
- * @direct: use a direct update
+ * @immediate: use an immediate update
*
* Root PD needs to be reserved when calling this.
*
static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_bo *bo,
- bool direct)
+ bool immediate)
{
struct ttm_operation_ctx ctx = { true, false };
unsigned level = adev->vm_manager.root_level;
memset(¶ms, 0, sizeof(params));
params.adev = adev;
params.vm = vm;
- params.direct = direct;
+ params.immediate = immediate;
r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT);
if (r)
* @adev: amdgpu_device pointer
* @vm: requesting vm
* @level: the page table level
- * @direct: use a direct update
+ * @immediate: use a immediate update
* @bp: resulting BO allocation parameters
*/
static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- int level, bool direct,
+ int level, bool immediate,
struct amdgpu_bo_param *bp)
{
memset(bp, 0, sizeof(*bp));
else if (!vm->root.base.bo || vm->root.base.bo->shadow)
bp->flags |= AMDGPU_GEM_CREATE_SHADOW;
bp->type = ttm_bo_type_kernel;
- bp->no_wait_gpu = direct;
+ bp->no_wait_gpu = immediate;
if (vm->root.base.bo)
bp->resv = vm->root.base.bo->tbo.base.resv;
}
* @adev: amdgpu_device pointer
* @vm: VM to allocate page tables for
* @cursor: Which page table to allocate
- * @direct: use a direct update
+ * @immediate: use an immediate update
*
* Make sure a specific page table or directory is allocated.
*
static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
struct amdgpu_vm_pt_cursor *cursor,
- bool direct)
+ bool immediate)
{
struct amdgpu_vm_pt *entry = cursor->entry;
struct amdgpu_bo_param bp;
if (entry->base.bo)
return 0;
- amdgpu_vm_bo_param(adev, vm, cursor->level, direct, &bp);
+ amdgpu_vm_bo_param(adev, vm, cursor->level, immediate, &bp);
r = amdgpu_bo_create(adev, &bp, &pt);
if (r)
pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);
amdgpu_vm_bo_base_init(&entry->base, vm, pt);
- r = amdgpu_vm_clear_bo(adev, vm, pt, direct);
+ r = amdgpu_vm_clear_bo(adev, vm, pt, immediate);
if (r)
goto error_free_pt;
*
* @adev: amdgpu_device pointer
* @vm: requested vm
- * @direct: submit directly to the paging queue
+ * @immediate: submit immediately to the paging queue
*
* Makes sure all directories are up to date.
*
* 0 for success, error for failure.
*/
int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
- struct amdgpu_vm *vm, bool direct)
+ struct amdgpu_vm *vm, bool immediate)
{
struct amdgpu_vm_update_params params;
int r;
memset(¶ms, 0, sizeof(params));
params.adev = adev;
params.vm = vm;
- params.direct = direct;
+ params.immediate = immediate;
r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT);
if (r)
uint64_t incr, entry_end, pe_start;
struct amdgpu_bo *pt;
- if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
+ if (!params->unlocked) {
/* make sure that the page tables covering the
* address range are actually allocated
*/
r = amdgpu_vm_alloc_pts(params->adev, params->vm,
- &cursor, params->direct);
+ &cursor, params->immediate);
if (r)
return r;
}
shift = amdgpu_vm_level_shift(adev, cursor.level);
parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
- if (adev->asic_type < CHIP_VEGA10 &&
- (flags & AMDGPU_PTE_VALID)) {
+ if (params->unlocked) {
+ /* Unlocked updates are only allowed on the leaves */
+ if (amdgpu_vm_pt_descendant(adev, &cursor))
+ continue;
+ } else if (adev->asic_type < CHIP_VEGA10 &&
+ (flags & AMDGPU_PTE_VALID)) {
/* No huge page support before GMC v9 */
if (cursor.level != AMDGPU_VM_PTB) {
if (!amdgpu_vm_pt_descendant(adev, &cursor))
*
* @adev: amdgpu_device pointer
* @vm: requested vm
- * @direct: direct submission in a page fault
+ * @immediate: immediate submission in a page fault
+ * @unlocked: unlocked invalidation during MM callback
* @resv: fences we need to sync to
* @start: start of mapped range
* @last: last mapped entry
* 0 for success, -EINVAL for failure.
*/
static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
- struct amdgpu_vm *vm, bool direct,
- struct dma_resv *resv,
+ struct amdgpu_vm *vm, bool immediate,
+ bool unlocked, struct dma_resv *resv,
uint64_t start, uint64_t last,
uint64_t flags, uint64_t addr,
dma_addr_t *pages_addr,
memset(¶ms, 0, sizeof(params));
params.adev = adev;
params.vm = vm;
- params.direct = direct;
+ params.immediate = immediate;
params.pages_addr = pages_addr;
+ params.unlocked = unlocked;
/* Implicitly sync to command submissions in the same VM before
* unmapping. Sync to moving fences before mapping.
goto error_unlock;
}
- if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) {
- struct amdgpu_bo *root = vm->root.base.bo;
+ if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
+ struct dma_fence *tmp = dma_fence_get_stub();
- if (!dma_fence_is_signaled(vm->last_direct))
- amdgpu_bo_fence(root, vm->last_direct, true);
+ amdgpu_bo_fence(vm->root.base.bo, vm->last_unlocked, true);
+ swap(vm->last_unlocked, tmp);
+ dma_fence_put(tmp);
}
r = vm->update_funcs->prepare(¶ms, resv, sync_mode);
}
last = min((uint64_t)mapping->last, start + max_entries - 1);
- r = amdgpu_vm_bo_update_mapping(adev, vm, false, resv,
+ r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv,
start, last, flags, addr,
dma_addr, fence);
if (r)
struct amdgpu_vm *vm = bo_va->base.vm;
struct amdgpu_bo_va_mapping *mapping;
dma_addr_t *pages_addr = NULL;
- struct ttm_mem_reg *mem;
+ struct ttm_resource *mem;
struct drm_mm_node *nodes;
struct dma_fence **last_update;
struct dma_resv *resv;
if (bo) {
flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
+
+ if (amdgpu_bo_encrypted(bo))
+ flags |= AMDGPU_PTE_TMZ;
+
bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
} else {
flags = 0x0;
mapping->start < AMDGPU_GMC_HOLE_START)
init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
- r = amdgpu_vm_bo_update_mapping(adev, vm, false, resv,
+ r = amdgpu_vm_bo_update_mapping(adev, vm, false, false, resv,
mapping->start, mapping->last,
init_pte_value, 0, NULL, &f);
amdgpu_vm_free_mapping(adev, vm, mapping, f);
if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) &&
(bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
bo_va->is_xgmi = true;
- mutex_lock(&adev->vm_manager.lock_pstate);
/* Power up XGMI if it can be potentially used */
- if (++adev->vm_manager.xgmi_map_counter == 1)
- amdgpu_xgmi_set_pstate(adev, 1);
- mutex_unlock(&adev->vm_manager.lock_pstate);
+ amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
}
return bo_va;
/* make sure object fit at this offset */
eaddr = saddr + size - 1;
if (saddr >= eaddr ||
- (bo && offset + size > amdgpu_bo_size(bo)))
+ (bo && offset + size > amdgpu_bo_size(bo)) ||
+ (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
return -EINVAL;
saddr /= AMDGPU_GPU_PAGE_SIZE;
/* make sure object fit at this offset */
eaddr = saddr + size - 1;
if (saddr >= eaddr ||
- (bo && offset + size > amdgpu_bo_size(bo)))
+ (bo && offset + size > amdgpu_bo_size(bo)) ||
+ (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT))
return -EINVAL;
/* Allocate all the needed memory */
dma_fence_put(bo_va->last_pt_update);
- if (bo && bo_va->is_xgmi) {
- mutex_lock(&adev->vm_manager.lock_pstate);
- if (--adev->vm_manager.xgmi_map_counter == 0)
- amdgpu_xgmi_set_pstate(adev, 0);
- mutex_unlock(&adev->vm_manager.lock_pstate);
- }
+ if (bo && bo_va->is_xgmi)
+ amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
kfree(bo_va);
}
return false;
/* Don't evict VM page tables while they are updated */
- if (!dma_fence_is_signaled(bo_base->vm->last_direct)) {
+ if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
amdgpu_vm_eviction_unlock(bo_base->vm);
return false;
}
if (timeout <= 0)
return timeout;
- return dma_fence_wait_timeout(vm->last_direct, true, timeout);
+ return dma_fence_wait_timeout(vm->last_unlocked, true, timeout);
}
/**
/* create scheduler entities for page table updates */
- r = drm_sched_entity_init(&vm->direct, DRM_SCHED_PRIORITY_NORMAL,
+ r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
adev->vm_manager.vm_pte_scheds,
adev->vm_manager.vm_pte_num_scheds, NULL);
if (r)
adev->vm_manager.vm_pte_scheds,
adev->vm_manager.vm_pte_num_scheds, NULL);
if (r)
- goto error_free_direct;
+ goto error_free_immediate;
vm->pte_support_ats = false;
vm->is_compute_context = false;
else
vm->update_funcs = &amdgpu_vm_sdma_funcs;
vm->last_update = NULL;
- vm->last_direct = dma_fence_get_stub();
+ vm->last_unlocked = dma_fence_get_stub();
mutex_init(&vm->eviction_lock);
vm->evicting = false;
vm->root.base.bo = NULL;
error_free_delayed:
- dma_fence_put(vm->last_direct);
+ dma_fence_put(vm->last_unlocked);
drm_sched_entity_destroy(&vm->delayed);
-error_free_direct:
- drm_sched_entity_destroy(&vm->direct);
+error_free_immediate:
+ drm_sched_entity_destroy(&vm->immediate);
return r;
}
!amdgpu_gmc_vram_full_visible(&adev->gmc)),
"CPU update of VM recommended only for large BAR system\n");
- if (vm->use_cpu_for_update)
+ if (vm->use_cpu_for_update) {
+ /* Sync with last SDMA update/clear before switching to CPU */
+ r = amdgpu_bo_sync_wait(vm->root.base.bo,
+ AMDGPU_FENCE_OWNER_UNDEFINED, true);
+ if (r)
+ goto free_idr;
+
vm->update_funcs = &amdgpu_vm_cpu_funcs;
- else
+ } else {
vm->update_funcs = &amdgpu_vm_sdma_funcs;
+ }
dma_fence_put(vm->last_update);
vm->last_update = NULL;
vm->is_compute_context = true;
vm->pasid = 0;
}
- dma_fence_wait(vm->last_direct, false);
- dma_fence_put(vm->last_direct);
+ dma_fence_wait(vm->last_unlocked, false);
+ dma_fence_put(vm->last_unlocked);
list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
amdgpu_bo_unref(&root);
WARN_ON(vm->root.base.bo);
- drm_sched_entity_destroy(&vm->direct);
+ drm_sched_entity_destroy(&vm->immediate);
drm_sched_entity_destroy(&vm->delayed);
if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
idr_init(&adev->vm_manager.pasid_idr);
spin_lock_init(&adev->vm_manager.pasid_lock);
-
- adev->vm_manager.xgmi_map_counter = 0;
- mutex_init(&adev->vm_manager.lock_pstate);
}
/**
value = 0;
}
- r = amdgpu_vm_bo_update_mapping(adev, vm, true, NULL, addr, addr + 1,
- flags, value, NULL, NULL);
+ r = amdgpu_vm_bo_update_mapping(adev, vm, true, false, NULL, addr,
+ addr + 1, flags, value, NULL, NULL);
if (r)
goto error_unlock;
projects / linux.git / blobdiff