1 // SPDX-License-Identifier: MIT
3 * Copyright © 2022 Intel Corporation
10 #include "xe_drm_client.h"
12 #include "xe_gt_tlb_invalidation.h"
13 #include "xe_migrate.h"
14 #include "xe_pt_types.h"
15 #include "xe_pt_walk.h"
16 #include "xe_res_cursor.h"
18 #include "xe_ttm_stolen_mgr.h"
23 /** @dir: Directory structure for the xe_pt_walk functionality */
24 struct xe_ptw_dir dir;
27 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
28 #define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
29 #define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
31 #define xe_pt_set_addr(__xe_pt, __addr)
32 #define xe_pt_addr(__xe_pt) 0ull
35 static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48};
36 static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48};
38 #define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
40 static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
42 return container_of(pt, struct xe_pt_dir, pt);
45 static struct xe_pt *xe_pt_entry(struct xe_pt_dir *pt_dir, unsigned int index)
47 return container_of(pt_dir->dir.entries[index], struct xe_pt, base);
50 static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
53 struct xe_device *xe = tile_to_xe(tile);
54 u16 pat_index = xe->pat.idx[XE_CACHE_WB];
57 if (!xe_vm_has_scratch(vm))
60 if (level > MAX_HUGEPTE_LEVEL)
61 return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo,
64 return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) |
69 * xe_pt_create() - Create a page-table.
70 * @vm: The vm to create for.
71 * @tile: The tile to create for.
72 * @level: The page-table level.
74 * Allocate and initialize a single struct xe_pt metadata structure. Also
75 * create the corresponding page-table bo, but don't initialize it. If the
76 * level is grater than zero, then it's assumed to be a directory page-
77 * table and the directory structure is also allocated and initialized to
80 * Return: A valid struct xe_pt pointer on success, Pointer error code on
83 struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
91 size = !level ? sizeof(struct xe_pt) : sizeof(struct xe_pt_dir) +
92 XE_PDES * sizeof(struct xe_ptw *);
93 pt = kzalloc(size, GFP_KERNEL);
95 return ERR_PTR(-ENOMEM);
97 bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
99 XE_BO_CREATE_VRAM_IF_DGFX(tile) |
100 XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT |
101 XE_BO_CREATE_PINNED_BIT |
102 XE_BO_CREATE_NO_RESV_EVICT |
110 pt->base.dir = level ? &as_xe_pt_dir(pt)->dir : NULL;
113 xe_drm_client_add_bo(vm->xef->client, pt->bo);
114 xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);
124 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
126 * @tile: The tile the scratch pagetable of which to use.
127 * @vm: The vm we populate for.
128 * @pt: The pagetable the bo of which to initialize.
130 * Populate the page-table bo of @pt with entries pointing into the tile's
131 * scratch page-table tree if any. Otherwise populate with zeros.
133 void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
136 struct iosys_map *map = &pt->bo->vmap;
140 if (!xe_vm_has_scratch(vm)) {
142 * FIXME: Some memory is allocated already allocated to zero?
143 * Find out which memory that is and avoid this memset...
145 xe_map_memset(vm->xe, map, 0, 0, SZ_4K);
147 empty = __xe_pt_empty_pte(tile, vm, pt->level);
148 for (i = 0; i < XE_PDES; i++)
149 xe_pt_write(vm->xe, map, i, empty);
154 * xe_pt_shift() - Return the ilog2 value of the size of the address range of
155 * a page-table at a certain level.
158 * Return: The ilog2 value of the size of the address range of a page-table
161 unsigned int xe_pt_shift(unsigned int level)
163 return XE_PTE_SHIFT + XE_PDE_SHIFT * level;
167 * xe_pt_destroy() - Destroy a page-table tree.
168 * @pt: The root of the page-table tree to destroy.
169 * @flags: vm flags. Currently unused.
170 * @deferred: List head of lockless list for deferred putting. NULL for
173 * Puts the page-table bo, recursively calls xe_pt_destroy on all children
174 * and finally frees @pt. TODO: Can we remove the @flags argument?
176 void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
183 XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
185 xe_bo_put_deferred(pt->bo, deferred);
187 if (pt->level > 0 && pt->num_live) {
188 struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
190 for (i = 0; i < XE_PDES; i++) {
191 if (xe_pt_entry(pt_dir, i))
192 xe_pt_destroy(xe_pt_entry(pt_dir, i), flags,
200 * DOC: Pagetable building
202 * Below we use the term "page-table" for both page-directories, containing
203 * pointers to lower level page-directories or page-tables, and level 0
204 * page-tables that contain only page-table-entries pointing to memory pages.
206 * When inserting an address range in an already existing page-table tree
207 * there will typically be a set of page-tables that are shared with other
208 * address ranges, and a set that are private to this address range.
209 * The set of shared page-tables can be at most two per level,
210 * and those can't be updated immediately because the entries of those
211 * page-tables may still be in use by the gpu for other mappings. Therefore
212 * when inserting entries into those, we instead stage those insertions by
213 * adding insertion data into struct xe_vm_pgtable_update structures. This
214 * data, (subtrees for the cpu and page-table-entries for the gpu) is then
215 * added in a separate commit step. CPU-data is committed while still under the
216 * vm lock, the object lock and for userptr, the notifier lock in read mode.
217 * The GPU async data is committed either by the GPU or CPU after fulfilling
218 * relevant dependencies.
219 * For non-shared page-tables (and, in fact, for shared ones that aren't
220 * existing at the time of staging), we add the data in-place without the
221 * special update structures. This private part of the page-table tree will
222 * remain disconnected from the vm page-table tree until data is committed to
223 * the shared page tables of the vm tree in the commit phase.
226 struct xe_pt_update {
227 /** @update: The update structure we're building for this parent. */
228 struct xe_vm_pgtable_update *update;
229 /** @parent: The parent. Used to detect a parent change. */
230 struct xe_pt *parent;
231 /** @preexisting: Whether the parent was pre-existing or allocated */
235 struct xe_pt_stage_bind_walk {
236 /** base: The base class. */
237 struct xe_pt_walk base;
239 /* Input parameters for the walk */
240 /** @vm: The vm we're building for. */
242 /** @tile: The tile we're building for. */
243 struct xe_tile *tile;
244 /** @default_pte: PTE flag only template. No address is associated */
246 /** @dma_offset: DMA offset to add to the PTE. */
249 * @needs_64k: This address range enforces 64K alignment and
254 * @vma: VMA being mapped
258 /* Also input, but is updated during the walk*/
259 /** @curs: The DMA address cursor. */
260 struct xe_res_cursor *curs;
261 /** @va_curs_start: The Virtual address coresponding to @curs->start */
265 struct xe_walk_update {
266 /** @wupd.entries: Caller provided storage. */
267 struct xe_vm_pgtable_update *entries;
268 /** @wupd.num_used_entries: Number of update @entries used. */
269 unsigned int num_used_entries;
270 /** @wupd.updates: Tracks the update entry at a given level */
271 struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1];
276 * @l0_end_addr: The end address of the current l0 leaf. Used for
277 * 64K granularity detection.
280 /** @addr_64K: The start address of the current 64K chunk. */
282 /** @found_64: Whether @add_64K actually points to a 64K chunk. */
287 xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
288 pgoff_t offset, bool alloc_entries)
290 struct xe_pt_update *upd = &wupd->updates[parent->level];
291 struct xe_vm_pgtable_update *entry;
294 * For *each level*, we could only have one active
295 * struct xt_pt_update at any one time. Once we move on to a
296 * new parent and page-directory, the old one is complete, and
297 * updates are either already stored in the build tree or in
300 if (likely(upd->parent == parent))
303 upd->parent = parent;
304 upd->preexisting = true;
306 if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1)
309 entry = wupd->entries + wupd->num_used_entries++;
312 entry->pt_bo = parent->bo;
318 entry->pt_entries = kmalloc_array(XE_PDES,
319 sizeof(*entry->pt_entries),
321 if (!entry->pt_entries)
329 * NOTE: This is a very frequently called function so we allow ourselves
330 * to annotate (using branch prediction hints) the fastpath of updating a
331 * non-pre-existing pagetable with leaf ptes.
334 xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
335 pgoff_t offset, struct xe_pt *xe_child, u64 pte)
337 struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level];
338 struct xe_pt_update *child_upd = xe_child ?
339 &xe_walk->wupd.updates[xe_child->level] : NULL;
342 ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true);
347 * Register this new pagetable so that it won't be recognized as
348 * a shared pagetable by a subsequent insertion.
350 if (unlikely(child_upd)) {
351 child_upd->update = NULL;
352 child_upd->parent = xe_child;
353 child_upd->preexisting = false;
356 if (likely(!upd->preexisting)) {
357 /* Continue building a non-connected subtree. */
358 struct iosys_map *map = &parent->bo->vmap;
360 if (unlikely(xe_child))
361 parent->base.dir->entries[offset] = &xe_child->base;
363 xe_pt_write(xe_walk->vm->xe, map, offset, pte);
366 /* Shared pt. Stage update. */
368 struct xe_vm_pgtable_update *entry = upd->update;
370 idx = offset - entry->ofs;
371 entry->pt_entries[idx].pt = xe_child;
372 entry->pt_entries[idx].pte = pte;
379 static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level,
380 struct xe_pt_stage_bind_walk *xe_walk)
384 if (level > MAX_HUGEPTE_LEVEL)
387 /* Does the virtual range requested cover a huge pte? */
388 if (!xe_pt_covers(addr, next, level, &xe_walk->base))
391 /* Does the DMA segment cover the whole pte? */
392 if (next - xe_walk->va_curs_start > xe_walk->curs->size)
395 /* null VMA's do not have dma addresses */
396 if (xe_vma_is_null(xe_walk->vma))
399 /* Is the DMA address huge PTE size aligned? */
401 dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs);
403 return IS_ALIGNED(dma, size);
407 * Scan the requested mapping to check whether it can be done entirely
411 xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
413 struct xe_res_cursor curs = *xe_walk->curs;
415 if (!IS_ALIGNED(addr, SZ_64K))
418 if (next > xe_walk->l0_end_addr)
421 /* null VMA's do not have dma addresses */
422 if (xe_vma_is_null(xe_walk->vma))
425 xe_res_next(&curs, addr - xe_walk->va_curs_start);
426 for (; addr < next; addr += SZ_64K) {
427 if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K)
430 xe_res_next(&curs, SZ_64K);
437 * For non-compact "normal" 4K level-0 pagetables, we want to try to group
438 * addresses together in 64K-contigous regions to add a 64K TLB hint for the
440 * This function determines whether the address is part of such a
441 * segment. For VRAM in normal pagetables, this is strictly necessary on
445 xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
447 /* Address is within an already found 64k region */
448 if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K)
451 xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk);
452 xe_walk->addr_64K = addr;
454 return xe_walk->found_64K;
458 xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset,
459 unsigned int level, u64 addr, u64 next,
460 struct xe_ptw **child,
461 enum page_walk_action *action,
462 struct xe_pt_walk *walk)
464 struct xe_pt_stage_bind_walk *xe_walk =
465 container_of(walk, typeof(*xe_walk), base);
466 u16 pat_index = xe_walk->vma->pat_index;
467 struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base);
468 struct xe_vm *vm = xe_walk->vm;
469 struct xe_pt *xe_child;
474 /* Is this a leaf entry ?*/
475 if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) {
476 struct xe_res_cursor *curs = xe_walk->curs;
477 bool is_null = xe_vma_is_null(xe_walk->vma);
479 XE_WARN_ON(xe_walk->va_curs_start != addr);
481 pte = vm->pt_ops->pte_encode_vma(is_null ? 0 :
482 xe_res_dma(curs) + xe_walk->dma_offset,
483 xe_walk->vma, pat_index, level);
484 pte |= xe_walk->default_pte;
487 * Set the XE_PTE_PS64 hint if possible, otherwise if
488 * this device *requires* 64K PTE size for VRAM, fail.
490 if (level == 0 && !xe_parent->is_compact) {
491 if (xe_pt_is_pte_ps64K(addr, next, xe_walk))
493 else if (XE_WARN_ON(xe_walk->needs_64K))
497 ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte);
502 xe_res_next(curs, next - addr);
503 xe_walk->va_curs_start = next;
504 xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level);
505 *action = ACTION_CONTINUE;
511 * Descending to lower level. Determine if we need to allocate a
512 * new page table or -directory, which we do if there is no
513 * previous one or there is one we can completely replace.
516 walk->shifts = xe_normal_pt_shifts;
517 xe_walk->l0_end_addr = next;
520 covers = xe_pt_covers(addr, next, level, &xe_walk->base);
521 if (covers || !*child) {
524 xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1);
525 if (IS_ERR(xe_child))
526 return PTR_ERR(xe_child);
528 xe_pt_set_addr(xe_child,
529 round_down(addr, 1ull << walk->shifts[level]));
532 xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);
534 *child = &xe_child->base;
537 * Prefer the compact pagetable layout for L0 if possible.
538 * TODO: Suballocate the pt bo to avoid wasting a lot of
541 if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&
542 covers && xe_pt_scan_64K(addr, next, xe_walk)) {
543 walk->shifts = xe_compact_pt_shifts;
545 xe_child->is_compact = true;
548 pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0, pat_index) | flags;
549 ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child,
553 *action = ACTION_SUBTREE;
557 static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
558 .pt_entry = xe_pt_stage_bind_entry,
562 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
564 * @tile: The tile we're building for.
565 * @vma: The vma indicating the address range.
566 * @entries: Storage for the update entries used for connecting the tree to
567 * the main tree at commit time.
568 * @num_entries: On output contains the number of @entries used.
570 * This function builds a disconnected page-table tree for a given address
571 * range. The tree is connected to the main vm tree for the gpu using
572 * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind().
573 * The function builds xe_vm_pgtable_update structures for already existing
574 * shared page-tables, and non-existing shared and non-shared page-tables
575 * are built and populated directly.
577 * Return 0 on success, negative error code on error.
580 xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
581 struct xe_vm_pgtable_update *entries, u32 *num_entries)
583 struct xe_device *xe = tile_to_xe(tile);
584 struct xe_bo *bo = xe_vma_bo(vma);
585 bool is_devmem = !xe_vma_is_userptr(vma) && bo &&
586 (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo));
587 struct xe_res_cursor curs;
588 struct xe_pt_stage_bind_walk xe_walk = {
590 .ops = &xe_pt_stage_bind_ops,
591 .shifts = xe_normal_pt_shifts,
592 .max_level = XE_PT_HIGHEST_LEVEL,
594 .vm = xe_vma_vm(vma),
597 .va_curs_start = xe_vma_start(vma),
599 .wupd.entries = entries,
600 .needs_64K = (xe_vma_vm(vma)->flags & XE_VM_FLAG_64K) && is_devmem,
602 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
605 if (vma && (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) &&
606 (is_devmem || !IS_DGFX(xe)))
607 xe_walk.default_pte |= XE_USM_PPGTT_PTE_AE;
610 xe_walk.default_pte |= XE_PPGTT_PTE_DM;
611 xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource);
614 if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo))
615 xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo));
617 xe_bo_assert_held(bo);
619 if (!xe_vma_is_null(vma)) {
620 if (xe_vma_is_userptr(vma))
621 xe_res_first_sg(vma->userptr.sg, 0, xe_vma_size(vma),
623 else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
624 xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
625 xe_vma_size(vma), &curs);
627 xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
628 xe_vma_size(vma), &curs);
630 curs.size = xe_vma_size(vma);
633 ret = xe_pt_walk_range(&pt->base, pt->level, xe_vma_start(vma),
634 xe_vma_end(vma), &xe_walk.base);
636 *num_entries = xe_walk.wupd.num_used_entries;
641 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
643 * @addr: The start address within the non-shared pagetable.
644 * @end: The end address within the non-shared pagetable.
645 * @level: The level of the non-shared pagetable.
646 * @walk: Walk info. The function adjusts the walk action.
647 * @action: next action to perform (see enum page_walk_action)
648 * @offset: Ignored on input, First non-shared entry on output.
649 * @end_offset: Ignored on input, Last non-shared entry + 1 on output.
651 * A non-shared page-table has some entries that belong to the address range
652 * and others that don't. This function determines the entries that belong
653 * fully to the address range. Depending on level, some entries may
654 * partially belong to the address range (that can't happen at level 0).
655 * The function detects that and adjust those offsets to not include those
656 * partial entries. Iff it does detect partial entries, we know that there must
657 * be shared page tables also at lower levels, so it adjusts the walk action
660 * Return: true if there were non-shared entries, false otherwise.
662 static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level,
663 struct xe_pt_walk *walk,
664 enum page_walk_action *action,
665 pgoff_t *offset, pgoff_t *end_offset)
667 u64 size = 1ull << walk->shifts[level];
669 *offset = xe_pt_offset(addr, level, walk);
670 *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset;
676 * If addr or next are not size aligned, there are shared pts at lower
677 * level, so in that case traverse down the subtree
679 *action = ACTION_CONTINUE;
680 if (!IS_ALIGNED(addr, size)) {
681 *action = ACTION_SUBTREE;
685 if (!IS_ALIGNED(end, size)) {
686 *action = ACTION_SUBTREE;
690 return *end_offset > *offset;
693 struct xe_pt_zap_ptes_walk {
694 /** @base: The walk base-class */
695 struct xe_pt_walk base;
697 /* Input parameters for the walk */
698 /** @tile: The tile we're building for */
699 struct xe_tile *tile;
702 /** @needs_invalidate: Whether we need to invalidate TLB*/
703 bool needs_invalidate;
706 static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
707 unsigned int level, u64 addr, u64 next,
708 struct xe_ptw **child,
709 enum page_walk_action *action,
710 struct xe_pt_walk *walk)
712 struct xe_pt_zap_ptes_walk *xe_walk =
713 container_of(walk, typeof(*xe_walk), base);
714 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
718 XE_WARN_ON(!level && xe_child->is_compact);
721 * Note that we're called from an entry callback, and we're dealing
722 * with the child of that entry rather than the parent, so need to
725 if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset,
727 xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,
728 offset * sizeof(u64), 0,
729 (end_offset - offset) * sizeof(u64));
730 xe_walk->needs_invalidate = true;
736 static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
737 .pt_entry = xe_pt_zap_ptes_entry,
741 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
742 * @tile: The tile we're zapping for.
743 * @vma: GPU VMA detailing address range.
745 * Eviction and Userptr invalidation needs to be able to zap the
746 * gpu ptes of a given address range in pagefaulting mode.
747 * In order to be able to do that, that function needs access to the shared
748 * page-table entrieaso it can either clear the leaf PTEs or
749 * clear the pointers to lower-level page-tables. The caller is required
750 * to hold the necessary locks to ensure neither the page-table connectivity
751 * nor the page-table entries of the range is updated from under us.
753 * Return: Whether ptes were actually updated and a TLB invalidation is
756 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
758 struct xe_pt_zap_ptes_walk xe_walk = {
760 .ops = &xe_pt_zap_ptes_ops,
761 .shifts = xe_normal_pt_shifts,
762 .max_level = XE_PT_HIGHEST_LEVEL,
766 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
768 if (!(vma->tile_present & BIT(tile->id)))
771 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
772 xe_vma_end(vma), &xe_walk.base);
774 return xe_walk.needs_invalidate;
778 xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
779 struct iosys_map *map, void *data,
780 u32 qword_ofs, u32 num_qwords,
781 const struct xe_vm_pgtable_update *update)
783 struct xe_pt_entry *ptes = update->pt_entries;
787 for (i = 0; i < num_qwords; i++) {
789 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
790 sizeof(u64), u64, ptes[i].pte);
792 ptr[i] = ptes[i].pte;
796 static void xe_pt_abort_bind(struct xe_vma *vma,
797 struct xe_vm_pgtable_update *entries,
802 for (i = 0; i < num_entries; i++) {
803 if (!entries[i].pt_entries)
806 for (j = 0; j < entries[i].qwords; j++)
807 xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL);
808 kfree(entries[i].pt_entries);
812 static void xe_pt_commit_locks_assert(struct xe_vma *vma)
814 struct xe_vm *vm = xe_vma_vm(vma);
816 lockdep_assert_held(&vm->lock);
818 if (xe_vma_is_userptr(vma))
819 lockdep_assert_held_read(&vm->userptr.notifier_lock);
820 else if (!xe_vma_is_null(vma))
821 dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
823 xe_vm_assert_held(vm);
826 static void xe_pt_commit_bind(struct xe_vma *vma,
827 struct xe_vm_pgtable_update *entries,
828 u32 num_entries, bool rebind,
829 struct llist_head *deferred)
833 xe_pt_commit_locks_assert(vma);
835 for (i = 0; i < num_entries; i++) {
836 struct xe_pt *pt = entries[i].pt;
837 struct xe_pt_dir *pt_dir;
840 pt->num_live += entries[i].qwords;
843 kfree(entries[i].pt_entries);
847 pt_dir = as_xe_pt_dir(pt);
848 for (j = 0; j < entries[i].qwords; j++) {
849 u32 j_ = j + entries[i].ofs;
850 struct xe_pt *newpte = entries[i].pt_entries[j].pt;
852 if (xe_pt_entry(pt_dir, j_))
853 xe_pt_destroy(xe_pt_entry(pt_dir, j_),
854 xe_vma_vm(vma)->flags, deferred);
856 pt_dir->dir.entries[j_] = &newpte->base;
858 kfree(entries[i].pt_entries);
863 xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
864 struct xe_vm_pgtable_update *entries, u32 *num_entries,
870 err = xe_pt_stage_bind(tile, vma, entries, num_entries);
872 xe_tile_assert(tile, *num_entries);
874 xe_pt_abort_bind(vma, entries, *num_entries);
879 static void xe_vm_dbg_print_entries(struct xe_device *xe,
880 const struct xe_vm_pgtable_update *entries,
881 unsigned int num_entries)
882 #if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
886 vm_dbg(&xe->drm, "%u entries to update\n", num_entries);
887 for (i = 0; i < num_entries; i++) {
888 const struct xe_vm_pgtable_update *entry = &entries[i];
889 struct xe_pt *xe_pt = entry->pt;
890 u64 page_size = 1ull << xe_pt_shift(xe_pt->level);
894 xe_assert(xe, !entry->pt->is_compact);
895 start = entry->ofs * page_size;
896 end = start + page_size * entry->qwords;
898 "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n",
899 i, xe_pt->level, entry->ofs, entry->qwords,
900 xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0);
907 #ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
909 static int xe_pt_userptr_inject_eagain(struct xe_vma *vma)
911 u32 divisor = vma->userptr.divisor ? vma->userptr.divisor : 2;
914 if (count++ % divisor == divisor - 1) {
915 struct xe_vm *vm = xe_vma_vm(vma);
917 vma->userptr.divisor = divisor << 1;
918 spin_lock(&vm->userptr.invalidated_lock);
919 list_move_tail(&vma->userptr.invalidate_link,
920 &vm->userptr.invalidated);
921 spin_unlock(&vm->userptr.invalidated_lock);
930 static bool xe_pt_userptr_inject_eagain(struct xe_vma *vma)
938 * struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks
939 * @base: Base we derive from.
940 * @bind: Whether this is a bind or an unbind operation. A bind operation
941 * makes the pre-commit callback error with -EAGAIN if it detects a
942 * pending invalidation.
943 * @locked: Whether the pre-commit callback locked the userptr notifier lock
944 * and it needs unlocking.
946 struct xe_pt_migrate_pt_update {
947 struct xe_migrate_pt_update base;
953 * This function adds the needed dependencies to a page-table update job
954 * to make sure racing jobs for separate bind engines don't race writing
955 * to the same page-table range, wreaking havoc. Initially use a single
956 * fence for the entire VM. An optimization would use smaller granularity.
958 static int xe_pt_vm_dependencies(struct xe_sched_job *job,
959 struct xe_range_fence_tree *rftree,
962 struct xe_range_fence *rtfence;
963 struct dma_fence *fence;
966 rtfence = xe_range_fence_tree_first(rftree, start, last);
968 fence = rtfence->fence;
970 if (!dma_fence_is_signaled(fence)) {
972 * Is this a CPU update? GPU is busy updating, so return
978 dma_fence_get(fence);
979 err = drm_sched_job_add_dependency(&job->drm, fence);
984 rtfence = xe_range_fence_tree_next(rtfence, start, last);
990 static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
992 struct xe_range_fence_tree *rftree =
993 &xe_vma_vm(pt_update->vma)->rftree[pt_update->tile_id];
995 return xe_pt_vm_dependencies(pt_update->job, rftree,
996 pt_update->start, pt_update->last);
999 static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
1001 struct xe_pt_migrate_pt_update *userptr_update =
1002 container_of(pt_update, typeof(*userptr_update), base);
1003 struct xe_vma *vma = pt_update->vma;
1004 unsigned long notifier_seq = vma->userptr.notifier_seq;
1005 struct xe_vm *vm = xe_vma_vm(vma);
1006 int err = xe_pt_vm_dependencies(pt_update->job,
1007 &vm->rftree[pt_update->tile_id],
1014 userptr_update->locked = false;
1017 * Wait until nobody is running the invalidation notifier, and
1018 * since we're exiting the loop holding the notifier lock,
1019 * nobody can proceed invalidating either.
1021 * Note that we don't update the vma->userptr.notifier_seq since
1022 * we don't update the userptr pages.
1025 down_read(&vm->userptr.notifier_lock);
1026 if (!mmu_interval_read_retry(&vma->userptr.notifier,
1030 up_read(&vm->userptr.notifier_lock);
1032 if (userptr_update->bind)
1035 notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier);
1038 /* Inject errors to test_whether they are handled correctly */
1039 if (userptr_update->bind && xe_pt_userptr_inject_eagain(vma)) {
1040 up_read(&vm->userptr.notifier_lock);
1044 userptr_update->locked = true;
1049 static const struct xe_migrate_pt_update_ops bind_ops = {
1050 .populate = xe_vm_populate_pgtable,
1051 .pre_commit = xe_pt_pre_commit,
1054 static const struct xe_migrate_pt_update_ops userptr_bind_ops = {
1055 .populate = xe_vm_populate_pgtable,
1056 .pre_commit = xe_pt_userptr_pre_commit,
1059 struct invalidation_fence {
1060 struct xe_gt_tlb_invalidation_fence base;
1063 struct dma_fence *fence;
1064 struct dma_fence_cb cb;
1065 struct work_struct work;
1069 invalidation_fence_get_driver_name(struct dma_fence *dma_fence)
1075 invalidation_fence_get_timeline_name(struct dma_fence *dma_fence)
1077 return "invalidation_fence";
1080 static const struct dma_fence_ops invalidation_fence_ops = {
1081 .get_driver_name = invalidation_fence_get_driver_name,
1082 .get_timeline_name = invalidation_fence_get_timeline_name,
1085 static void invalidation_fence_cb(struct dma_fence *fence,
1086 struct dma_fence_cb *cb)
1088 struct invalidation_fence *ifence =
1089 container_of(cb, struct invalidation_fence, cb);
1091 trace_xe_gt_tlb_invalidation_fence_cb(&ifence->base);
1092 if (!ifence->fence->error) {
1093 queue_work(system_wq, &ifence->work);
1095 ifence->base.base.error = ifence->fence->error;
1096 dma_fence_signal(&ifence->base.base);
1097 dma_fence_put(&ifence->base.base);
1099 dma_fence_put(ifence->fence);
1102 static void invalidation_fence_work_func(struct work_struct *w)
1104 struct invalidation_fence *ifence =
1105 container_of(w, struct invalidation_fence, work);
1107 trace_xe_gt_tlb_invalidation_fence_work_func(&ifence->base);
1108 xe_gt_tlb_invalidation_vma(ifence->gt, &ifence->base, ifence->vma);
1111 static int invalidation_fence_init(struct xe_gt *gt,
1112 struct invalidation_fence *ifence,
1113 struct dma_fence *fence,
1118 trace_xe_gt_tlb_invalidation_fence_create(&ifence->base);
1120 spin_lock_irq(>->tlb_invalidation.lock);
1121 dma_fence_init(&ifence->base.base, &invalidation_fence_ops,
1122 >->tlb_invalidation.lock,
1123 gt->tlb_invalidation.fence_context,
1124 ++gt->tlb_invalidation.fence_seqno);
1125 spin_unlock_irq(>->tlb_invalidation.lock);
1127 INIT_LIST_HEAD(&ifence->base.link);
1129 dma_fence_get(&ifence->base.base); /* Ref for caller */
1130 ifence->fence = fence;
1134 INIT_WORK(&ifence->work, invalidation_fence_work_func);
1135 ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb);
1136 if (ret == -ENOENT) {
1137 dma_fence_put(ifence->fence); /* Usually dropped in CB */
1138 invalidation_fence_work_func(&ifence->work);
1140 dma_fence_put(&ifence->base.base); /* Caller ref */
1141 dma_fence_put(&ifence->base.base); /* Creation ref */
1144 xe_gt_assert(gt, !ret || ret == -ENOENT);
1146 return ret && ret != -ENOENT ? ret : 0;
1149 static void xe_pt_calc_rfence_interval(struct xe_vma *vma,
1150 struct xe_pt_migrate_pt_update *update,
1151 struct xe_vm_pgtable_update *entries,
1156 for (i = 0; i < num_entries; i++) {
1157 const struct xe_vm_pgtable_update *entry = &entries[i];
1159 if (entry->pt->level > level)
1160 level = entry->pt->level;
1163 /* Greedy (non-optimal) calculation but simple */
1164 update->base.start = ALIGN_DOWN(xe_vma_start(vma),
1165 0x1ull << xe_pt_shift(level));
1166 update->base.last = ALIGN(xe_vma_end(vma),
1167 0x1ull << xe_pt_shift(level)) - 1;
1171 * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma
1173 * @tile: The tile to bind for.
1174 * @vma: The vma to bind.
1175 * @q: The exec_queue with which to do pipelined page-table updates.
1176 * @syncs: Entries to sync on before binding the built tree to the live vm tree.
1177 * @num_syncs: Number of @sync entries.
1178 * @rebind: Whether we're rebinding this vma to the same address range without
1179 * an unbind in-between.
1181 * This function builds a page-table tree (see xe_pt_stage_bind() for more
1182 * information on page-table building), and the xe_vm_pgtable_update entries
1183 * abstracting the operations needed to attach it to the main vm tree. It
1184 * then takes the relevant locks and updates the metadata side of the main
1185 * vm tree and submits the operations for pipelined attachment of the
1186 * gpu page-table to the vm main tree, (which can be done either by the
1189 * Return: A valid dma-fence representing the pipelined attachment operation
1190 * on success, an error pointer on error.
1193 __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
1194 struct xe_sync_entry *syncs, u32 num_syncs,
1197 struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
1198 struct xe_pt_migrate_pt_update bind_pt_update = {
1200 .ops = xe_vma_is_userptr(vma) ? &userptr_bind_ops : &bind_ops,
1202 .tile_id = tile->id,
1206 struct xe_vm *vm = xe_vma_vm(vma);
1208 struct dma_fence *fence;
1209 struct invalidation_fence *ifence = NULL;
1210 struct xe_range_fence *rfence;
1213 bind_pt_update.locked = false;
1214 xe_bo_assert_held(xe_vma_bo(vma));
1215 xe_vm_assert_held(vm);
1217 vm_dbg(&xe_vma_vm(vma)->xe->drm,
1218 "Preparing bind, with range [%llx...%llx) engine %p.\n",
1219 xe_vma_start(vma), xe_vma_end(vma), q);
1221 err = xe_pt_prepare_bind(tile, vma, entries, &num_entries, rebind);
1224 xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
1226 xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries);
1227 xe_pt_calc_rfence_interval(vma, &bind_pt_update, entries,
1231 * If rebind, we have to invalidate TLB on !LR vms to invalidate
1232 * cached PTEs point to freed memory. on LR vms this is done
1233 * automatically when the context is re-enabled by the rebind worker,
1234 * or in fault mode it was invalidated on PTE zapping.
1236 * If !rebind, and scratch enabled VMs, there is a chance the scratch
1237 * PTE is already cached in the TLB so it needs to be invalidated.
1238 * on !LR VMs this is done in the ring ops preceding a batch, but on
1239 * non-faulting LR, in particular on user-space batch buffer chaining,
1240 * it needs to be done here.
1242 if ((rebind && !xe_vm_in_lr_mode(vm) && !vm->batch_invalidate_tlb) ||
1243 (!rebind && xe_vm_has_scratch(vm) && xe_vm_in_preempt_fence_mode(vm))) {
1244 ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
1246 return ERR_PTR(-ENOMEM);
1249 rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
1252 return ERR_PTR(-ENOMEM);
1255 fence = xe_migrate_update_pgtables(tile->migrate,
1256 vm, xe_vma_bo(vma), q,
1257 entries, num_entries,
1259 &bind_pt_update.base);
1260 if (!IS_ERR(fence)) {
1261 bool last_munmap_rebind = vma->gpuva.flags & XE_VMA_LAST_REBIND;
1262 LLIST_HEAD(deferred);
1265 err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
1266 &xe_range_fence_kfree_ops,
1267 bind_pt_update.base.start,
1268 bind_pt_update.base.last, fence);
1270 dma_fence_wait(fence, false);
1272 /* TLB invalidation must be done before signaling rebind */
1274 int err = invalidation_fence_init(tile->primary_gt, ifence, fence,
1277 dma_fence_put(fence);
1279 return ERR_PTR(err);
1281 fence = &ifence->base.base;
1284 /* add shared fence now for pagetable delayed destroy */
1285 dma_resv_add_fence(xe_vm_resv(vm), fence, !rebind &&
1286 last_munmap_rebind ?
1287 DMA_RESV_USAGE_KERNEL :
1288 DMA_RESV_USAGE_BOOKKEEP);
1290 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1291 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
1292 DMA_RESV_USAGE_BOOKKEEP);
1293 xe_pt_commit_bind(vma, entries, num_entries, rebind,
1294 bind_pt_update.locked ? &deferred : NULL);
1296 /* This vma is live (again?) now */
1297 vma->tile_present |= BIT(tile->id);
1299 if (bind_pt_update.locked) {
1300 vma->userptr.initial_bind = true;
1301 up_read(&vm->userptr.notifier_lock);
1302 xe_bo_put_commit(&deferred);
1304 if (!rebind && last_munmap_rebind &&
1305 xe_vm_in_preempt_fence_mode(vm))
1306 xe_vm_queue_rebind_worker(vm);
1310 if (bind_pt_update.locked)
1311 up_read(&vm->userptr.notifier_lock);
1312 xe_pt_abort_bind(vma, entries, num_entries);
1318 return ERR_PTR(err);
1321 struct xe_pt_stage_unbind_walk {
1322 /** @base: The pagewalk base-class. */
1323 struct xe_pt_walk base;
1325 /* Input parameters for the walk */
1326 /** @tile: The tile we're unbinding from. */
1327 struct xe_tile *tile;
1330 * @modified_start: Walk range start, modified to include any
1331 * shared pagetables that we're the only user of and can thus
1335 /** @modified_end: Walk range start, modified like @modified_start. */
1339 /* @wupd: Structure to track the page-table updates we're building */
1340 struct xe_walk_update wupd;
1344 * Check whether this range is the only one populating this pagetable,
1345 * and in that case, update the walk range checks so that higher levels don't
1346 * view us as a shared pagetable.
1348 static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
1349 const struct xe_pt *child,
1350 enum page_walk_action *action,
1351 struct xe_pt_walk *walk)
1353 struct xe_pt_stage_unbind_walk *xe_walk =
1354 container_of(walk, typeof(*xe_walk), base);
1355 unsigned int shift = walk->shifts[level];
1356 u64 size = 1ull << shift;
1358 if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) &&
1359 ((next - addr) >> shift) == child->num_live) {
1360 u64 size = 1ull << walk->shifts[level + 1];
1362 *action = ACTION_CONTINUE;
1364 if (xe_walk->modified_start >= addr)
1365 xe_walk->modified_start = round_down(addr, size);
1366 if (xe_walk->modified_end <= next)
1367 xe_walk->modified_end = round_up(next, size);
1375 static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
1376 unsigned int level, u64 addr, u64 next,
1377 struct xe_ptw **child,
1378 enum page_walk_action *action,
1379 struct xe_pt_walk *walk)
1381 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1383 XE_WARN_ON(!*child);
1384 XE_WARN_ON(!level && xe_child->is_compact);
1386 xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
1392 xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
1393 unsigned int level, u64 addr, u64 next,
1394 struct xe_ptw **child,
1395 enum page_walk_action *action,
1396 struct xe_pt_walk *walk)
1398 struct xe_pt_stage_unbind_walk *xe_walk =
1399 container_of(walk, typeof(*xe_walk), base);
1400 struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1402 u64 size = 1ull << walk->shifts[--level];
1404 if (!IS_ALIGNED(addr, size))
1405 addr = xe_walk->modified_start;
1406 if (!IS_ALIGNED(next, size))
1407 next = xe_walk->modified_end;
1409 /* Parent == *child is the root pt. Don't kill it. */
1410 if (parent != *child &&
1411 xe_pt_check_kill(addr, next, level, xe_child, action, walk))
1414 if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset,
1418 (void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false);
1419 xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
1424 static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = {
1425 .pt_entry = xe_pt_stage_unbind_entry,
1426 .pt_post_descend = xe_pt_stage_unbind_post_descend,
1430 * xe_pt_stage_unbind() - Build page-table update structures for an unbind
1432 * @tile: The tile we're unbinding for.
1433 * @vma: The vma we're unbinding.
1434 * @entries: Caller-provided storage for the update structures.
1436 * Builds page-table update structures for an unbind operation. The function
1437 * will attempt to remove all page-tables that we're the only user
1438 * of, and for that to work, the unbind operation must be committed in the
1439 * same critical section that blocks racing binds to the same page-table tree.
1441 * Return: The number of entries used.
1443 static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,
1444 struct xe_vm_pgtable_update *entries)
1446 struct xe_pt_stage_unbind_walk xe_walk = {
1448 .ops = &xe_pt_stage_unbind_ops,
1449 .shifts = xe_normal_pt_shifts,
1450 .max_level = XE_PT_HIGHEST_LEVEL,
1453 .modified_start = xe_vma_start(vma),
1454 .modified_end = xe_vma_end(vma),
1455 .wupd.entries = entries,
1457 struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
1459 (void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
1460 xe_vma_end(vma), &xe_walk.base);
1462 return xe_walk.wupd.num_used_entries;
1466 xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
1467 struct xe_tile *tile, struct iosys_map *map,
1468 void *ptr, u32 qword_ofs, u32 num_qwords,
1469 const struct xe_vm_pgtable_update *update)
1471 struct xe_vma *vma = pt_update->vma;
1472 u64 empty = __xe_pt_empty_pte(tile, xe_vma_vm(vma), update->pt->level);
1475 if (map && map->is_iomem)
1476 for (i = 0; i < num_qwords; ++i)
1477 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
1478 sizeof(u64), u64, empty);
1480 memset64(map->vaddr + qword_ofs * sizeof(u64), empty,
1483 memset64(ptr, empty, num_qwords);
1487 xe_pt_commit_unbind(struct xe_vma *vma,
1488 struct xe_vm_pgtable_update *entries, u32 num_entries,
1489 struct llist_head *deferred)
1493 xe_pt_commit_locks_assert(vma);
1495 for (j = 0; j < num_entries; ++j) {
1496 struct xe_vm_pgtable_update *entry = &entries[j];
1497 struct xe_pt *pt = entry->pt;
1499 pt->num_live -= entry->qwords;
1501 struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
1504 for (i = entry->ofs; i < entry->ofs + entry->qwords;
1506 if (xe_pt_entry(pt_dir, i))
1507 xe_pt_destroy(xe_pt_entry(pt_dir, i),
1508 xe_vma_vm(vma)->flags, deferred);
1510 pt_dir->dir.entries[i] = NULL;
1516 static const struct xe_migrate_pt_update_ops unbind_ops = {
1517 .populate = xe_migrate_clear_pgtable_callback,
1518 .pre_commit = xe_pt_pre_commit,
1521 static const struct xe_migrate_pt_update_ops userptr_unbind_ops = {
1522 .populate = xe_migrate_clear_pgtable_callback,
1523 .pre_commit = xe_pt_userptr_pre_commit,
1527 * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma
1529 * @tile: The tile to unbind for.
1530 * @vma: The vma to unbind.
1531 * @q: The exec_queue with which to do pipelined page-table updates.
1532 * @syncs: Entries to sync on before disconnecting the tree to be destroyed.
1533 * @num_syncs: Number of @sync entries.
1535 * This function builds a the xe_vm_pgtable_update entries abstracting the
1536 * operations needed to detach the page-table tree to be destroyed from the
1538 * It then takes the relevant locks and submits the operations for
1539 * pipelined detachment of the gpu page-table from the vm main tree,
1540 * (which can be done either by the cpu and the GPU), Finally it frees the
1541 * detached page-table tree.
1543 * Return: A valid dma-fence representing the pipelined detachment operation
1544 * on success, an error pointer on error.
1547 __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
1548 struct xe_sync_entry *syncs, u32 num_syncs)
1550 struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
1551 struct xe_pt_migrate_pt_update unbind_pt_update = {
1553 .ops = xe_vma_is_userptr(vma) ? &userptr_unbind_ops :
1556 .tile_id = tile->id,
1559 struct xe_vm *vm = xe_vma_vm(vma);
1561 struct dma_fence *fence = NULL;
1562 struct invalidation_fence *ifence;
1563 struct xe_range_fence *rfence;
1565 LLIST_HEAD(deferred);
1567 xe_bo_assert_held(xe_vma_bo(vma));
1568 xe_vm_assert_held(vm);
1570 vm_dbg(&xe_vma_vm(vma)->xe->drm,
1571 "Preparing unbind, with range [%llx...%llx) engine %p.\n",
1572 xe_vma_start(vma), xe_vma_end(vma), q);
1574 num_entries = xe_pt_stage_unbind(tile, vma, entries);
1575 xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
1577 xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries);
1578 xe_pt_calc_rfence_interval(vma, &unbind_pt_update, entries,
1581 ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
1583 return ERR_PTR(-ENOMEM);
1585 rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
1588 return ERR_PTR(-ENOMEM);
1592 * Even if we were already evicted and unbind to destroy, we need to
1593 * clear again here. The eviction may have updated pagetables at a
1594 * lower level, because it needs to be more conservative.
1596 fence = xe_migrate_update_pgtables(tile->migrate,
1599 entries, num_entries,
1601 &unbind_pt_update.base);
1602 if (!IS_ERR(fence)) {
1605 err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
1606 &xe_range_fence_kfree_ops,
1607 unbind_pt_update.base.start,
1608 unbind_pt_update.base.last, fence);
1610 dma_fence_wait(fence, false);
1612 /* TLB invalidation must be done before signaling unbind */
1613 err = invalidation_fence_init(tile->primary_gt, ifence, fence, vma);
1615 dma_fence_put(fence);
1617 return ERR_PTR(err);
1619 fence = &ifence->base.base;
1621 /* add shared fence now for pagetable delayed destroy */
1622 dma_resv_add_fence(xe_vm_resv(vm), fence,
1623 DMA_RESV_USAGE_BOOKKEEP);
1625 /* This fence will be installed by caller when doing eviction */
1626 if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1627 dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
1628 DMA_RESV_USAGE_BOOKKEEP);
1629 xe_pt_commit_unbind(vma, entries, num_entries,
1630 unbind_pt_update.locked ? &deferred : NULL);
1631 vma->tile_present &= ~BIT(tile->id);
1637 if (!vma->tile_present)
1638 list_del_init(&vma->combined_links.rebind);
1640 if (unbind_pt_update.locked) {
1641 xe_tile_assert(tile, xe_vma_is_userptr(vma));
1643 if (!vma->tile_present) {
1644 spin_lock(&vm->userptr.invalidated_lock);
1645 list_del_init(&vma->userptr.invalidate_link);
1646 spin_unlock(&vm->userptr.invalidated_lock);
1648 up_read(&vm->userptr.notifier_lock);
1649 xe_bo_put_commit(&deferred);