drivers/gpu/drm/i915/i915_vma_resource.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include <linux/interval_tree_generic.h>
   7 #include <linux/sched/mm.h>
   8
   9 #include "i915_sw_fence.h"
  10 #include "i915_vma_resource.h"
  11 #include "i915_drv.h"
  12 #include "intel_memory_region.h"
  13
  14 #include "gt/intel_gtt.h"
  15
  16 static struct kmem_cache *slab_vma_resources;
  17
  18 /**
  19  * DOC:
  20  * We use a per-vm interval tree to keep track of vma_resources
  21  * scheduled for unbind but not yet unbound. The tree is protected by
  22  * the vm mutex, and nodes are removed just after the unbind fence signals.
  23  * The removal takes the vm mutex from a kernel thread which we need to
  24  * keep in mind so that we don't grab the mutex and try to wait for all
  25  * pending unbinds to complete, because that will temporaryily block many
  26  * of the workqueue threads, and people will get angry.
  27  *
  28  * We should consider using a single ordered fence per VM instead but that
  29  * requires ordering the unbinds and might introduce unnecessary waiting
  30  * for unrelated unbinds. Amount of code will probably be roughly the same
  31  * due to the simplicity of using the interval tree interface.
  32  *
  33  * Another drawback of this interval tree is that the complexity of insertion
  34  * and removal of fences increases as O(ln(pending_unbinds)) instead of
  35  * O(1) for a single fence without interval tree.
  36  */
  37 #define VMA_RES_START(_node) ((_node)->start - (_node)->guard)
  38 #define VMA_RES_LAST(_node) ((_node)->start + (_node)->node_size + (_node)->guard - 1)
  39 INTERVAL_TREE_DEFINE(struct i915_vma_resource, rb,
  40                      u64, __subtree_last,
  41                      VMA_RES_START, VMA_RES_LAST, static, vma_res_itree);
  42
  43 /* Callbacks for the unbind dma-fence. */
  44
  45 /**
  46  * i915_vma_resource_alloc - Allocate a vma resource
  47  *
  48  * Return: A pointer to a cleared struct i915_vma_resource or
  49  * a -ENOMEM error pointer if allocation fails.
  50  */
  51 struct i915_vma_resource *i915_vma_resource_alloc(void)
  52 {
  53         struct i915_vma_resource *vma_res =
  54                 kmem_cache_zalloc(slab_vma_resources, GFP_KERNEL);
  55
  56         return vma_res ? vma_res : ERR_PTR(-ENOMEM);
  57 }
  58
  59 /**
  60  * i915_vma_resource_free - Free a vma resource
  61  * @vma_res: The vma resource to free.
  62  */
  63 void i915_vma_resource_free(struct i915_vma_resource *vma_res)
  64 {
  65         if (vma_res)
  66                 kmem_cache_free(slab_vma_resources, vma_res);
  67 }
  68
  69 static const char *get_driver_name(struct dma_fence *fence)
  70 {
  71         return "vma unbind fence";
  72 }
  73
  74 static const char *get_timeline_name(struct dma_fence *fence)
  75 {
  76         return "unbound";
  77 }
  78
  79 static void unbind_fence_free_rcu(struct rcu_head *head)
  80 {
  81         struct i915_vma_resource *vma_res =
  82                 container_of(head, typeof(*vma_res), unbind_fence.rcu);
  83
  84         i915_vma_resource_free(vma_res);
  85 }
  86
  87 static void unbind_fence_release(struct dma_fence *fence)
  88 {
  89         struct i915_vma_resource *vma_res =
  90                 container_of(fence, typeof(*vma_res), unbind_fence);
  91
  92         i915_sw_fence_fini(&vma_res->chain);
  93
  94         call_rcu(&fence->rcu, unbind_fence_free_rcu);
  95 }
  96
  97 static const struct dma_fence_ops unbind_fence_ops = {
  98         .get_driver_name = get_driver_name,
  99         .get_timeline_name = get_timeline_name,
 100         .release = unbind_fence_release,
 101 };
 102
 103 static void __i915_vma_resource_unhold(struct i915_vma_resource *vma_res)
 104 {
 105         struct i915_address_space *vm;
 106
 107         if (!refcount_dec_and_test(&vma_res->hold_count))
 108                 return;
 109
 110         dma_fence_signal(&vma_res->unbind_fence);
 111
 112         vm = vma_res->vm;
 113         if (vma_res->wakeref)
 114                 intel_runtime_pm_put(&vm->i915->runtime_pm, vma_res->wakeref);
 115
 116         vma_res->vm = NULL;
 117         if (!RB_EMPTY_NODE(&vma_res->rb)) {
 118                 mutex_lock(&vm->mutex);
 119                 vma_res_itree_remove(vma_res, &vm->pending_unbind);
 120                 mutex_unlock(&vm->mutex);
 121         }
 122
 123         if (vma_res->bi.pages_rsgt)
 124                 i915_refct_sgt_put(vma_res->bi.pages_rsgt);
 125 }
 126
 127 /**
 128  * i915_vma_resource_unhold - Unhold the signaling of the vma resource unbind
 129  * fence.
 130  * @vma_res: The vma resource.
 131  * @lockdep_cookie: The lockdep cookie returned from i915_vma_resource_hold.
 132  *
 133  * The function may leave a dma_fence critical section.
 134  */
 135 void i915_vma_resource_unhold(struct i915_vma_resource *vma_res,
 136                               bool lockdep_cookie)
 137 {
 138         dma_fence_end_signalling(lockdep_cookie);
 139
 140         if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
 141                 unsigned long irq_flags;
 142
 143                 /* Inefficient open-coded might_lock_irqsave() */
 144                 spin_lock_irqsave(&vma_res->lock, irq_flags);
 145                 spin_unlock_irqrestore(&vma_res->lock, irq_flags);
 146         }
 147
 148         __i915_vma_resource_unhold(vma_res);
 149 }
 150
 151 /**
 152  * i915_vma_resource_hold - Hold the signaling of the vma resource unbind fence.
 153  * @vma_res: The vma resource.
 154  * @lockdep_cookie: Pointer to a bool serving as a lockdep cooke that should
 155  * be given as an argument to the pairing i915_vma_resource_unhold.
 156  *
 157  * If returning true, the function enters a dma_fence signalling critical
 158  * section if not in one already.
 159  *
 160  * Return: true if holding successful, false if not.
 161  */
 162 bool i915_vma_resource_hold(struct i915_vma_resource *vma_res,
 163                             bool *lockdep_cookie)
 164 {
 165         bool held = refcount_inc_not_zero(&vma_res->hold_count);
 166
 167         if (held)
 168                 *lockdep_cookie = dma_fence_begin_signalling();
 169
 170         return held;
 171 }
 172
 173 static void i915_vma_resource_unbind_work(struct work_struct *work)
 174 {
 175         struct i915_vma_resource *vma_res =
 176                 container_of(work, typeof(*vma_res), work);
 177         struct i915_address_space *vm = vma_res->vm;
 178         bool lockdep_cookie;
 179
 180         lockdep_cookie = dma_fence_begin_signalling();
 181         if (likely(!vma_res->skip_pte_rewrite))
 182                 vma_res->ops->unbind_vma(vm, vma_res);
 183
 184         dma_fence_end_signalling(lockdep_cookie);
 185         __i915_vma_resource_unhold(vma_res);
 186         i915_vma_resource_put(vma_res);
 187 }
 188
 189 static int
 190 i915_vma_resource_fence_notify(struct i915_sw_fence *fence,
 191                                enum i915_sw_fence_notify state)
 192 {
 193         struct i915_vma_resource *vma_res =
 194                 container_of(fence, typeof(*vma_res), chain);
 195         struct dma_fence *unbind_fence =
 196                 &vma_res->unbind_fence;
 197
 198         switch (state) {
 199         case FENCE_COMPLETE:
 200                 dma_fence_get(unbind_fence);
 201                 if (vma_res->immediate_unbind) {
 202                         i915_vma_resource_unbind_work(&vma_res->work);
 203                 } else {
 204                         INIT_WORK(&vma_res->work, i915_vma_resource_unbind_work);
 205                         queue_work(system_unbound_wq, &vma_res->work);
 206                 }
 207                 break;
 208         case FENCE_FREE:
 209                 i915_vma_resource_put(vma_res);
 210                 break;
 211         }
 212
 213         return NOTIFY_DONE;
 214 }
 215
 216 /**
 217  * i915_vma_resource_unbind - Unbind a vma resource
 218  * @vma_res: The vma resource to unbind.
 219  * @tlb: pointer to vma->obj->mm.tlb associated with the resource
 220  *       to be stored at vma_res->tlb. When not-NULL, it will be used
 221  *       to do TLB cache invalidation before freeing a VMA resource.
 222  *       Used only for async unbind.
 223  *
 224  * At this point this function does little more than publish a fence that
 225  * signals immediately unless signaling is held back.
 226  *
 227  * Return: A refcounted pointer to a dma-fence that signals when unbinding is
 228  * complete.
 229  */
 230 struct dma_fence *i915_vma_resource_unbind(struct i915_vma_resource *vma_res,
 231                                            u32 *tlb)
 232 {
 233         struct i915_address_space *vm = vma_res->vm;
 234
 235         vma_res->tlb = tlb;
 236
 237         /* Reference for the sw fence */
 238         i915_vma_resource_get(vma_res);
 239
 240         /* Caller must already have a wakeref in this case. */
 241         if (vma_res->needs_wakeref)
 242                 vma_res->wakeref = intel_runtime_pm_get_if_in_use(&vm->i915->runtime_pm);
 243
 244         if (atomic_read(&vma_res->chain.pending) <= 1) {
 245                 RB_CLEAR_NODE(&vma_res->rb);
 246                 vma_res->immediate_unbind = 1;
 247         } else {
 248                 vma_res_itree_insert(vma_res, &vma_res->vm->pending_unbind);
 249         }
 250
 251         i915_sw_fence_commit(&vma_res->chain);
 252
 253         return &vma_res->unbind_fence;
 254 }
 255
 256 /**
 257  * __i915_vma_resource_init - Initialize a vma resource.
 258  * @vma_res: The vma resource to initialize
 259  *
 260  * Initializes the private members of a vma resource.
 261  */
 262 void __i915_vma_resource_init(struct i915_vma_resource *vma_res)
 263 {
 264         spin_lock_init(&vma_res->lock);
 265         dma_fence_init(&vma_res->unbind_fence, &unbind_fence_ops,
 266                        &vma_res->lock, 0, 0);
 267         refcount_set(&vma_res->hold_count, 1);
 268         i915_sw_fence_init(&vma_res->chain, i915_vma_resource_fence_notify);
 269 }
 270
 271 static void
 272 i915_vma_resource_color_adjust_range(struct i915_address_space *vm,
 273                                      u64 *start,
 274                                      u64 *end)
 275 {
 276         if (i915_vm_has_cache_coloring(vm)) {
 277                 if (*start)
 278                         *start -= I915_GTT_PAGE_SIZE;
 279                 *end += I915_GTT_PAGE_SIZE;
 280         }
 281 }
 282
 283 /**
 284  * i915_vma_resource_bind_dep_sync - Wait for / sync all unbinds touching a
 285  * certain vm range.
 286  * @vm: The vm to look at.
 287  * @offset: The range start.
 288  * @size: The range size.
 289  * @intr: Whether to wait interrubtible.
 290  *
 291  * The function needs to be called with the vm lock held.
 292  *
 293  * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true
 294  */
 295 int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm,
 296                                     u64 offset,
 297                                     u64 size,
 298                                     bool intr)
 299 {
 300         struct i915_vma_resource *node;
 301         u64 last = offset + size - 1;
 302
 303         lockdep_assert_held(&vm->mutex);
 304         might_sleep();
 305
 306         i915_vma_resource_color_adjust_range(vm, &offset, &last);
 307         node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last);
 308         while (node) {
 309                 int ret = dma_fence_wait(&node->unbind_fence, intr);
 310
 311                 if (ret)
 312                         return ret;
 313
 314                 node = vma_res_itree_iter_next(node, offset, last);
 315         }
 316
 317         return 0;
 318 }
 319
 320 /**
 321  * i915_vma_resource_bind_dep_sync_all - Wait for / sync all unbinds of a vm,
 322  * releasing the vm lock while waiting.
 323  * @vm: The vm to look at.
 324  *
 325  * The function may not be called with the vm lock held.
 326  * Typically this is called at vm destruction to finish any pending
 327  * unbind operations. The vm mutex is released while waiting to avoid
 328  * stalling kernel workqueues trying to grab the mutex.
 329  */
 330 void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm)
 331 {
 332         struct i915_vma_resource *node;
 333         struct dma_fence *fence;
 334
 335         do {
 336                 fence = NULL;
 337                 mutex_lock(&vm->mutex);
 338                 node = vma_res_itree_iter_first(&vm->pending_unbind, 0,
 339                                                 U64_MAX);
 340                 if (node)
 341                         fence = dma_fence_get_rcu(&node->unbind_fence);
 342                 mutex_unlock(&vm->mutex);
 343
 344                 if (fence) {
 345                         /*
 346                          * The wait makes sure the node eventually removes
 347                          * itself from the tree.
 348                          */
 349                         dma_fence_wait(fence, false);
 350                         dma_fence_put(fence);
 351                 }
 352         } while (node);
 353 }
 354
 355 /**
 356  * i915_vma_resource_bind_dep_await - Have a struct i915_sw_fence await all
 357  * pending unbinds in a certain range of a vm.
 358  * @vm: The vm to look at.
 359  * @sw_fence: The struct i915_sw_fence that will be awaiting the unbinds.
 360  * @offset: The range start.
 361  * @size: The range size.
 362  * @intr: Whether to wait interrubtible.
 363  * @gfp: Allocation mode for memory allocations.
 364  *
 365  * The function makes @sw_fence await all pending unbinds in a certain
 366  * vm range before calling the complete notifier. To be able to await
 367  * each individual unbind, the function needs to allocate memory using
 368  * the @gpf allocation mode. If that fails, the function will instead
 369  * wait for the unbind fence to signal, using @intr to judge whether to
 370  * wait interruptible or not. Note that @gfp should ideally be selected so
 371  * as to avoid any expensive memory allocation stalls and rather fail and
 372  * synchronize itself. For now the vm mutex is required when calling this
 373  * function with means that @gfp can't call into direct reclaim. In reality
 374  * this means that during heavy memory pressure, we will sync in this
 375  * function.
 376  *
 377  * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true
 378  */
 379 int i915_vma_resource_bind_dep_await(struct i915_address_space *vm,
 380                                      struct i915_sw_fence *sw_fence,
 381                                      u64 offset,
 382                                      u64 size,
 383                                      bool intr,
 384                                      gfp_t gfp)
 385 {
 386         struct i915_vma_resource *node;
 387         u64 last = offset + size - 1;
 388
 389         lockdep_assert_held(&vm->mutex);
 390         might_alloc(gfp);
 391         might_sleep();
 392
 393         i915_vma_resource_color_adjust_range(vm, &offset, &last);
 394         node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last);
 395         while (node) {
 396                 int ret;
 397
 398                 ret = i915_sw_fence_await_dma_fence(sw_fence,
 399                                                     &node->unbind_fence,
 400                                                     0, gfp);
 401                 if (ret < 0) {
 402                         ret = dma_fence_wait(&node->unbind_fence, intr);
 403                         if (ret)
 404                                 return ret;
 405                 }
 406
 407                 node = vma_res_itree_iter_next(node, offset, last);
 408         }
 409
 410         return 0;
 411 }
 412
 413 void i915_vma_resource_module_exit(void)
 414 {
 415         kmem_cache_destroy(slab_vma_resources);
 416 }
 417
 418 int __init i915_vma_resource_module_init(void)
 419 {
 420         slab_vma_resources = KMEM_CACHE(i915_vma_resource, SLAB_HWCACHE_ALIGN);
 421         if (!slab_vma_resources)
 422                 return -ENOMEM;
 423
 424         return 0;
 425 }