drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2018 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_internal.h"
   7 #include "gem/i915_gem_object.h"
   8
   9 #include "i915_drv.h"
  10 #include "intel_engine_pm.h"
  11 #include "intel_gt_buffer_pool.h"
  12
  13 static struct list_head *
  14 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz)
  15 {
  16         int n;
  17
  18         /*
  19          * Compute a power-of-two bucket, but throw everything greater than
  20          * 16KiB into the same bucket: i.e. the buckets hold objects of
  21          * (1 page, 2 pages, 4 pages, 8+ pages).
  22          */
  23         n = fls(sz >> PAGE_SHIFT) - 1;
  24         if (n >= ARRAY_SIZE(pool->cache_list))
  25                 n = ARRAY_SIZE(pool->cache_list) - 1;
  26
  27         return &pool->cache_list[n];
  28 }
  29
  30 static void node_free(struct intel_gt_buffer_pool_node *node)
  31 {
  32         i915_gem_object_put(node->obj);
  33         i915_active_fini(&node->active);
  34         kfree_rcu(node, rcu);
  35 }
  36
  37 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep)
  38 {
  39         struct intel_gt_buffer_pool_node *node, *stale = NULL;
  40         bool active = false;
  41         int n;
  42
  43         /* Free buffers that have not been used in the past second */
  44         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
  45                 struct list_head *list = &pool->cache_list[n];
  46
  47                 if (list_empty(list))
  48                         continue;
  49
  50                 if (spin_trylock_irq(&pool->lock)) {
  51                         struct list_head *pos;
  52
  53                         /* Most recent at head; oldest at tail */
  54                         list_for_each_prev(pos, list) {
  55                                 unsigned long age;
  56
  57                                 node = list_entry(pos, typeof(*node), link);
  58
  59                                 age = READ_ONCE(node->age);
  60                                 if (!age || jiffies - age < keep)
  61                                         break;
  62
  63                                 /* Check we are the first to claim this node */
  64                                 if (!xchg(&node->age, 0))
  65                                         break;
  66
  67                                 node->free = stale;
  68                                 stale = node;
  69                         }
  70                         if (!list_is_last(pos, list))
  71                                 __list_del_many(pos, list);
  72
  73                         spin_unlock_irq(&pool->lock);
  74                 }
  75
  76                 active |= !list_empty(list);
  77         }
  78
  79         while ((node = stale)) {
  80                 stale = stale->free;
  81                 node_free(node);
  82         }
  83
  84         return active;
  85 }
  86
  87 static void pool_free_work(struct work_struct *wrk)
  88 {
  89         struct intel_gt_buffer_pool *pool =
  90                 container_of(wrk, typeof(*pool), work.work);
  91         struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
  92
  93         if (pool_free_older_than(pool, HZ))
  94                 queue_delayed_work(gt->i915->unordered_wq, &pool->work,
  95                                    round_jiffies_up_relative(HZ));
  96 }
  97
  98 static void pool_retire(struct i915_active *ref)
  99 {
 100         struct intel_gt_buffer_pool_node *node =
 101                 container_of(ref, typeof(*node), active);
 102         struct intel_gt_buffer_pool *pool = node->pool;
 103         struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
 104         struct list_head *list = bucket_for_size(pool, node->obj->base.size);
 105         unsigned long flags;
 106
 107         if (node->pinned) {
 108                 i915_gem_object_unpin_pages(node->obj);
 109
 110                 /* Return this object to the shrinker pool */
 111                 i915_gem_object_make_purgeable(node->obj);
 112                 node->pinned = false;
 113         }
 114
 115         GEM_BUG_ON(node->age);
 116         spin_lock_irqsave(&pool->lock, flags);
 117         list_add_rcu(&node->link, list);
 118         WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */
 119         spin_unlock_irqrestore(&pool->lock, flags);
 120
 121         queue_delayed_work(gt->i915->unordered_wq, &pool->work,
 122                            round_jiffies_up_relative(HZ));
 123 }
 124
 125 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node)
 126 {
 127         assert_object_held(node->obj);
 128
 129         if (node->pinned)
 130                 return;
 131
 132         __i915_gem_object_pin_pages(node->obj);
 133         /* Hide this pinned object from the shrinker until retired */
 134         i915_gem_object_make_unshrinkable(node->obj);
 135         node->pinned = true;
 136 }
 137
 138 static struct intel_gt_buffer_pool_node *
 139 node_create(struct intel_gt_buffer_pool *pool, size_t sz,
 140             enum i915_map_type type)
 141 {
 142         struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
 143         struct intel_gt_buffer_pool_node *node;
 144         struct drm_i915_gem_object *obj;
 145
 146         node = kmalloc(sizeof(*node),
 147                        GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 148         if (!node)
 149                 return ERR_PTR(-ENOMEM);
 150
 151         node->age = 0;
 152         node->pool = pool;
 153         node->pinned = false;
 154         i915_active_init(&node->active, NULL, pool_retire, 0);
 155
 156         obj = i915_gem_object_create_internal(gt->i915, sz);
 157         if (IS_ERR(obj)) {
 158                 i915_active_fini(&node->active);
 159                 kfree(node);
 160                 return ERR_CAST(obj);
 161         }
 162
 163         i915_gem_object_set_readonly(obj);
 164
 165         node->type = type;
 166         node->obj = obj;
 167         return node;
 168 }
 169
 170 struct intel_gt_buffer_pool_node *
 171 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size,
 172                          enum i915_map_type type)
 173 {
 174         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 175         struct intel_gt_buffer_pool_node *node;
 176         struct list_head *list;
 177         int ret;
 178
 179         size = PAGE_ALIGN(size);
 180         list = bucket_for_size(pool, size);
 181
 182         rcu_read_lock();
 183         list_for_each_entry_rcu(node, list, link) {
 184                 unsigned long age;
 185
 186                 if (node->obj->base.size < size)
 187                         continue;
 188
 189                 if (node->type != type)
 190                         continue;
 191
 192                 age = READ_ONCE(node->age);
 193                 if (!age)
 194                         continue;
 195
 196                 if (cmpxchg(&node->age, age, 0) == age) {
 197                         spin_lock_irq(&pool->lock);
 198                         list_del_rcu(&node->link);
 199                         spin_unlock_irq(&pool->lock);
 200                         break;
 201                 }
 202         }
 203         rcu_read_unlock();
 204
 205         if (&node->link == list) {
 206                 node = node_create(pool, size, type);
 207                 if (IS_ERR(node))
 208                         return node;
 209         }
 210
 211         ret = i915_active_acquire(&node->active);
 212         if (ret) {
 213                 node_free(node);
 214                 return ERR_PTR(ret);
 215         }
 216
 217         return node;
 218 }
 219
 220 void intel_gt_init_buffer_pool(struct intel_gt *gt)
 221 {
 222         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 223         int n;
 224
 225         spin_lock_init(&pool->lock);
 226         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 227                 INIT_LIST_HEAD(&pool->cache_list[n]);
 228         INIT_DELAYED_WORK(&pool->work, pool_free_work);
 229 }
 230
 231 void intel_gt_flush_buffer_pool(struct intel_gt *gt)
 232 {
 233         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 234
 235         do {
 236                 while (pool_free_older_than(pool, 0))
 237                         ;
 238         } while (cancel_delayed_work_sync(&pool->work));
 239 }
 240
 241 void intel_gt_fini_buffer_pool(struct intel_gt *gt)
 242 {
 243         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 244         int n;
 245
 246         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 247                 GEM_BUG_ON(!list_empty(&pool->cache_list[n]));
 248 }