drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2018 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_internal.h"
   7 #include "gem/i915_gem_object.h"
   8
   9 #include "i915_drv.h"
  10 #include "intel_engine_pm.h"
  11 #include "intel_gt_buffer_pool.h"
  12
  13 static struct list_head *
  14 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz)
  15 {
  16         int n;
  17
  18         /*
  19          * Compute a power-of-two bucket, but throw everything greater than
  20          * 16KiB into the same bucket: i.e. the buckets hold objects of
  21          * (1 page, 2 pages, 4 pages, 8+ pages).
  22          */
  23         n = fls(sz >> PAGE_SHIFT) - 1;
  24         if (n >= ARRAY_SIZE(pool->cache_list))
  25                 n = ARRAY_SIZE(pool->cache_list) - 1;
  26
  27         return &pool->cache_list[n];
  28 }
  29
  30 static void node_free(struct intel_gt_buffer_pool_node *node)
  31 {
  32         i915_gem_object_put(node->obj);
  33         i915_active_fini(&node->active);
  34         kfree_rcu(node, rcu);
  35 }
  36
  37 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep)
  38 {
  39         struct intel_gt_buffer_pool_node *node, *stale = NULL;
  40         bool active = false;
  41         int n;
  42
  43         /* Free buffers that have not been used in the past second */
  44         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
  45                 struct list_head *list = &pool->cache_list[n];
  46
  47                 if (list_empty(list))
  48                         continue;
  49
  50                 if (spin_trylock_irq(&pool->lock)) {
  51                         struct list_head *pos;
  52
  53                         /* Most recent at head; oldest at tail */
  54                         list_for_each_prev(pos, list) {
  55                                 unsigned long age;
  56
  57                                 node = list_entry(pos, typeof(*node), link);
  58
  59                                 age = READ_ONCE(node->age);
  60                                 if (!age || jiffies - age < keep)
  61                                         break;
  62
  63                                 /* Check we are the first to claim this node */
  64                                 if (!xchg(&node->age, 0))
  65                                         break;
  66
  67                                 node->free = stale;
  68                                 stale = node;
  69                         }
  70                         if (!list_is_last(pos, list))
  71                                 __list_del_many(pos, list);
  72
  73                         spin_unlock_irq(&pool->lock);
  74                 }
  75
  76                 active |= !list_empty(list);
  77         }
  78
  79         while ((node = stale)) {
  80                 stale = stale->free;
  81                 node_free(node);
  82         }
  83
  84         return active;
  85 }
  86
  87 static void pool_free_work(struct work_struct *wrk)
  88 {
  89         struct intel_gt_buffer_pool *pool =
  90                 container_of(wrk, typeof(*pool), work.work);
  91
  92         if (pool_free_older_than(pool, HZ))
  93                 schedule_delayed_work(&pool->work,
  94                                       round_jiffies_up_relative(HZ));
  95 }
  96
  97 static void pool_retire(struct i915_active *ref)
  98 {
  99         struct intel_gt_buffer_pool_node *node =
 100                 container_of(ref, typeof(*node), active);
 101         struct intel_gt_buffer_pool *pool = node->pool;
 102         struct list_head *list = bucket_for_size(pool, node->obj->base.size);
 103         unsigned long flags;
 104
 105         if (node->pinned) {
 106                 i915_gem_object_unpin_pages(node->obj);
 107
 108                 /* Return this object to the shrinker pool */
 109                 i915_gem_object_make_purgeable(node->obj);
 110                 node->pinned = false;
 111         }
 112
 113         GEM_BUG_ON(node->age);
 114         spin_lock_irqsave(&pool->lock, flags);
 115         list_add_rcu(&node->link, list);
 116         WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */
 117         spin_unlock_irqrestore(&pool->lock, flags);
 118
 119         schedule_delayed_work(&pool->work,
 120                               round_jiffies_up_relative(HZ));
 121 }
 122
 123 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node)
 124 {
 125         assert_object_held(node->obj);
 126
 127         if (node->pinned)
 128                 return;
 129
 130         __i915_gem_object_pin_pages(node->obj);
 131         /* Hide this pinned object from the shrinker until retired */
 132         i915_gem_object_make_unshrinkable(node->obj);
 133         node->pinned = true;
 134 }
 135
 136 static struct intel_gt_buffer_pool_node *
 137 node_create(struct intel_gt_buffer_pool *pool, size_t sz,
 138             enum i915_map_type type)
 139 {
 140         struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
 141         struct intel_gt_buffer_pool_node *node;
 142         struct drm_i915_gem_object *obj;
 143
 144         node = kmalloc(sizeof(*node),
 145                        GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 146         if (!node)
 147                 return ERR_PTR(-ENOMEM);
 148
 149         node->age = 0;
 150         node->pool = pool;
 151         node->pinned = false;
 152         i915_active_init(&node->active, NULL, pool_retire, 0);
 153
 154         obj = i915_gem_object_create_internal(gt->i915, sz);
 155         if (IS_ERR(obj)) {
 156                 i915_active_fini(&node->active);
 157                 kfree(node);
 158                 return ERR_CAST(obj);
 159         }
 160
 161         i915_gem_object_set_readonly(obj);
 162
 163         node->type = type;
 164         node->obj = obj;
 165         return node;
 166 }
 167
 168 struct intel_gt_buffer_pool_node *
 169 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size,
 170                          enum i915_map_type type)
 171 {
 172         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 173         struct intel_gt_buffer_pool_node *node;
 174         struct list_head *list;
 175         int ret;
 176
 177         size = PAGE_ALIGN(size);
 178         list = bucket_for_size(pool, size);
 179
 180         rcu_read_lock();
 181         list_for_each_entry_rcu(node, list, link) {
 182                 unsigned long age;
 183
 184                 if (node->obj->base.size < size)
 185                         continue;
 186
 187                 if (node->type != type)
 188                         continue;
 189
 190                 age = READ_ONCE(node->age);
 191                 if (!age)
 192                         continue;
 193
 194                 if (cmpxchg(&node->age, age, 0) == age) {
 195                         spin_lock_irq(&pool->lock);
 196                         list_del_rcu(&node->link);
 197                         spin_unlock_irq(&pool->lock);
 198                         break;
 199                 }
 200         }
 201         rcu_read_unlock();
 202
 203         if (&node->link == list) {
 204                 node = node_create(pool, size, type);
 205                 if (IS_ERR(node))
 206                         return node;
 207         }
 208
 209         ret = i915_active_acquire(&node->active);
 210         if (ret) {
 211                 node_free(node);
 212                 return ERR_PTR(ret);
 213         }
 214
 215         return node;
 216 }
 217
 218 void intel_gt_init_buffer_pool(struct intel_gt *gt)
 219 {
 220         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 221         int n;
 222
 223         spin_lock_init(&pool->lock);
 224         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 225                 INIT_LIST_HEAD(&pool->cache_list[n]);
 226         INIT_DELAYED_WORK(&pool->work, pool_free_work);
 227 }
 228
 229 void intel_gt_flush_buffer_pool(struct intel_gt *gt)
 230 {
 231         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 232
 233         do {
 234                 while (pool_free_older_than(pool, 0))
 235                         ;
 236         } while (cancel_delayed_work_sync(&pool->work));
 237 }
 238
 239 void intel_gt_fini_buffer_pool(struct intel_gt *gt)
 240 {
 241         struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
 242         int n;
 243
 244         for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
 245                 GEM_BUG_ON(!list_empty(&pool->cache_list[n]));
 246 }