drivers/gpu/drm/msm/msm_gem_shrinker.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2016 Red Hat
   4  * Author: Rob Clark <[email protected]>
   5  */
   6
   7 #include <linux/vmalloc.h>
   8 #include <linux/sched/mm.h>
   9
  10 #include "msm_drv.h"
  11 #include "msm_gem.h"
  12 #include "msm_gpu.h"
  13 #include "msm_gpu_trace.h"
  14
  15 /* Default disabled for now until it has some more testing on the different
  16  * iommu combinations that can be paired with the driver:
  17  */
  18 bool enable_eviction = false;
  19 MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers");
  20 module_param(enable_eviction, bool, 0600);
  21
  22 static bool can_swap(void)
  23 {
  24         return enable_eviction && get_nr_swap_pages() > 0;
  25 }
  26
  27 static unsigned long
  28 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
  29 {
  30         struct msm_drm_private *priv =
  31                 container_of(shrinker, struct msm_drm_private, shrinker);
  32         unsigned count = priv->shrinkable_count;
  33
  34         if (can_swap())
  35                 count += priv->evictable_count;
  36
  37         return count;
  38 }
  39
  40 static bool
  41 purge(struct msm_gem_object *msm_obj)
  42 {
  43         if (!is_purgeable(msm_obj))
  44                 return false;
  45
  46         /*
  47          * This will move the obj out of still_in_list to
  48          * the purged list
  49          */
  50         msm_gem_purge(&msm_obj->base);
  51
  52         return true;
  53 }
  54
  55 static bool
  56 evict(struct msm_gem_object *msm_obj)
  57 {
  58         if (is_unevictable(msm_obj))
  59                 return false;
  60
  61         msm_gem_evict(&msm_obj->base);
  62
  63         return true;
  64 }
  65
  66 static unsigned long
  67 scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list,
  68                 bool (*shrink)(struct msm_gem_object *msm_obj))
  69 {
  70         unsigned freed = 0;
  71         struct list_head still_in_list;
  72
  73         INIT_LIST_HEAD(&still_in_list);
  74
  75         mutex_lock(&priv->mm_lock);
  76
  77         while (freed < nr_to_scan) {
  78                 struct msm_gem_object *msm_obj = list_first_entry_or_null(
  79                                 list, typeof(*msm_obj), mm_list);
  80
  81                 if (!msm_obj)
  82                         break;
  83
  84                 list_move_tail(&msm_obj->mm_list, &still_in_list);
  85
  86                 /*
  87                  * If it is in the process of being freed, msm_gem_free_object
  88                  * can be blocked on mm_lock waiting to remove it.  So just
  89                  * skip it.
  90                  */
  91                 if (!kref_get_unless_zero(&msm_obj->base.refcount))
  92                         continue;
  93
  94                 /*
  95                  * Now that we own a reference, we can drop mm_lock for the
  96                  * rest of the loop body, to reduce contention with the
  97                  * retire_submit path (which could make more objects purgeable)
  98                  */
  99
 100                 mutex_unlock(&priv->mm_lock);
 101
 102                 /*
 103                  * Note that this still needs to be trylock, since we can
 104                  * hit shrinker in response to trying to get backing pages
 105                  * for this obj (ie. while it's lock is already held)
 106                  */
 107                 if (!msm_gem_trylock(&msm_obj->base))
 108                         goto tail;
 109
 110                 if (shrink(msm_obj))
 111                         freed += msm_obj->base.size >> PAGE_SHIFT;
 112
 113                 msm_gem_unlock(&msm_obj->base);
 114
 115 tail:
 116                 drm_gem_object_put(&msm_obj->base);
 117                 mutex_lock(&priv->mm_lock);
 118         }
 119
 120         list_splice_tail(&still_in_list, list);
 121         mutex_unlock(&priv->mm_lock);
 122
 123         return freed;
 124 }
 125
 126 static unsigned long
 127 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 128 {
 129         struct msm_drm_private *priv =
 130                 container_of(shrinker, struct msm_drm_private, shrinker);
 131         unsigned long freed;
 132
 133         freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge);
 134
 135         if (freed > 0)
 136                 trace_msm_gem_purge(freed << PAGE_SHIFT);
 137
 138         if (can_swap() && freed < sc->nr_to_scan) {
 139                 int evicted = scan(priv, sc->nr_to_scan - freed,
 140                                 &priv->inactive_willneed, evict);
 141
 142                 if (evicted > 0)
 143                         trace_msm_gem_evict(evicted << PAGE_SHIFT);
 144
 145                 freed += evicted;
 146         }
 147
 148         return (freed > 0) ? freed : SHRINK_STOP;
 149 }
 150
 151 #ifdef CONFIG_DEBUG_FS
 152 unsigned long
 153 msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
 154 {
 155         struct msm_drm_private *priv = dev->dev_private;
 156         struct shrink_control sc = {
 157                 .nr_to_scan = nr_to_scan,
 158         };
 159         int ret;
 160
 161         fs_reclaim_acquire(GFP_KERNEL);
 162         ret = msm_gem_shrinker_scan(&priv->shrinker, &sc);
 163         fs_reclaim_release(GFP_KERNEL);
 164
 165         return ret;
 166 }
 167 #endif
 168
 169 /* since we don't know any better, lets bail after a few
 170  * and if necessary the shrinker will be invoked again.
 171  * Seems better than unmapping *everything*
 172  */
 173 static const int vmap_shrink_limit = 15;
 174
 175 static bool
 176 vmap_shrink(struct msm_gem_object *msm_obj)
 177 {
 178         if (!is_vunmapable(msm_obj))
 179                 return false;
 180
 181         msm_gem_vunmap(&msm_obj->base);
 182
 183         return true;
 184 }
 185
 186 static int
 187 msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
 188 {
 189         struct msm_drm_private *priv =
 190                 container_of(nb, struct msm_drm_private, vmap_notifier);
 191         struct list_head *mm_lists[] = {
 192                 &priv->inactive_dontneed,
 193                 &priv->inactive_willneed,
 194                 priv->gpu ? &priv->gpu->active_list : NULL,
 195                 NULL,
 196         };
 197         unsigned idx, unmapped = 0;
 198
 199         for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) {
 200                 unmapped += scan(priv, vmap_shrink_limit - unmapped,
 201                                 mm_lists[idx], vmap_shrink);
 202         }
 203
 204         *(unsigned long *)ptr += unmapped;
 205
 206         if (unmapped > 0)
 207                 trace_msm_gem_purge_vmaps(unmapped);
 208
 209         return NOTIFY_DONE;
 210 }
 211
 212 /**
 213  * msm_gem_shrinker_init - Initialize msm shrinker
 214  * @dev: drm device
 215  *
 216  * This function registers and sets up the msm shrinker.
 217  */
 218 void msm_gem_shrinker_init(struct drm_device *dev)
 219 {
 220         struct msm_drm_private *priv = dev->dev_private;
 221         priv->shrinker.count_objects = msm_gem_shrinker_count;
 222         priv->shrinker.scan_objects = msm_gem_shrinker_scan;
 223         priv->shrinker.seeks = DEFAULT_SEEKS;
 224         WARN_ON(register_shrinker(&priv->shrinker));
 225
 226         priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
 227         WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
 228 }
 229
 230 /**
 231  * msm_gem_shrinker_cleanup - Clean up msm shrinker
 232  * @dev: drm device
 233  *
 234  * This function unregisters the msm shrinker.
 235  */
 236 void msm_gem_shrinker_cleanup(struct drm_device *dev)
 237 {
 238         struct msm_drm_private *priv = dev->dev_private;
 239
 240         if (priv->shrinker.nr_deferred) {
 241                 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
 242                 unregister_shrinker(&priv->shrinker);
 243         }
 244 }