drivers/gpu/drm/ttm/ttm_resource.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/dma-buf-map.h>
  26 #include <linux/io-mapping.h>
  27 #include <linux/scatterlist.h>
  28
  29 #include <drm/ttm/ttm_resource.h>
  30 #include <drm/ttm/ttm_bo_driver.h>
  31
  32 void ttm_resource_init(struct ttm_buffer_object *bo,
  33                        const struct ttm_place *place,
  34                        struct ttm_resource *res)
  35 {
  36         res->start = 0;
  37         res->num_pages = PFN_UP(bo->base.size);
  38         res->mem_type = place->mem_type;
  39         res->placement = place->flags;
  40         res->bus.addr = NULL;
  41         res->bus.offset = 0;
  42         res->bus.is_iomem = false;
  43         res->bus.caching = ttm_cached;
  44 }
  45 EXPORT_SYMBOL(ttm_resource_init);
  46
  47 int ttm_resource_alloc(struct ttm_buffer_object *bo,
  48                        const struct ttm_place *place,
  49                        struct ttm_resource **res_ptr)
  50 {
  51         struct ttm_resource_manager *man =
  52                 ttm_manager_type(bo->bdev, place->mem_type);
  53
  54         return man->func->alloc(man, bo, place, res_ptr);
  55 }
  56
  57 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
  58 {
  59         struct ttm_resource_manager *man;
  60
  61         if (!*res)
  62                 return;
  63
  64         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
  65         man->func->free(man, *res);
  66         *res = NULL;
  67 }
  68 EXPORT_SYMBOL(ttm_resource_free);
  69
  70 /**
  71  * ttm_resource_manager_init
  72  *
  73  * @man: memory manager object to init
  74  * @p_size: size managed area in pages.
  75  *
  76  * Initialise core parts of a manager object.
  77  */
  78 void ttm_resource_manager_init(struct ttm_resource_manager *man,
  79                                unsigned long p_size)
  80 {
  81         unsigned i;
  82
  83         spin_lock_init(&man->move_lock);
  84         man->size = p_size;
  85
  86         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
  87                 INIT_LIST_HEAD(&man->lru[i]);
  88         man->move = NULL;
  89 }
  90 EXPORT_SYMBOL(ttm_resource_manager_init);
  91
  92 /*
  93  * ttm_resource_manager_evict_all
  94  *
  95  * @bdev - device to use
  96  * @man - manager to use
  97  *
  98  * Evict all the objects out of a memory manager until it is empty.
  99  * Part of memory manager cleanup sequence.
 100  */
 101 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 102                                    struct ttm_resource_manager *man)
 103 {
 104         struct ttm_operation_ctx ctx = {
 105                 .interruptible = false,
 106                 .no_wait_gpu = false,
 107                 .force_alloc = true
 108         };
 109         struct dma_fence *fence;
 110         int ret;
 111         unsigned i;
 112
 113         /*
 114          * Can't use standard list traversal since we're unlocking.
 115          */
 116
 117         spin_lock(&bdev->lru_lock);
 118         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 119                 while (!list_empty(&man->lru[i])) {
 120                         spin_unlock(&bdev->lru_lock);
 121                         ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
 122                                                   NULL);
 123                         if (ret)
 124                                 return ret;
 125                         spin_lock(&bdev->lru_lock);
 126                 }
 127         }
 128         spin_unlock(&bdev->lru_lock);
 129
 130         spin_lock(&man->move_lock);
 131         fence = dma_fence_get(man->move);
 132         spin_unlock(&man->move_lock);
 133
 134         if (fence) {
 135                 ret = dma_fence_wait(fence, false);
 136                 dma_fence_put(fence);
 137                 if (ret)
 138                         return ret;
 139         }
 140
 141         return 0;
 142 }
 143 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
 144
 145 /**
 146  * ttm_resource_manager_debug
 147  *
 148  * @man: manager type to dump.
 149  * @p: printer to use for debug.
 150  */
 151 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
 152                                 struct drm_printer *p)
 153 {
 154         drm_printf(p, "  use_type: %d\n", man->use_type);
 155         drm_printf(p, "  use_tt: %d\n", man->use_tt);
 156         drm_printf(p, "  size: %llu\n", man->size);
 157         if (man->func->debug)
 158                 man->func->debug(man, p);
 159 }
 160 EXPORT_SYMBOL(ttm_resource_manager_debug);
 161
 162 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
 163                                           struct dma_buf_map *dmap,
 164                                           pgoff_t i)
 165 {
 166         struct ttm_kmap_iter_iomap *iter_io =
 167                 container_of(iter, typeof(*iter_io), base);
 168         void __iomem *addr;
 169
 170 retry:
 171         while (i >= iter_io->cache.end) {
 172                 iter_io->cache.sg = iter_io->cache.sg ?
 173                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
 174                 iter_io->cache.i = iter_io->cache.end;
 175                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
 176                         PAGE_SHIFT;
 177                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
 178                         iter_io->start;
 179         }
 180
 181         if (i < iter_io->cache.i) {
 182                 iter_io->cache.end = 0;
 183                 iter_io->cache.sg = NULL;
 184                 goto retry;
 185         }
 186
 187         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
 188                                        (((resource_size_t)i - iter_io->cache.i)
 189                                         << PAGE_SHIFT));
 190         dma_buf_map_set_vaddr_iomem(dmap, addr);
 191 }
 192
 193 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
 194                                             struct dma_buf_map *map)
 195 {
 196         io_mapping_unmap_local(map->vaddr_iomem);
 197 }
 198
 199 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
 200         .map_local =  ttm_kmap_iter_iomap_map_local,
 201         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
 202         .maps_tt = false,
 203 };
 204
 205 /**
 206  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
 207  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
 208  * @iomap: The struct io_mapping representing the underlying linear io_memory.
 209  * @st: sg_table into @iomap, representing the memory of the struct
 210  * ttm_resource.
 211  * @start: Offset that needs to be subtracted from @st to make
 212  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 213  *
 214  * Return: Pointer to the embedded struct ttm_kmap_iter.
 215  */
 216 struct ttm_kmap_iter *
 217 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
 218                          struct io_mapping *iomap,
 219                          struct sg_table *st,
 220                          resource_size_t start)
 221 {
 222         iter_io->base.ops = &ttm_kmap_iter_io_ops;
 223         iter_io->iomap = iomap;
 224         iter_io->st = st;
 225         iter_io->start = start;
 226         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
 227
 228         return &iter_io->base;
 229 }
 230 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
 231
 232 /**
 233  * DOC: Linear io iterator
 234  *
 235  * This code should die in the not too near future. Best would be if we could
 236  * make io-mapping use memremap for all io memory, and have memremap
 237  * implement a kmap_local functionality. We could then strip a huge amount of
 238  * code. These linear io iterators are implemented to mimic old functionality,
 239  * and they don't use kmap_local semantics at all internally. Rather ioremap or
 240  * friends, and at least on 32-bit they add global TLB flushes and points
 241  * of failure.
 242  */
 243
 244 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
 245                                               struct dma_buf_map *dmap,
 246                                               pgoff_t i)
 247 {
 248         struct ttm_kmap_iter_linear_io *iter_io =
 249                 container_of(iter, typeof(*iter_io), base);
 250
 251         *dmap = iter_io->dmap;
 252         dma_buf_map_incr(dmap, i * PAGE_SIZE);
 253 }
 254
 255 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
 256         .map_local =  ttm_kmap_iter_linear_io_map_local,
 257         .maps_tt = false,
 258 };
 259
 260 /**
 261  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
 262  * @iter_io: The iterator to initialize
 263  * @bdev: The TTM device
 264  * @mem: The ttm resource representing the iomap.
 265  *
 266  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
 267  * pointing at a linear chunk of io memory.
 268  *
 269  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
 270  * failure.
 271  */
 272 struct ttm_kmap_iter *
 273 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 274                              struct ttm_device *bdev,
 275                              struct ttm_resource *mem)
 276 {
 277         int ret;
 278
 279         ret = ttm_mem_io_reserve(bdev, mem);
 280         if (ret)
 281                 goto out_err;
 282         if (!mem->bus.is_iomem) {
 283                 ret = -EINVAL;
 284                 goto out_io_free;
 285         }
 286
 287         if (mem->bus.addr) {
 288                 dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
 289                 iter_io->needs_unmap = false;
 290         } else {
 291                 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
 292
 293                 iter_io->needs_unmap = true;
 294                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
 295                 if (mem->bus.caching == ttm_write_combined)
 296                         dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
 297                                                     ioremap_wc(mem->bus.offset,
 298                                                                bus_size));
 299                 else if (mem->bus.caching == ttm_cached)
 300                         dma_buf_map_set_vaddr(&iter_io->dmap,
 301                                               memremap(mem->bus.offset, bus_size,
 302                                                        MEMREMAP_WB |
 303                                                        MEMREMAP_WT |
 304                                                        MEMREMAP_WC));
 305
 306                 /* If uncached requested or if mapping cached or wc failed */
 307                 if (dma_buf_map_is_null(&iter_io->dmap))
 308                         dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
 309                                                     ioremap(mem->bus.offset,
 310                                                             bus_size));
 311
 312                 if (dma_buf_map_is_null(&iter_io->dmap)) {
 313                         ret = -ENOMEM;
 314                         goto out_io_free;
 315                 }
 316         }
 317
 318         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
 319         return &iter_io->base;
 320
 321 out_io_free:
 322         ttm_mem_io_free(bdev, mem);
 323 out_err:
 324         return ERR_PTR(ret);
 325 }
 326
 327 /**
 328  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
 329  * @iter_io: The iterator to initialize
 330  * @bdev: The TTM device
 331  * @mem: The ttm resource representing the iomap.
 332  *
 333  * This function is for internal TTM use only. It cleans up a memcpy kmap
 334  * iterator initialized by ttm_kmap_iter_linear_io_init.
 335  */
 336 void
 337 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
 338                              struct ttm_device *bdev,
 339                              struct ttm_resource *mem)
 340 {
 341         if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
 342                 if (iter_io->dmap.is_iomem)
 343                         iounmap(iter_io->dmap.vaddr_iomem);
 344                 else
 345                         memunmap(iter_io->dmap.vaddr);
 346         }
 347
 348         ttm_mem_io_free(bdev, mem);
 349 }