drivers/gpu/drm/ttm/ttm_resource.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/iosys-map.h>
  26 #include <linux/io-mapping.h>
  27 #include <linux/scatterlist.h>
  28
  29 #include <drm/ttm/ttm_bo.h>
  30 #include <drm/ttm/ttm_placement.h>
  31 #include <drm/ttm/ttm_resource.h>
  32
  33 #include <drm/drm_util.h>
  34
  35 /**
  36  * ttm_lru_bulk_move_init - initialize a bulk move structure
  37  * @bulk: the structure to init
  38  *
  39  * For now just memset the structure to zero.
  40  */
  41 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
  42 {
  43         memset(bulk, 0, sizeof(*bulk));
  44 }
  45 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
  46
  47 /**
  48  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
  49  *
  50  * @bulk: bulk move structure
  51  *
  52  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
  53  * resource order never changes. Should be called with &ttm_device.lru_lock held.
  54  */
  55 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
  56 {
  57         unsigned i, j;
  58
  59         for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
  60                 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
  61                         struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
  62                         struct ttm_resource_manager *man;
  63
  64                         if (!pos->first)
  65                                 continue;
  66
  67                         lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
  68                         dma_resv_assert_held(pos->first->bo->base.resv);
  69                         dma_resv_assert_held(pos->last->bo->base.resv);
  70
  71                         man = ttm_manager_type(pos->first->bo->bdev, i);
  72                         list_bulk_move_tail(&man->lru[j], &pos->first->lru,
  73                                             &pos->last->lru);
  74                 }
  75         }
  76 }
  77 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
  78
  79 /* Return the bulk move pos object for this resource */
  80 static struct ttm_lru_bulk_move_pos *
  81 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
  82 {
  83         return &bulk->pos[res->mem_type][res->bo->priority];
  84 }
  85
  86 /* Move the resource to the tail of the bulk move range */
  87 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
  88                                        struct ttm_resource *res)
  89 {
  90         if (pos->last != res) {
  91                 if (pos->first == res)
  92                         pos->first = list_next_entry(res, lru);
  93                 list_move(&res->lru, &pos->last->lru);
  94                 pos->last = res;
  95         }
  96 }
  97
  98 /* Add the resource to a bulk_move cursor */
  99 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
 100                                   struct ttm_resource *res)
 101 {
 102         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
 103
 104         if (!pos->first) {
 105                 pos->first = res;
 106                 pos->last = res;
 107         } else {
 108                 ttm_lru_bulk_move_pos_tail(pos, res);
 109         }
 110 }
 111
 112 /* Remove the resource from a bulk_move range */
 113 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
 114                                   struct ttm_resource *res)
 115 {
 116         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
 117
 118         if (unlikely(WARN_ON(!pos->first || !pos->last) ||
 119                      (pos->first == res && pos->last == res))) {
 120                 pos->first = NULL;
 121                 pos->last = NULL;
 122         } else if (pos->first == res) {
 123                 pos->first = list_next_entry(res, lru);
 124         } else if (pos->last == res) {
 125                 pos->last = list_prev_entry(res, lru);
 126         } else {
 127                 list_move(&res->lru, &pos->last->lru);
 128         }
 129 }
 130
 131 /* Add the resource to a bulk move if the BO is configured for it */
 132 void ttm_resource_add_bulk_move(struct ttm_resource *res,
 133                                 struct ttm_buffer_object *bo)
 134 {
 135         if (bo->bulk_move && !bo->pin_count)
 136                 ttm_lru_bulk_move_add(bo->bulk_move, res);
 137 }
 138
 139 /* Remove the resource from a bulk move if the BO is configured for it */
 140 void ttm_resource_del_bulk_move(struct ttm_resource *res,
 141                                 struct ttm_buffer_object *bo)
 142 {
 143         if (bo->bulk_move && !bo->pin_count)
 144                 ttm_lru_bulk_move_del(bo->bulk_move, res);
 145 }
 146
 147 /* Move a resource to the LRU or bulk tail */
 148 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
 149 {
 150         struct ttm_buffer_object *bo = res->bo;
 151         struct ttm_device *bdev = bo->bdev;
 152
 153         lockdep_assert_held(&bo->bdev->lru_lock);
 154
 155         if (bo->pin_count) {
 156                 list_move_tail(&res->lru, &bdev->pinned);
 157
 158         } else  if (bo->bulk_move) {
 159                 struct ttm_lru_bulk_move_pos *pos =
 160                         ttm_lru_bulk_move_pos(bo->bulk_move, res);
 161
 162                 ttm_lru_bulk_move_pos_tail(pos, res);
 163         } else {
 164                 struct ttm_resource_manager *man;
 165
 166                 man = ttm_manager_type(bdev, res->mem_type);
 167                 list_move_tail(&res->lru, &man->lru[bo->priority]);
 168         }
 169 }
 170
 171 /**
 172  * ttm_resource_init - resource object constructure
 173  * @bo: buffer object this resources is allocated for
 174  * @place: placement of the resource
 175  * @res: the resource object to inistilize
 176  *
 177  * Initialize a new resource object. Counterpart of ttm_resource_fini().
 178  */
 179 void ttm_resource_init(struct ttm_buffer_object *bo,
 180                        const struct ttm_place *place,
 181                        struct ttm_resource *res)
 182 {
 183         struct ttm_resource_manager *man;
 184
 185         res->start = 0;
 186         res->size = bo->base.size;
 187         res->mem_type = place->mem_type;
 188         res->placement = place->flags;
 189         res->bus.addr = NULL;
 190         res->bus.offset = 0;
 191         res->bus.is_iomem = false;
 192         res->bus.caching = ttm_cached;
 193         res->bo = bo;
 194
 195         man = ttm_manager_type(bo->bdev, place->mem_type);
 196         spin_lock(&bo->bdev->lru_lock);
 197         if (bo->pin_count)
 198                 list_add_tail(&res->lru, &bo->bdev->pinned);
 199         else
 200                 list_add_tail(&res->lru, &man->lru[bo->priority]);
 201         man->usage += res->size;
 202         spin_unlock(&bo->bdev->lru_lock);
 203 }
 204 EXPORT_SYMBOL(ttm_resource_init);
 205
 206 /**
 207  * ttm_resource_fini - resource destructor
 208  * @man: the resource manager this resource belongs to
 209  * @res: the resource to clean up
 210  *
 211  * Should be used by resource manager backends to clean up the TTM resource
 212  * objects before freeing the underlying structure. Makes sure the resource is
 213  * removed from the LRU before destruction.
 214  * Counterpart of ttm_resource_init().
 215  */
 216 void ttm_resource_fini(struct ttm_resource_manager *man,
 217                        struct ttm_resource *res)
 218 {
 219         struct ttm_device *bdev = man->bdev;
 220
 221         spin_lock(&bdev->lru_lock);
 222         list_del_init(&res->lru);
 223         man->usage -= res->size;
 224         spin_unlock(&bdev->lru_lock);
 225 }
 226 EXPORT_SYMBOL(ttm_resource_fini);
 227
 228 int ttm_resource_alloc(struct ttm_buffer_object *bo,
 229                        const struct ttm_place *place,
 230                        struct ttm_resource **res_ptr)
 231 {
 232         struct ttm_resource_manager *man =
 233                 ttm_manager_type(bo->bdev, place->mem_type);
 234         int ret;
 235
 236         ret = man->func->alloc(man, bo, place, res_ptr);
 237         if (ret)
 238                 return ret;
 239
 240         spin_lock(&bo->bdev->lru_lock);
 241         ttm_resource_add_bulk_move(*res_ptr, bo);
 242         spin_unlock(&bo->bdev->lru_lock);
 243         return 0;
 244 }
 245 EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);
 246
 247 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
 248 {
 249         struct ttm_resource_manager *man;
 250
 251         if (!*res)
 252                 return;
 253
 254         spin_lock(&bo->bdev->lru_lock);
 255         ttm_resource_del_bulk_move(*res, bo);
 256         spin_unlock(&bo->bdev->lru_lock);
 257         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
 258         man->func->free(man, *res);
 259         *res = NULL;
 260 }
 261 EXPORT_SYMBOL(ttm_resource_free);
 262
 263 /**
 264  * ttm_resource_intersects - test for intersection
 265  *
 266  * @bdev: TTM device structure
 267  * @res: The resource to test
 268  * @place: The placement to test
 269  * @size: How many bytes the new allocation needs.
 270  *
 271  * Test if @res intersects with @place and @size. Used for testing if evictions
 272  * are valueable or not.
 273  *
 274  * Returns true if the res placement intersects with @place and @size.
 275  */
 276 bool ttm_resource_intersects(struct ttm_device *bdev,
 277                              struct ttm_resource *res,
 278                              const struct ttm_place *place,
 279                              size_t size)
 280 {
 281         struct ttm_resource_manager *man;
 282
 283         if (!res)
 284                 return false;
 285
 286         man = ttm_manager_type(bdev, res->mem_type);
 287         if (!place || !man->func->intersects)
 288                 return true;
 289
 290         return man->func->intersects(man, res, place, size);
 291 }
 292
 293 /**
 294  * ttm_resource_compatible - check if resource is compatible with placement
 295  *
 296  * @res: the resource to check
 297  * @placement: the placement to check against
 298  *
 299  * Returns true if the placement is compatible.
 300  */
 301 bool ttm_resource_compatible(struct ttm_resource *res,
 302                              struct ttm_placement *placement)
 303 {
 304         struct ttm_buffer_object *bo = res->bo;
 305         struct ttm_device *bdev = bo->bdev;
 306         unsigned i;
 307
 308         if (res->placement & TTM_PL_FLAG_TEMPORARY)
 309                 return false;
 310
 311         for (i = 0; i < placement->num_placement; i++) {
 312                 const struct ttm_place *place = &placement->placement[i];
 313                 struct ttm_resource_manager *man;
 314
 315                 if (res->mem_type != place->mem_type)
 316                         continue;
 317
 318                 man = ttm_manager_type(bdev, res->mem_type);
 319                 if (man->func->compatible &&
 320                     !man->func->compatible(man, res, place, bo->base.size))
 321                         continue;
 322
 323                 if ((!(place->flags & TTM_PL_FLAG_CONTIGUOUS) ||
 324                      (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
 325                         return true;
 326         }
 327         return false;
 328 }
 329
 330 void ttm_resource_set_bo(struct ttm_resource *res,
 331                          struct ttm_buffer_object *bo)
 332 {
 333         spin_lock(&bo->bdev->lru_lock);
 334         res->bo = bo;
 335         spin_unlock(&bo->bdev->lru_lock);
 336 }
 337
 338 /**
 339  * ttm_resource_manager_init
 340  *
 341  * @man: memory manager object to init
 342  * @bdev: ttm device this manager belongs to
 343  * @size: size of managed resources in arbitrary units
 344  *
 345  * Initialise core parts of a manager object.
 346  */
 347 void ttm_resource_manager_init(struct ttm_resource_manager *man,
 348                                struct ttm_device *bdev,
 349                                uint64_t size)
 350 {
 351         unsigned i;
 352
 353         spin_lock_init(&man->move_lock);
 354         man->bdev = bdev;
 355         man->size = size;
 356         man->usage = 0;
 357
 358         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 359                 INIT_LIST_HEAD(&man->lru[i]);
 360         man->move = NULL;
 361 }
 362 EXPORT_SYMBOL(ttm_resource_manager_init);
 363
 364 /*
 365  * ttm_resource_manager_evict_all
 366  *
 367  * @bdev - device to use
 368  * @man - manager to use
 369  *
 370  * Evict all the objects out of a memory manager until it is empty.
 371  * Part of memory manager cleanup sequence.
 372  */
 373 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 374                                    struct ttm_resource_manager *man)
 375 {
 376         struct ttm_operation_ctx ctx = {
 377                 .interruptible = false,
 378                 .no_wait_gpu = false,
 379                 .force_alloc = true
 380         };
 381         struct dma_fence *fence;
 382         int ret;
 383         unsigned i;
 384
 385         /*
 386          * Can't use standard list traversal since we're unlocking.
 387          */
 388
 389         spin_lock(&bdev->lru_lock);
 390         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 391                 while (!list_empty(&man->lru[i])) {
 392                         spin_unlock(&bdev->lru_lock);
 393                         ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
 394                                                   NULL);
 395                         if (ret)
 396                                 return ret;
 397                         spin_lock(&bdev->lru_lock);
 398                 }
 399         }
 400         spin_unlock(&bdev->lru_lock);
 401
 402         spin_lock(&man->move_lock);
 403         fence = dma_fence_get(man->move);
 404         spin_unlock(&man->move_lock);
 405
 406         if (fence) {
 407                 ret = dma_fence_wait(fence, false);
 408                 dma_fence_put(fence);
 409                 if (ret)
 410                         return ret;
 411         }
 412
 413         return 0;
 414 }
 415 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
 416
 417 /**
 418  * ttm_resource_manager_usage
 419  *
 420  * @man: A memory manager object.
 421  *
 422  * Return how many resources are currently used.
 423  */
 424 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
 425 {
 426         uint64_t usage;
 427
 428         spin_lock(&man->bdev->lru_lock);
 429         usage = man->usage;
 430         spin_unlock(&man->bdev->lru_lock);
 431         return usage;
 432 }
 433 EXPORT_SYMBOL(ttm_resource_manager_usage);
 434
 435 /**
 436  * ttm_resource_manager_debug
 437  *
 438  * @man: manager type to dump.
 439  * @p: printer to use for debug.
 440  */
 441 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
 442                                 struct drm_printer *p)
 443 {
 444         drm_printf(p, "  use_type: %d\n", man->use_type);
 445         drm_printf(p, "  use_tt: %d\n", man->use_tt);
 446         drm_printf(p, "  size: %llu\n", man->size);
 447         drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
 448         if (man->func->debug)
 449                 man->func->debug(man, p);
 450 }
 451 EXPORT_SYMBOL(ttm_resource_manager_debug);
 452
 453 /**
 454  * ttm_resource_manager_first
 455  *
 456  * @man: resource manager to iterate over
 457  * @cursor: cursor to record the position
 458  *
 459  * Returns the first resource from the resource manager.
 460  */
 461 struct ttm_resource *
 462 ttm_resource_manager_first(struct ttm_resource_manager *man,
 463                            struct ttm_resource_cursor *cursor)
 464 {
 465         struct ttm_resource *res;
 466
 467         lockdep_assert_held(&man->bdev->lru_lock);
 468
 469         for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
 470              ++cursor->priority)
 471                 list_for_each_entry(res, &man->lru[cursor->priority], lru)
 472                         return res;
 473
 474         return NULL;
 475 }
 476
 477 /**
 478  * ttm_resource_manager_next
 479  *
 480  * @man: resource manager to iterate over
 481  * @cursor: cursor to record the position
 482  * @res: the current resource pointer
 483  *
 484  * Returns the next resource from the resource manager.
 485  */
 486 struct ttm_resource *
 487 ttm_resource_manager_next(struct ttm_resource_manager *man,
 488                           struct ttm_resource_cursor *cursor,
 489                           struct ttm_resource *res)
 490 {
 491         lockdep_assert_held(&man->bdev->lru_lock);
 492
 493         list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
 494                 return res;
 495
 496         for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
 497              ++cursor->priority)
 498                 list_for_each_entry(res, &man->lru[cursor->priority], lru)
 499                         return res;
 500
 501         return NULL;
 502 }
 503
 504 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
 505                                           struct iosys_map *dmap,
 506                                           pgoff_t i)
 507 {
 508         struct ttm_kmap_iter_iomap *iter_io =
 509                 container_of(iter, typeof(*iter_io), base);
 510         void __iomem *addr;
 511
 512 retry:
 513         while (i >= iter_io->cache.end) {
 514                 iter_io->cache.sg = iter_io->cache.sg ?
 515                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
 516                 iter_io->cache.i = iter_io->cache.end;
 517                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
 518                         PAGE_SHIFT;
 519                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
 520                         iter_io->start;
 521         }
 522
 523         if (i < iter_io->cache.i) {
 524                 iter_io->cache.end = 0;
 525                 iter_io->cache.sg = NULL;
 526                 goto retry;
 527         }
 528
 529         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
 530                                        (((resource_size_t)i - iter_io->cache.i)
 531                                         << PAGE_SHIFT));
 532         iosys_map_set_vaddr_iomem(dmap, addr);
 533 }
 534
 535 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
 536                                             struct iosys_map *map)
 537 {
 538         io_mapping_unmap_local(map->vaddr_iomem);
 539 }
 540
 541 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
 542         .map_local =  ttm_kmap_iter_iomap_map_local,
 543         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
 544         .maps_tt = false,
 545 };
 546
 547 /**
 548  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
 549  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
 550  * @iomap: The struct io_mapping representing the underlying linear io_memory.
 551  * @st: sg_table into @iomap, representing the memory of the struct
 552  * ttm_resource.
 553  * @start: Offset that needs to be subtracted from @st to make
 554  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 555  *
 556  * Return: Pointer to the embedded struct ttm_kmap_iter.
 557  */
 558 struct ttm_kmap_iter *
 559 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
 560                          struct io_mapping *iomap,
 561                          struct sg_table *st,
 562                          resource_size_t start)
 563 {
 564         iter_io->base.ops = &ttm_kmap_iter_io_ops;
 565         iter_io->iomap = iomap;
 566         iter_io->st = st;
 567         iter_io->start = start;
 568         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
 569
 570         return &iter_io->base;
 571 }
 572 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
 573
 574 /**
 575  * DOC: Linear io iterator
 576  *
 577  * This code should die in the not too near future. Best would be if we could
 578  * make io-mapping use memremap for all io memory, and have memremap
 579  * implement a kmap_local functionality. We could then strip a huge amount of
 580  * code. These linear io iterators are implemented to mimic old functionality,
 581  * and they don't use kmap_local semantics at all internally. Rather ioremap or
 582  * friends, and at least on 32-bit they add global TLB flushes and points
 583  * of failure.
 584  */
 585
 586 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
 587                                               struct iosys_map *dmap,
 588                                               pgoff_t i)
 589 {
 590         struct ttm_kmap_iter_linear_io *iter_io =
 591                 container_of(iter, typeof(*iter_io), base);
 592
 593         *dmap = iter_io->dmap;
 594         iosys_map_incr(dmap, i * PAGE_SIZE);
 595 }
 596
 597 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
 598         .map_local =  ttm_kmap_iter_linear_io_map_local,
 599         .maps_tt = false,
 600 };
 601
 602 /**
 603  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
 604  * @iter_io: The iterator to initialize
 605  * @bdev: The TTM device
 606  * @mem: The ttm resource representing the iomap.
 607  *
 608  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
 609  * pointing at a linear chunk of io memory.
 610  *
 611  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
 612  * failure.
 613  */
 614 struct ttm_kmap_iter *
 615 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 616                              struct ttm_device *bdev,
 617                              struct ttm_resource *mem)
 618 {
 619         int ret;
 620
 621         ret = ttm_mem_io_reserve(bdev, mem);
 622         if (ret)
 623                 goto out_err;
 624         if (!mem->bus.is_iomem) {
 625                 ret = -EINVAL;
 626                 goto out_io_free;
 627         }
 628
 629         if (mem->bus.addr) {
 630                 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
 631                 iter_io->needs_unmap = false;
 632         } else {
 633                 iter_io->needs_unmap = true;
 634                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
 635                 if (mem->bus.caching == ttm_write_combined)
 636                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
 637                                                   ioremap_wc(mem->bus.offset,
 638                                                              mem->size));
 639                 else if (mem->bus.caching == ttm_cached)
 640                         iosys_map_set_vaddr(&iter_io->dmap,
 641                                             memremap(mem->bus.offset, mem->size,
 642                                                      MEMREMAP_WB |
 643                                                      MEMREMAP_WT |
 644                                                      MEMREMAP_WC));
 645
 646                 /* If uncached requested or if mapping cached or wc failed */
 647                 if (iosys_map_is_null(&iter_io->dmap))
 648                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
 649                                                   ioremap(mem->bus.offset,
 650                                                           mem->size));
 651
 652                 if (iosys_map_is_null(&iter_io->dmap)) {
 653                         ret = -ENOMEM;
 654                         goto out_io_free;
 655                 }
 656         }
 657
 658         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
 659         return &iter_io->base;
 660
 661 out_io_free:
 662         ttm_mem_io_free(bdev, mem);
 663 out_err:
 664         return ERR_PTR(ret);
 665 }
 666
 667 /**
 668  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
 669  * @iter_io: The iterator to initialize
 670  * @bdev: The TTM device
 671  * @mem: The ttm resource representing the iomap.
 672  *
 673  * This function is for internal TTM use only. It cleans up a memcpy kmap
 674  * iterator initialized by ttm_kmap_iter_linear_io_init.
 675  */
 676 void
 677 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
 678                              struct ttm_device *bdev,
 679                              struct ttm_resource *mem)
 680 {
 681         if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
 682                 if (iter_io->dmap.is_iomem)
 683                         iounmap(iter_io->dmap.vaddr_iomem);
 684                 else
 685                         memunmap(iter_io->dmap.vaddr);
 686         }
 687
 688         ttm_mem_io_free(bdev, mem);
 689 }
 690
 691 #if defined(CONFIG_DEBUG_FS)
 692
 693 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
 694 {
 695         struct ttm_resource_manager *man =
 696                 (struct ttm_resource_manager *)m->private;
 697         struct drm_printer p = drm_seq_file_printer(m);
 698         ttm_resource_manager_debug(man, &p);
 699         return 0;
 700 }
 701 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
 702
 703 #endif
 704
 705 /**
 706  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
 707  * resource manager.
 708  * @man: The TTM resource manager for which the debugfs stats file be creates
 709  * @parent: debugfs directory in which the file will reside
 710  * @name: The filename to create.
 711  *
 712  * This function setups up a debugfs file that can be used to look
 713  * at debug statistics of the specified ttm_resource_manager.
 714  */
 715 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
 716                                          struct dentry * parent,
 717                                          const char *name)
 718 {
 719 #if defined(CONFIG_DEBUG_FS)
 720         debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
 721 #endif
 722 }
 723 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);