drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #include <drm/ttm/ttm_bo_driver.h>
  33 #include <drm/ttm/ttm_placement.h>
  34 #include <drm/drm_vma_manager.h>
  35 #include <linux/io.h>
  36 #include <linux/highmem.h>
  37 #include <linux/wait.h>
  38 #include <linux/slab.h>
  39 #include <linux/vmalloc.h>
  40 #include <linux/module.h>
  41 #include <linux/dma-resv.h>
  42
  43 struct ttm_transfer_obj {
  44         struct ttm_buffer_object base;
  45         struct ttm_buffer_object *bo;
  46 };
  47
  48 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  49 {
  50         ttm_bo_mem_put(bo, &bo->mem);
  51 }
  52
  53 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  54                    struct ttm_operation_ctx *ctx,
  55                     struct ttm_mem_reg *new_mem)
  56 {
  57         struct ttm_tt *ttm = bo->ttm;
  58         struct ttm_mem_reg *old_mem = &bo->mem;
  59         int ret;
  60
  61         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  62                 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
  63
  64                 if (unlikely(ret != 0)) {
  65                         if (ret != -ERESTARTSYS)
  66                                 pr_err("Failed to expire sync object before unbinding TTM\n");
  67                         return ret;
  68                 }
  69
  70                 ttm_tt_unbind(ttm);
  71                 ttm_bo_free_old_node(bo);
  72                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  73                                 TTM_PL_MASK_MEM);
  74                 old_mem->mem_type = TTM_PL_SYSTEM;
  75         }
  76
  77         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  78         if (unlikely(ret != 0))
  79                 return ret;
  80
  81         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  82                 ret = ttm_tt_bind(ttm, new_mem, ctx);
  83                 if (unlikely(ret != 0))
  84                         return ret;
  85         }
  86
  87         *old_mem = *new_mem;
  88         new_mem->mm_node = NULL;
  89
  90         return 0;
  91 }
  92 EXPORT_SYMBOL(ttm_bo_move_ttm);
  93
  94 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
  95 {
  96         if (likely(man->io_reserve_fastpath))
  97                 return 0;
  98
  99         if (interruptible)
 100                 return mutex_lock_interruptible(&man->io_reserve_mutex);
 101
 102         mutex_lock(&man->io_reserve_mutex);
 103         return 0;
 104 }
 105
 106 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
 107 {
 108         if (likely(man->io_reserve_fastpath))
 109                 return;
 110
 111         mutex_unlock(&man->io_reserve_mutex);
 112 }
 113
 114 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
 115 {
 116         struct ttm_buffer_object *bo;
 117
 118         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
 119                 return -EAGAIN;
 120
 121         bo = list_first_entry(&man->io_reserve_lru,
 122                               struct ttm_buffer_object,
 123                               io_reserve_lru);
 124         list_del_init(&bo->io_reserve_lru);
 125         ttm_bo_unmap_virtual_locked(bo);
 126
 127         return 0;
 128 }
 129
 130
 131 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
 132                        struct ttm_mem_reg *mem)
 133 {
 134         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 135         int ret = 0;
 136
 137         if (!bdev->driver->io_mem_reserve)
 138                 return 0;
 139         if (likely(man->io_reserve_fastpath))
 140                 return bdev->driver->io_mem_reserve(bdev, mem);
 141
 142         if (bdev->driver->io_mem_reserve &&
 143             mem->bus.io_reserved_count++ == 0) {
 144 retry:
 145                 ret = bdev->driver->io_mem_reserve(bdev, mem);
 146                 if (ret == -EAGAIN) {
 147                         ret = ttm_mem_io_evict(man);
 148                         if (ret == 0)
 149                                 goto retry;
 150                 }
 151         }
 152         return ret;
 153 }
 154
 155 void ttm_mem_io_free(struct ttm_bo_device *bdev,
 156                      struct ttm_mem_reg *mem)
 157 {
 158         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 159
 160         if (likely(man->io_reserve_fastpath))
 161                 return;
 162
 163         if (bdev->driver->io_mem_reserve &&
 164             --mem->bus.io_reserved_count == 0 &&
 165             bdev->driver->io_mem_free)
 166                 bdev->driver->io_mem_free(bdev, mem);
 167
 168 }
 169
 170 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
 171 {
 172         struct ttm_mem_reg *mem = &bo->mem;
 173         int ret;
 174
 175         if (!mem->bus.io_reserved_vm) {
 176                 struct ttm_mem_type_manager *man =
 177                         &bo->bdev->man[mem->mem_type];
 178
 179                 ret = ttm_mem_io_reserve(bo->bdev, mem);
 180                 if (unlikely(ret != 0))
 181                         return ret;
 182                 mem->bus.io_reserved_vm = true;
 183                 if (man->use_io_reserve_lru)
 184                         list_add_tail(&bo->io_reserve_lru,
 185                                       &man->io_reserve_lru);
 186         }
 187         return 0;
 188 }
 189
 190 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 191 {
 192         struct ttm_mem_reg *mem = &bo->mem;
 193
 194         if (mem->bus.io_reserved_vm) {
 195                 mem->bus.io_reserved_vm = false;
 196                 list_del_init(&bo->io_reserve_lru);
 197                 ttm_mem_io_free(bo->bdev, mem);
 198         }
 199 }
 200
 201 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 202                         void **virtual)
 203 {
 204         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 205         int ret;
 206         void *addr;
 207
 208         *virtual = NULL;
 209         (void) ttm_mem_io_lock(man, false);
 210         ret = ttm_mem_io_reserve(bdev, mem);
 211         ttm_mem_io_unlock(man);
 212         if (ret || !mem->bus.is_iomem)
 213                 return ret;
 214
 215         if (mem->bus.addr) {
 216                 addr = mem->bus.addr;
 217         } else {
 218                 if (mem->placement & TTM_PL_FLAG_WC)
 219                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 220                 else
 221                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 222                 if (!addr) {
 223                         (void) ttm_mem_io_lock(man, false);
 224                         ttm_mem_io_free(bdev, mem);
 225                         ttm_mem_io_unlock(man);
 226                         return -ENOMEM;
 227                 }
 228         }
 229         *virtual = addr;
 230         return 0;
 231 }
 232
 233 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 234                          void *virtual)
 235 {
 236         struct ttm_mem_type_manager *man;
 237
 238         man = &bdev->man[mem->mem_type];
 239
 240         if (virtual && mem->bus.addr == NULL)
 241                 iounmap(virtual);
 242         (void) ttm_mem_io_lock(man, false);
 243         ttm_mem_io_free(bdev, mem);
 244         ttm_mem_io_unlock(man);
 245 }
 246
 247 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 248 {
 249         uint32_t *dstP =
 250             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 251         uint32_t *srcP =
 252             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 253
 254         int i;
 255         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 256                 iowrite32(ioread32(srcP++), dstP++);
 257         return 0;
 258 }
 259
 260 #ifdef CONFIG_X86
 261 #define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
 262 #define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
 263 #else
 264 #define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0,  __prot)
 265 #define __ttm_kunmap_atomic(__addr) vunmap(__addr)
 266 #endif
 267
 268
 269 /**
 270  * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
 271  * specified page protection.
 272  *
 273  * @page: The page to map.
 274  * @prot: The page protection.
 275  *
 276  * This function maps a TTM page using the kmap_atomic api if available,
 277  * otherwise falls back to vmap. The user must make sure that the
 278  * specified page does not have an aliased mapping with a different caching
 279  * policy unless the architecture explicitly allows it. Also mapping and
 280  * unmapping using this api must be correctly nested. Unmapping should
 281  * occur in the reverse order of mapping.
 282  */
 283 void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
 284 {
 285         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
 286                 return kmap_atomic(page);
 287         else
 288                 return __ttm_kmap_atomic_prot(page, prot);
 289 }
 290 EXPORT_SYMBOL(ttm_kmap_atomic_prot);
 291
 292 /**
 293  * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
 294  * ttm_kmap_atomic_prot.
 295  *
 296  * @addr: The virtual address from the map.
 297  * @prot: The page protection.
 298  */
 299 void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
 300 {
 301         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
 302                 kunmap_atomic(addr);
 303         else
 304                 __ttm_kunmap_atomic(addr);
 305 }
 306 EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
 307
 308 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 309                                 unsigned long page,
 310                                 pgprot_t prot)
 311 {
 312         struct page *d = ttm->pages[page];
 313         void *dst;
 314
 315         if (!d)
 316                 return -ENOMEM;
 317
 318         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 319         dst = ttm_kmap_atomic_prot(d, prot);
 320         if (!dst)
 321                 return -ENOMEM;
 322
 323         memcpy_fromio(dst, src, PAGE_SIZE);
 324
 325         ttm_kunmap_atomic_prot(dst, prot);
 326
 327         return 0;
 328 }
 329
 330 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 331                                 unsigned long page,
 332                                 pgprot_t prot)
 333 {
 334         struct page *s = ttm->pages[page];
 335         void *src;
 336
 337         if (!s)
 338                 return -ENOMEM;
 339
 340         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 341         src = ttm_kmap_atomic_prot(s, prot);
 342         if (!src)
 343                 return -ENOMEM;
 344
 345         memcpy_toio(dst, src, PAGE_SIZE);
 346
 347         ttm_kunmap_atomic_prot(src, prot);
 348
 349         return 0;
 350 }
 351
 352 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 353                        struct ttm_operation_ctx *ctx,
 354                        struct ttm_mem_reg *new_mem)
 355 {
 356         struct ttm_bo_device *bdev = bo->bdev;
 357         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 358         struct ttm_tt *ttm = bo->ttm;
 359         struct ttm_mem_reg *old_mem = &bo->mem;
 360         struct ttm_mem_reg old_copy = *old_mem;
 361         void *old_iomap;
 362         void *new_iomap;
 363         int ret;
 364         unsigned long i;
 365         unsigned long page;
 366         unsigned long add = 0;
 367         int dir;
 368
 369         ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 370         if (ret)
 371                 return ret;
 372
 373         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 374         if (ret)
 375                 return ret;
 376         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 377         if (ret)
 378                 goto out;
 379
 380         /*
 381          * Single TTM move. NOP.
 382          */
 383         if (old_iomap == NULL && new_iomap == NULL)
 384                 goto out2;
 385
 386         /*
 387          * Don't move nonexistent data. Clear destination instead.
 388          */
 389         if (old_iomap == NULL &&
 390             (ttm == NULL || (ttm->state == tt_unpopulated &&
 391                              !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
 392                 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
 393                 goto out2;
 394         }
 395
 396         /*
 397          * TTM might be null for moves within the same region.
 398          */
 399         if (ttm) {
 400                 ret = ttm_tt_populate(ttm, ctx);
 401                 if (ret)
 402                         goto out1;
 403         }
 404
 405         add = 0;
 406         dir = 1;
 407
 408         if ((old_mem->mem_type == new_mem->mem_type) &&
 409             (new_mem->start < old_mem->start + old_mem->size)) {
 410                 dir = -1;
 411                 add = new_mem->num_pages - 1;
 412         }
 413
 414         for (i = 0; i < new_mem->num_pages; ++i) {
 415                 page = i * dir + add;
 416                 if (old_iomap == NULL) {
 417                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 418                                                     PAGE_KERNEL);
 419                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 420                                                    prot);
 421                 } else if (new_iomap == NULL) {
 422                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 423                                                     PAGE_KERNEL);
 424                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 425                                                    prot);
 426                 } else {
 427                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 428                 }
 429                 if (ret)
 430                         goto out1;
 431         }
 432         mb();
 433 out2:
 434         old_copy = *old_mem;
 435         *old_mem = *new_mem;
 436         new_mem->mm_node = NULL;
 437
 438         if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 439                 ttm_tt_destroy(ttm);
 440                 bo->ttm = NULL;
 441         }
 442
 443 out1:
 444         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 445 out:
 446         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 447
 448         /*
 449          * On error, keep the mm node!
 450          */
 451         if (!ret)
 452                 ttm_bo_mem_put(bo, &old_copy);
 453         return ret;
 454 }
 455 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 456
 457 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 458 {
 459         struct ttm_transfer_obj *fbo;
 460
 461         fbo = container_of(bo, struct ttm_transfer_obj, base);
 462         ttm_bo_put(fbo->bo);
 463         kfree(fbo);
 464 }
 465
 466 /**
 467  * ttm_buffer_object_transfer
 468  *
 469  * @bo: A pointer to a struct ttm_buffer_object.
 470  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 471  * holding the data of @bo with the old placement.
 472  *
 473  * This is a utility function that may be called after an accelerated move
 474  * has been scheduled. A new buffer object is created as a placeholder for
 475  * the old data while it's being copied. When that buffer object is idle,
 476  * it can be destroyed, releasing the space of the old placement.
 477  * Returns:
 478  * !0: Failure.
 479  */
 480
 481 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 482                                       struct ttm_buffer_object **new_obj)
 483 {
 484         struct ttm_transfer_obj *fbo;
 485         int ret;
 486
 487         fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
 488         if (!fbo)
 489                 return -ENOMEM;
 490
 491         fbo->base = *bo;
 492         fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT;
 493
 494         ttm_bo_get(bo);
 495         fbo->bo = bo;
 496
 497         /**
 498          * Fix up members that we shouldn't copy directly:
 499          * TODO: Explicit member copy would probably be better here.
 500          */
 501
 502         atomic_inc(&ttm_bo_glob.bo_count);
 503         INIT_LIST_HEAD(&fbo->base.ddestroy);
 504         INIT_LIST_HEAD(&fbo->base.lru);
 505         INIT_LIST_HEAD(&fbo->base.swap);
 506         INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
 507         mutex_init(&fbo->base.wu_mutex);
 508         fbo->base.moving = NULL;
 509         drm_vma_node_reset(&fbo->base.base.vma_node);
 510
 511         kref_init(&fbo->base.list_kref);
 512         kref_init(&fbo->base.kref);
 513         fbo->base.destroy = &ttm_transfered_destroy;
 514         fbo->base.acc_size = 0;
 515         if (bo->base.resv == &bo->base._resv)
 516                 fbo->base.base.resv = &fbo->base.base._resv;
 517
 518         dma_resv_init(&fbo->base.base._resv);
 519         ret = dma_resv_trylock(&fbo->base.base._resv);
 520         WARN_ON(!ret);
 521
 522         *new_obj = &fbo->base;
 523         return 0;
 524 }
 525
 526 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 527 {
 528         /* Cached mappings need no adjustment */
 529         if (caching_flags & TTM_PL_FLAG_CACHED)
 530                 return tmp;
 531
 532 #if defined(__i386__) || defined(__x86_64__)
 533         if (caching_flags & TTM_PL_FLAG_WC)
 534                 tmp = pgprot_writecombine(tmp);
 535         else if (boot_cpu_data.x86 > 3)
 536                 tmp = pgprot_noncached(tmp);
 537 #endif
 538 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
 539     defined(__powerpc__) || defined(__mips__)
 540         if (caching_flags & TTM_PL_FLAG_WC)
 541                 tmp = pgprot_writecombine(tmp);
 542         else
 543                 tmp = pgprot_noncached(tmp);
 544 #endif
 545 #if defined(__sparc__)
 546         tmp = pgprot_noncached(tmp);
 547 #endif
 548         return tmp;
 549 }
 550 EXPORT_SYMBOL(ttm_io_prot);
 551
 552 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 553                           unsigned long offset,
 554                           unsigned long size,
 555                           struct ttm_bo_kmap_obj *map)
 556 {
 557         struct ttm_mem_reg *mem = &bo->mem;
 558
 559         if (bo->mem.bus.addr) {
 560                 map->bo_kmap_type = ttm_bo_map_premapped;
 561                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 562         } else {
 563                 map->bo_kmap_type = ttm_bo_map_iomap;
 564                 if (mem->placement & TTM_PL_FLAG_WC)
 565                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 566                                                   size);
 567                 else
 568                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 569                                                        size);
 570         }
 571         return (!map->virtual) ? -ENOMEM : 0;
 572 }
 573
 574 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 575                            unsigned long start_page,
 576                            unsigned long num_pages,
 577                            struct ttm_bo_kmap_obj *map)
 578 {
 579         struct ttm_mem_reg *mem = &bo->mem;
 580         struct ttm_operation_ctx ctx = {
 581                 .interruptible = false,
 582                 .no_wait_gpu = false
 583         };
 584         struct ttm_tt *ttm = bo->ttm;
 585         pgprot_t prot;
 586         int ret;
 587
 588         BUG_ON(!ttm);
 589
 590         ret = ttm_tt_populate(ttm, &ctx);
 591         if (ret)
 592                 return ret;
 593
 594         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 595                 /*
 596                  * We're mapping a single page, and the desired
 597                  * page protection is consistent with the bo.
 598                  */
 599
 600                 map->bo_kmap_type = ttm_bo_map_kmap;
 601                 map->page = ttm->pages[start_page];
 602                 map->virtual = kmap(map->page);
 603         } else {
 604                 /*
 605                  * We need to use vmap to get the desired page protection
 606                  * or to make the buffer object look contiguous.
 607                  */
 608                 prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
 609                 map->bo_kmap_type = ttm_bo_map_vmap;
 610                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 611                                     0, prot);
 612         }
 613         return (!map->virtual) ? -ENOMEM : 0;
 614 }
 615
 616 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 617                 unsigned long start_page, unsigned long num_pages,
 618                 struct ttm_bo_kmap_obj *map)
 619 {
 620         struct ttm_mem_type_manager *man =
 621                 &bo->bdev->man[bo->mem.mem_type];
 622         unsigned long offset, size;
 623         int ret;
 624
 625         map->virtual = NULL;
 626         map->bo = bo;
 627         if (num_pages > bo->num_pages)
 628                 return -EINVAL;
 629         if (start_page > bo->num_pages)
 630                 return -EINVAL;
 631
 632         (void) ttm_mem_io_lock(man, false);
 633         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 634         ttm_mem_io_unlock(man);
 635         if (ret)
 636                 return ret;
 637         if (!bo->mem.bus.is_iomem) {
 638                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 639         } else {
 640                 offset = start_page << PAGE_SHIFT;
 641                 size = num_pages << PAGE_SHIFT;
 642                 return ttm_bo_ioremap(bo, offset, size, map);
 643         }
 644 }
 645 EXPORT_SYMBOL(ttm_bo_kmap);
 646
 647 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 648 {
 649         struct ttm_buffer_object *bo = map->bo;
 650         struct ttm_mem_type_manager *man =
 651                 &bo->bdev->man[bo->mem.mem_type];
 652
 653         if (!map->virtual)
 654                 return;
 655         switch (map->bo_kmap_type) {
 656         case ttm_bo_map_iomap:
 657                 iounmap(map->virtual);
 658                 break;
 659         case ttm_bo_map_vmap:
 660                 vunmap(map->virtual);
 661                 break;
 662         case ttm_bo_map_kmap:
 663                 kunmap(map->page);
 664                 break;
 665         case ttm_bo_map_premapped:
 666                 break;
 667         default:
 668                 BUG();
 669         }
 670         (void) ttm_mem_io_lock(man, false);
 671         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 672         ttm_mem_io_unlock(man);
 673         map->virtual = NULL;
 674         map->page = NULL;
 675 }
 676 EXPORT_SYMBOL(ttm_bo_kunmap);
 677
 678 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 679                               struct dma_fence *fence,
 680                               bool evict,
 681                               struct ttm_mem_reg *new_mem)
 682 {
 683         struct ttm_bo_device *bdev = bo->bdev;
 684         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 685         struct ttm_mem_reg *old_mem = &bo->mem;
 686         int ret;
 687         struct ttm_buffer_object *ghost_obj;
 688
 689         dma_resv_add_excl_fence(bo->base.resv, fence);
 690         if (evict) {
 691                 ret = ttm_bo_wait(bo, false, false);
 692                 if (ret)
 693                         return ret;
 694
 695                 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 696                         ttm_tt_destroy(bo->ttm);
 697                         bo->ttm = NULL;
 698                 }
 699                 ttm_bo_free_old_node(bo);
 700         } else {
 701                 /**
 702                  * This should help pipeline ordinary buffer moves.
 703                  *
 704                  * Hang old buffer memory on a new buffer object,
 705                  * and leave it to be released when the GPU
 706                  * operation has completed.
 707                  */
 708
 709                 dma_fence_put(bo->moving);
 710                 bo->moving = dma_fence_get(fence);
 711
 712                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 713                 if (ret)
 714                         return ret;
 715
 716                 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
 717
 718                 /**
 719                  * If we're not moving to fixed memory, the TTM object
 720                  * needs to stay alive. Otherwhise hang it on the ghost
 721                  * bo to be unbound and destroyed.
 722                  */
 723
 724                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 725                         ghost_obj->ttm = NULL;
 726                 else
 727                         bo->ttm = NULL;
 728
 729                 dma_resv_unlock(&ghost_obj->base._resv);
 730                 ttm_bo_put(ghost_obj);
 731         }
 732
 733         *old_mem = *new_mem;
 734         new_mem->mm_node = NULL;
 735
 736         return 0;
 737 }
 738 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 739
 740 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 741                          struct dma_fence *fence, bool evict,
 742                          struct ttm_mem_reg *new_mem)
 743 {
 744         struct ttm_bo_device *bdev = bo->bdev;
 745         struct ttm_mem_reg *old_mem = &bo->mem;
 746
 747         struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
 748         struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
 749
 750         int ret;
 751
 752         dma_resv_add_excl_fence(bo->base.resv, fence);
 753
 754         if (!evict) {
 755                 struct ttm_buffer_object *ghost_obj;
 756
 757                 /**
 758                  * This should help pipeline ordinary buffer moves.
 759                  *
 760                  * Hang old buffer memory on a new buffer object,
 761                  * and leave it to be released when the GPU
 762                  * operation has completed.
 763                  */
 764
 765                 dma_fence_put(bo->moving);
 766                 bo->moving = dma_fence_get(fence);
 767
 768                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 769                 if (ret)
 770                         return ret;
 771
 772                 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
 773
 774                 /**
 775                  * If we're not moving to fixed memory, the TTM object
 776                  * needs to stay alive. Otherwhise hang it on the ghost
 777                  * bo to be unbound and destroyed.
 778                  */
 779
 780                 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
 781                         ghost_obj->ttm = NULL;
 782                 else
 783                         bo->ttm = NULL;
 784
 785                 dma_resv_unlock(&ghost_obj->base._resv);
 786                 ttm_bo_put(ghost_obj);
 787
 788         } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
 789
 790                 /**
 791                  * BO doesn't have a TTM we need to bind/unbind. Just remember
 792                  * this eviction and free up the allocation
 793                  */
 794
 795                 spin_lock(&from->move_lock);
 796                 if (!from->move || dma_fence_is_later(fence, from->move)) {
 797                         dma_fence_put(from->move);
 798                         from->move = dma_fence_get(fence);
 799                 }
 800                 spin_unlock(&from->move_lock);
 801
 802                 ttm_bo_free_old_node(bo);
 803
 804                 dma_fence_put(bo->moving);
 805                 bo->moving = dma_fence_get(fence);
 806
 807         } else {
 808                 /**
 809                  * Last resort, wait for the move to be completed.
 810                  *
 811                  * Should never happen in pratice.
 812                  */
 813
 814                 ret = ttm_bo_wait(bo, false, false);
 815                 if (ret)
 816                         return ret;
 817
 818                 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
 819                         ttm_tt_destroy(bo->ttm);
 820                         bo->ttm = NULL;
 821                 }
 822                 ttm_bo_free_old_node(bo);
 823         }
 824
 825         *old_mem = *new_mem;
 826         new_mem->mm_node = NULL;
 827
 828         return 0;
 829 }
 830 EXPORT_SYMBOL(ttm_bo_pipeline_move);
 831
 832 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 833 {
 834         struct ttm_buffer_object *ghost;
 835         int ret;
 836
 837         ret = ttm_buffer_object_transfer(bo, &ghost);
 838         if (ret)
 839                 return ret;
 840
 841         ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);
 842         /* Last resort, wait for the BO to be idle when we are OOM */
 843         if (ret)
 844                 ttm_bo_wait(bo, false, false);
 845
 846         memset(&bo->mem, 0, sizeof(bo->mem));
 847         bo->mem.mem_type = TTM_PL_SYSTEM;
 848         bo->ttm = NULL;
 849
 850         dma_resv_unlock(&ghost->base._resv);
 851         ttm_bo_put(ghost);
 852
 853         return 0;
 854 }