drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include <drm/ttm/ttm_bo_driver.h>
  32 #include <drm/ttm/ttm_placement.h>
  33 #include <drm/drm_vma_manager.h>
  34 #include <linux/io.h>
  35 #include <linux/highmem.h>
  36 #include <linux/wait.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/module.h>
  40 #include <linux/reservation.h>
  41
  42 struct ttm_transfer_obj {
  43         struct ttm_buffer_object base;
  44         struct ttm_buffer_object *bo;
  45 };
  46
  47 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  48 {
  49         ttm_bo_mem_put(bo, &bo->mem);
  50 }
  51
  52 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  53                    struct ttm_operation_ctx *ctx,
  54                     struct ttm_mem_reg *new_mem)
  55 {
  56         struct ttm_tt *ttm = bo->ttm;
  57         struct ttm_mem_reg *old_mem = &bo->mem;
  58         int ret;
  59
  60         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  61                 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
  62
  63                 if (unlikely(ret != 0)) {
  64                         if (ret != -ERESTARTSYS)
  65                                 pr_err("Failed to expire sync object before unbinding TTM\n");
  66                         return ret;
  67                 }
  68
  69                 ttm_tt_unbind(ttm);
  70                 ttm_bo_free_old_node(bo);
  71                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  72                                 TTM_PL_MASK_MEM);
  73                 old_mem->mem_type = TTM_PL_SYSTEM;
  74         }
  75
  76         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  77         if (unlikely(ret != 0))
  78                 return ret;
  79
  80         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  81                 ret = ttm_tt_bind(ttm, new_mem, ctx);
  82                 if (unlikely(ret != 0))
  83                         return ret;
  84         }
  85
  86         *old_mem = *new_mem;
  87         new_mem->mm_node = NULL;
  88
  89         return 0;
  90 }
  91 EXPORT_SYMBOL(ttm_bo_move_ttm);
  92
  93 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
  94 {
  95         if (likely(man->io_reserve_fastpath))
  96                 return 0;
  97
  98         if (interruptible)
  99                 return mutex_lock_interruptible(&man->io_reserve_mutex);
 100
 101         mutex_lock(&man->io_reserve_mutex);
 102         return 0;
 103 }
 104 EXPORT_SYMBOL(ttm_mem_io_lock);
 105
 106 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
 107 {
 108         if (likely(man->io_reserve_fastpath))
 109                 return;
 110
 111         mutex_unlock(&man->io_reserve_mutex);
 112 }
 113 EXPORT_SYMBOL(ttm_mem_io_unlock);
 114
 115 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
 116 {
 117         struct ttm_buffer_object *bo;
 118
 119         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
 120                 return -EAGAIN;
 121
 122         bo = list_first_entry(&man->io_reserve_lru,
 123                               struct ttm_buffer_object,
 124                               io_reserve_lru);
 125         list_del_init(&bo->io_reserve_lru);
 126         ttm_bo_unmap_virtual_locked(bo);
 127
 128         return 0;
 129 }
 130
 131
 132 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
 133                        struct ttm_mem_reg *mem)
 134 {
 135         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 136         int ret = 0;
 137
 138         if (!bdev->driver->io_mem_reserve)
 139                 return 0;
 140         if (likely(man->io_reserve_fastpath))
 141                 return bdev->driver->io_mem_reserve(bdev, mem);
 142
 143         if (bdev->driver->io_mem_reserve &&
 144             mem->bus.io_reserved_count++ == 0) {
 145 retry:
 146                 ret = bdev->driver->io_mem_reserve(bdev, mem);
 147                 if (ret == -EAGAIN) {
 148                         ret = ttm_mem_io_evict(man);
 149                         if (ret == 0)
 150                                 goto retry;
 151                 }
 152         }
 153         return ret;
 154 }
 155 EXPORT_SYMBOL(ttm_mem_io_reserve);
 156
 157 void ttm_mem_io_free(struct ttm_bo_device *bdev,
 158                      struct ttm_mem_reg *mem)
 159 {
 160         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 161
 162         if (likely(man->io_reserve_fastpath))
 163                 return;
 164
 165         if (bdev->driver->io_mem_reserve &&
 166             --mem->bus.io_reserved_count == 0 &&
 167             bdev->driver->io_mem_free)
 168                 bdev->driver->io_mem_free(bdev, mem);
 169
 170 }
 171 EXPORT_SYMBOL(ttm_mem_io_free);
 172
 173 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
 174 {
 175         struct ttm_mem_reg *mem = &bo->mem;
 176         int ret;
 177
 178         if (!mem->bus.io_reserved_vm) {
 179                 struct ttm_mem_type_manager *man =
 180                         &bo->bdev->man[mem->mem_type];
 181
 182                 ret = ttm_mem_io_reserve(bo->bdev, mem);
 183                 if (unlikely(ret != 0))
 184                         return ret;
 185                 mem->bus.io_reserved_vm = true;
 186                 if (man->use_io_reserve_lru)
 187                         list_add_tail(&bo->io_reserve_lru,
 188                                       &man->io_reserve_lru);
 189         }
 190         return 0;
 191 }
 192
 193 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 194 {
 195         struct ttm_mem_reg *mem = &bo->mem;
 196
 197         if (mem->bus.io_reserved_vm) {
 198                 mem->bus.io_reserved_vm = false;
 199                 list_del_init(&bo->io_reserve_lru);
 200                 ttm_mem_io_free(bo->bdev, mem);
 201         }
 202 }
 203
 204 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 205                         void **virtual)
 206 {
 207         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 208         int ret;
 209         void *addr;
 210
 211         *virtual = NULL;
 212         (void) ttm_mem_io_lock(man, false);
 213         ret = ttm_mem_io_reserve(bdev, mem);
 214         ttm_mem_io_unlock(man);
 215         if (ret || !mem->bus.is_iomem)
 216                 return ret;
 217
 218         if (mem->bus.addr) {
 219                 addr = mem->bus.addr;
 220         } else {
 221                 if (mem->placement & TTM_PL_FLAG_WC)
 222                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 223                 else
 224                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 225                 if (!addr) {
 226                         (void) ttm_mem_io_lock(man, false);
 227                         ttm_mem_io_free(bdev, mem);
 228                         ttm_mem_io_unlock(man);
 229                         return -ENOMEM;
 230                 }
 231         }
 232         *virtual = addr;
 233         return 0;
 234 }
 235
 236 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 237                          void *virtual)
 238 {
 239         struct ttm_mem_type_manager *man;
 240
 241         man = &bdev->man[mem->mem_type];
 242
 243         if (virtual && mem->bus.addr == NULL)
 244                 iounmap(virtual);
 245         (void) ttm_mem_io_lock(man, false);
 246         ttm_mem_io_free(bdev, mem);
 247         ttm_mem_io_unlock(man);
 248 }
 249
 250 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 251 {
 252         uint32_t *dstP =
 253             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 254         uint32_t *srcP =
 255             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 256
 257         int i;
 258         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 259                 iowrite32(ioread32(srcP++), dstP++);
 260         return 0;
 261 }
 262
 263 #ifdef CONFIG_X86
 264 #define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
 265 #define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
 266 #else
 267 #define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0,  __prot)
 268 #define __ttm_kunmap_atomic(__addr) vunmap(__addr)
 269 #endif
 270
 271
 272 /**
 273  * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
 274  * specified page protection.
 275  *
 276  * @page: The page to map.
 277  * @prot: The page protection.
 278  *
 279  * This function maps a TTM page using the kmap_atomic api if available,
 280  * otherwise falls back to vmap. The user must make sure that the
 281  * specified page does not have an aliased mapping with a different caching
 282  * policy unless the architecture explicitly allows it. Also mapping and
 283  * unmapping using this api must be correctly nested. Unmapping should
 284  * occur in the reverse order of mapping.
 285  */
 286 void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
 287 {
 288         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
 289                 return kmap_atomic(page);
 290         else
 291                 return __ttm_kmap_atomic_prot(page, prot);
 292 }
 293 EXPORT_SYMBOL(ttm_kmap_atomic_prot);
 294
 295 /**
 296  * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
 297  * ttm_kmap_atomic_prot.
 298  *
 299  * @addr: The virtual address from the map.
 300  * @prot: The page protection.
 301  */
 302 void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
 303 {
 304         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
 305                 kunmap_atomic(addr);
 306         else
 307                 __ttm_kunmap_atomic(addr);
 308 }
 309 EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
 310
 311 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 312                                 unsigned long page,
 313                                 pgprot_t prot)
 314 {
 315         struct page *d = ttm->pages[page];
 316         void *dst;
 317
 318         if (!d)
 319                 return -ENOMEM;
 320
 321         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 322         dst = ttm_kmap_atomic_prot(d, prot);
 323         if (!dst)
 324                 return -ENOMEM;
 325
 326         memcpy_fromio(dst, src, PAGE_SIZE);
 327
 328         ttm_kunmap_atomic_prot(dst, prot);
 329
 330         return 0;
 331 }
 332
 333 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 334                                 unsigned long page,
 335                                 pgprot_t prot)
 336 {
 337         struct page *s = ttm->pages[page];
 338         void *src;
 339
 340         if (!s)
 341                 return -ENOMEM;
 342
 343         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 344         src = ttm_kmap_atomic_prot(s, prot);
 345         if (!src)
 346                 return -ENOMEM;
 347
 348         memcpy_toio(dst, src, PAGE_SIZE);
 349
 350         ttm_kunmap_atomic_prot(src, prot);
 351
 352         return 0;
 353 }
 354
 355 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 356                        struct ttm_operation_ctx *ctx,
 357                        struct ttm_mem_reg *new_mem)
 358 {
 359         struct ttm_bo_device *bdev = bo->bdev;
 360         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 361         struct ttm_tt *ttm = bo->ttm;
 362         struct ttm_mem_reg *old_mem = &bo->mem;
 363         struct ttm_mem_reg old_copy = *old_mem;
 364         void *old_iomap;
 365         void *new_iomap;
 366         int ret;
 367         unsigned long i;
 368         unsigned long page;
 369         unsigned long add = 0;
 370         int dir;
 371
 372         ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 373         if (ret)
 374                 return ret;
 375
 376         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 377         if (ret)
 378                 return ret;
 379         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 380         if (ret)
 381                 goto out;
 382
 383         /*
 384          * Single TTM move. NOP.
 385          */
 386         if (old_iomap == NULL && new_iomap == NULL)
 387                 goto out2;
 388
 389         /*
 390          * Don't move nonexistent data. Clear destination instead.
 391          */
 392         if (old_iomap == NULL &&
 393             (ttm == NULL || (ttm->state == tt_unpopulated &&
 394                              !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
 395                 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
 396                 goto out2;
 397         }
 398
 399         /*
 400          * TTM might be null for moves within the same region.
 401          */
 402         if (ttm) {
 403                 ret = ttm_tt_populate(ttm, ctx);
 404                 if (ret)
 405                         goto out1;
 406         }
 407
 408         add = 0;
 409         dir = 1;
 410
 411         if ((old_mem->mem_type == new_mem->mem_type) &&
 412             (new_mem->start < old_mem->start + old_mem->size)) {
 413                 dir = -1;
 414                 add = new_mem->num_pages - 1;
 415         }
 416
 417         for (i = 0; i < new_mem->num_pages; ++i) {
 418                 page = i * dir + add;
 419                 if (old_iomap == NULL) {
 420                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 421                                                     PAGE_KERNEL);
 422                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 423                                                    prot);
 424                 } else if (new_iomap == NULL) {
 425                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 426                                                     PAGE_KERNEL);
 427                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 428                                                    prot);
 429                 } else {
 430                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 431                 }
 432                 if (ret)
 433                         goto out1;
 434         }
 435         mb();
 436 out2:
 437         old_copy = *old_mem;
 438         *old_mem = *new_mem;
 439         new_mem->mm_node = NULL;
 440
 441         if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 442                 ttm_tt_destroy(ttm);
 443                 bo->ttm = NULL;
 444         }
 445
 446 out1:
 447         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 448 out:
 449         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 450
 451         /*
 452          * On error, keep the mm node!
 453          */
 454         if (!ret)
 455                 ttm_bo_mem_put(bo, &old_copy);
 456         return ret;
 457 }
 458 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 459
 460 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 461 {
 462         struct ttm_transfer_obj *fbo;
 463
 464         fbo = container_of(bo, struct ttm_transfer_obj, base);
 465         ttm_bo_unref(&fbo->bo);
 466         kfree(fbo);
 467 }
 468
 469 /**
 470  * ttm_buffer_object_transfer
 471  *
 472  * @bo: A pointer to a struct ttm_buffer_object.
 473  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 474  * holding the data of @bo with the old placement.
 475  *
 476  * This is a utility function that may be called after an accelerated move
 477  * has been scheduled. A new buffer object is created as a placeholder for
 478  * the old data while it's being copied. When that buffer object is idle,
 479  * it can be destroyed, releasing the space of the old placement.
 480  * Returns:
 481  * !0: Failure.
 482  */
 483
 484 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 485                                       struct ttm_buffer_object **new_obj)
 486 {
 487         struct ttm_transfer_obj *fbo;
 488         int ret;
 489
 490         fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
 491         if (!fbo)
 492                 return -ENOMEM;
 493
 494         fbo->base = *bo;
 495         fbo->bo = ttm_bo_reference(bo);
 496
 497         /**
 498          * Fix up members that we shouldn't copy directly:
 499          * TODO: Explicit member copy would probably be better here.
 500          */
 501
 502         atomic_inc(&bo->bdev->glob->bo_count);
 503         INIT_LIST_HEAD(&fbo->base.ddestroy);
 504         INIT_LIST_HEAD(&fbo->base.lru);
 505         INIT_LIST_HEAD(&fbo->base.swap);
 506         INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
 507         mutex_init(&fbo->base.wu_mutex);
 508         fbo->base.moving = NULL;
 509         drm_vma_node_reset(&fbo->base.vma_node);
 510         atomic_set(&fbo->base.cpu_writers, 0);
 511
 512         kref_init(&fbo->base.list_kref);
 513         kref_init(&fbo->base.kref);
 514         fbo->base.destroy = &ttm_transfered_destroy;
 515         fbo->base.acc_size = 0;
 516         fbo->base.resv = &fbo->base.ttm_resv;
 517         reservation_object_init(fbo->base.resv);
 518         ret = reservation_object_trylock(fbo->base.resv);
 519         WARN_ON(!ret);
 520
 521         *new_obj = &fbo->base;
 522         return 0;
 523 }
 524
 525 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 526 {
 527         /* Cached mappings need no adjustment */
 528         if (caching_flags & TTM_PL_FLAG_CACHED)
 529                 return tmp;
 530
 531 #if defined(__i386__) || defined(__x86_64__)
 532         if (caching_flags & TTM_PL_FLAG_WC)
 533                 tmp = pgprot_writecombine(tmp);
 534         else if (boot_cpu_data.x86 > 3)
 535                 tmp = pgprot_noncached(tmp);
 536 #endif
 537 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
 538     defined(__powerpc__)
 539         if (caching_flags & TTM_PL_FLAG_WC)
 540                 tmp = pgprot_writecombine(tmp);
 541         else
 542                 tmp = pgprot_noncached(tmp);
 543 #endif
 544 #if defined(__sparc__) || defined(__mips__)
 545         tmp = pgprot_noncached(tmp);
 546 #endif
 547         return tmp;
 548 }
 549 EXPORT_SYMBOL(ttm_io_prot);
 550
 551 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 552                           unsigned long offset,
 553                           unsigned long size,
 554                           struct ttm_bo_kmap_obj *map)
 555 {
 556         struct ttm_mem_reg *mem = &bo->mem;
 557
 558         if (bo->mem.bus.addr) {
 559                 map->bo_kmap_type = ttm_bo_map_premapped;
 560                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 561         } else {
 562                 map->bo_kmap_type = ttm_bo_map_iomap;
 563                 if (mem->placement & TTM_PL_FLAG_WC)
 564                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 565                                                   size);
 566                 else
 567                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 568                                                        size);
 569         }
 570         return (!map->virtual) ? -ENOMEM : 0;
 571 }
 572
 573 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 574                            unsigned long start_page,
 575                            unsigned long num_pages,
 576                            struct ttm_bo_kmap_obj *map)
 577 {
 578         struct ttm_mem_reg *mem = &bo->mem;
 579         struct ttm_operation_ctx ctx = {
 580                 .interruptible = false,
 581                 .no_wait_gpu = false
 582         };
 583         struct ttm_tt *ttm = bo->ttm;
 584         pgprot_t prot;
 585         int ret;
 586
 587         BUG_ON(!ttm);
 588
 589         ret = ttm_tt_populate(ttm, &ctx);
 590         if (ret)
 591                 return ret;
 592
 593         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 594                 /*
 595                  * We're mapping a single page, and the desired
 596                  * page protection is consistent with the bo.
 597                  */
 598
 599                 map->bo_kmap_type = ttm_bo_map_kmap;
 600                 map->page = ttm->pages[start_page];
 601                 map->virtual = kmap(map->page);
 602         } else {
 603                 /*
 604                  * We need to use vmap to get the desired page protection
 605                  * or to make the buffer object look contiguous.
 606                  */
 607                 prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
 608                 map->bo_kmap_type = ttm_bo_map_vmap;
 609                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 610                                     0, prot);
 611         }
 612         return (!map->virtual) ? -ENOMEM : 0;
 613 }
 614
 615 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 616                 unsigned long start_page, unsigned long num_pages,
 617                 struct ttm_bo_kmap_obj *map)
 618 {
 619         struct ttm_mem_type_manager *man =
 620                 &bo->bdev->man[bo->mem.mem_type];
 621         unsigned long offset, size;
 622         int ret;
 623
 624         map->virtual = NULL;
 625         map->bo = bo;
 626         if (num_pages > bo->num_pages)
 627                 return -EINVAL;
 628         if (start_page > bo->num_pages)
 629                 return -EINVAL;
 630 #if 0
 631         if (num_pages > 1 && !capable(CAP_SYS_ADMIN))
 632                 return -EPERM;
 633 #endif
 634         (void) ttm_mem_io_lock(man, false);
 635         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 636         ttm_mem_io_unlock(man);
 637         if (ret)
 638                 return ret;
 639         if (!bo->mem.bus.is_iomem) {
 640                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 641         } else {
 642                 offset = start_page << PAGE_SHIFT;
 643                 size = num_pages << PAGE_SHIFT;
 644                 return ttm_bo_ioremap(bo, offset, size, map);
 645         }
 646 }
 647 EXPORT_SYMBOL(ttm_bo_kmap);
 648
 649 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 650 {
 651         struct ttm_buffer_object *bo = map->bo;
 652         struct ttm_mem_type_manager *man =
 653                 &bo->bdev->man[bo->mem.mem_type];
 654
 655         if (!map->virtual)
 656                 return;
 657         switch (map->bo_kmap_type) {
 658         case ttm_bo_map_iomap:
 659                 iounmap(map->virtual);
 660                 break;
 661         case ttm_bo_map_vmap:
 662                 vunmap(map->virtual);
 663                 break;
 664         case ttm_bo_map_kmap:
 665                 kunmap(map->page);
 666                 break;
 667         case ttm_bo_map_premapped:
 668                 break;
 669         default:
 670                 BUG();
 671         }
 672         (void) ttm_mem_io_lock(man, false);
 673         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 674         ttm_mem_io_unlock(man);
 675         map->virtual = NULL;
 676         map->page = NULL;
 677 }
 678 EXPORT_SYMBOL(ttm_bo_kunmap);
 679
 680 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 681                               struct dma_fence *fence,
 682                               bool evict,
 683                               struct ttm_mem_reg *new_mem)
 684 {
 685         struct ttm_bo_device *bdev = bo->bdev;
 686         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 687         struct ttm_mem_reg *old_mem = &bo->mem;
 688         int ret;
 689         struct ttm_buffer_object *ghost_obj;
 690
 691         reservation_object_add_excl_fence(bo->resv, fence);
 692         if (evict) {
 693                 ret = ttm_bo_wait(bo, false, false);
 694                 if (ret)
 695                         return ret;
 696
 697                 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 698                         ttm_tt_destroy(bo->ttm);
 699                         bo->ttm = NULL;
 700                 }
 701                 ttm_bo_free_old_node(bo);
 702         } else {
 703                 /**
 704                  * This should help pipeline ordinary buffer moves.
 705                  *
 706                  * Hang old buffer memory on a new buffer object,
 707                  * and leave it to be released when the GPU
 708                  * operation has completed.
 709                  */
 710
 711                 dma_fence_put(bo->moving);
 712                 bo->moving = dma_fence_get(fence);
 713
 714                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 715                 if (ret)
 716                         return ret;
 717
 718                 reservation_object_add_excl_fence(ghost_obj->resv, fence);
 719
 720                 /**
 721                  * If we're not moving to fixed memory, the TTM object
 722                  * needs to stay alive. Otherwhise hang it on the ghost
 723                  * bo to be unbound and destroyed.
 724                  */
 725
 726                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 727                         ghost_obj->ttm = NULL;
 728                 else
 729                         bo->ttm = NULL;
 730
 731                 ttm_bo_unreserve(ghost_obj);
 732                 ttm_bo_unref(&ghost_obj);
 733         }
 734
 735         *old_mem = *new_mem;
 736         new_mem->mm_node = NULL;
 737
 738         return 0;
 739 }
 740 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 741
 742 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 743                          struct dma_fence *fence, bool evict,
 744                          struct ttm_mem_reg *new_mem)
 745 {
 746         struct ttm_bo_device *bdev = bo->bdev;
 747         struct ttm_mem_reg *old_mem = &bo->mem;
 748
 749         struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
 750         struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
 751
 752         int ret;
 753
 754         reservation_object_add_excl_fence(bo->resv, fence);
 755
 756         if (!evict) {
 757                 struct ttm_buffer_object *ghost_obj;
 758
 759                 /**
 760                  * This should help pipeline ordinary buffer moves.
 761                  *
 762                  * Hang old buffer memory on a new buffer object,
 763                  * and leave it to be released when the GPU
 764                  * operation has completed.
 765                  */
 766
 767                 dma_fence_put(bo->moving);
 768                 bo->moving = dma_fence_get(fence);
 769
 770                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 771                 if (ret)
 772                         return ret;
 773
 774                 reservation_object_add_excl_fence(ghost_obj->resv, fence);
 775
 776                 /**
 777                  * If we're not moving to fixed memory, the TTM object
 778                  * needs to stay alive. Otherwhise hang it on the ghost
 779                  * bo to be unbound and destroyed.
 780                  */
 781
 782                 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
 783                         ghost_obj->ttm = NULL;
 784                 else
 785                         bo->ttm = NULL;
 786
 787                 ttm_bo_unreserve(ghost_obj);
 788                 ttm_bo_unref(&ghost_obj);
 789
 790         } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
 791
 792                 /**
 793                  * BO doesn't have a TTM we need to bind/unbind. Just remember
 794                  * this eviction and free up the allocation
 795                  */
 796
 797                 spin_lock(&from->move_lock);
 798                 if (!from->move || dma_fence_is_later(fence, from->move)) {
 799                         dma_fence_put(from->move);
 800                         from->move = dma_fence_get(fence);
 801                 }
 802                 spin_unlock(&from->move_lock);
 803
 804                 ttm_bo_free_old_node(bo);
 805
 806                 dma_fence_put(bo->moving);
 807                 bo->moving = dma_fence_get(fence);
 808
 809         } else {
 810                 /**
 811                  * Last resort, wait for the move to be completed.
 812                  *
 813                  * Should never happen in pratice.
 814                  */
 815
 816                 ret = ttm_bo_wait(bo, false, false);
 817                 if (ret)
 818                         return ret;
 819
 820                 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
 821                         ttm_tt_destroy(bo->ttm);
 822                         bo->ttm = NULL;
 823                 }
 824                 ttm_bo_free_old_node(bo);
 825         }
 826
 827         *old_mem = *new_mem;
 828         new_mem->mm_node = NULL;
 829
 830         return 0;
 831 }
 832 EXPORT_SYMBOL(ttm_bo_pipeline_move);
 833
 834 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
 835 {
 836         struct ttm_buffer_object *ghost;
 837         int ret;
 838
 839         ret = ttm_buffer_object_transfer(bo, &ghost);
 840         if (ret)
 841                 return ret;
 842
 843         ret = reservation_object_copy_fences(ghost->resv, bo->resv);
 844         /* Last resort, wait for the BO to be idle when we are OOM */
 845         if (ret)
 846                 ttm_bo_wait(bo, false, false);
 847
 848         memset(&bo->mem, 0, sizeof(bo->mem));
 849         bo->mem.mem_type = TTM_PL_SYSTEM;
 850         bo->ttm = NULL;
 851
 852         ttm_bo_unreserve(ghost);
 853         ttm_bo_unref(&ghost);
 854
 855         return 0;
 856 }