drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_module.h>
  35 #include <drm/ttm/ttm_bo_driver.h>
  36 #include <drm/ttm/ttm_placement.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40 #include <linux/mm.h>
  41 #include <linux/file.h>
  42 #include <linux/module.h>
  43 #include <linux/atomic.h>
  44 #include <linux/reservation.h>
  45
  46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  47
  48 static struct attribute ttm_bo_count = {
  49         .name = "bo_count",
  50         .mode = S_IRUGO
  51 };
  52
  53 /* default destructor */
  54 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  55 {
  56         kfree(bo);
  57 }
  58
  59 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
  60                                           uint32_t *mem_type)
  61 {
  62         int pos;
  63
  64         pos = ffs(place->flags & TTM_PL_MASK_MEM);
  65         if (unlikely(!pos))
  66                 return -EINVAL;
  67
  68         *mem_type = pos - 1;
  69         return 0;
  70 }
  71
  72 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
  73 {
  74         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  75         struct drm_printer p = drm_debug_printer(TTM_PFX);
  76
  77         pr_err("    has_type: %d\n", man->has_type);
  78         pr_err("    use_type: %d\n", man->use_type);
  79         pr_err("    flags: 0x%08X\n", man->flags);
  80         pr_err("    gpu_offset: 0x%08llX\n", man->gpu_offset);
  81         pr_err("    size: %llu\n", man->size);
  82         pr_err("    available_caching: 0x%08X\n", man->available_caching);
  83         pr_err("    default_caching: 0x%08X\n", man->default_caching);
  84         if (mem_type != TTM_PL_SYSTEM)
  85                 (*man->func->debug)(man, &p);
  86 }
  87
  88 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  89                                         struct ttm_placement *placement)
  90 {
  91         int i, ret, mem_type;
  92
  93         pr_err("No space for %p (%lu pages, %luK, %luM)\n",
  94                bo, bo->mem.num_pages, bo->mem.size >> 10,
  95                bo->mem.size >> 20);
  96         for (i = 0; i < placement->num_placement; i++) {
  97                 ret = ttm_mem_type_from_place(&placement->placement[i],
  98                                                 &mem_type);
  99                 if (ret)
 100                         return;
 101                 pr_err("  placement[%d]=0x%08X (%d)\n",
 102                        i, placement->placement[i].flags, mem_type);
 103                 ttm_mem_type_debug(bo->bdev, mem_type);
 104         }
 105 }
 106
 107 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 108                                   struct attribute *attr,
 109                                   char *buffer)
 110 {
 111         struct ttm_bo_global *glob =
 112                 container_of(kobj, struct ttm_bo_global, kobj);
 113
 114         return snprintf(buffer, PAGE_SIZE, "%d\n",
 115                                 atomic_read(&glob->bo_count));
 116 }
 117
 118 static struct attribute *ttm_bo_global_attrs[] = {
 119         &ttm_bo_count,
 120         NULL
 121 };
 122
 123 static const struct sysfs_ops ttm_bo_global_ops = {
 124         .show = &ttm_bo_global_show
 125 };
 126
 127 static struct kobj_type ttm_bo_glob_kobj_type  = {
 128         .release = &ttm_bo_global_kobj_release,
 129         .sysfs_ops = &ttm_bo_global_ops,
 130         .default_attrs = ttm_bo_global_attrs
 131 };
 132
 133
 134 static inline uint32_t ttm_bo_type_flags(unsigned type)
 135 {
 136         return 1 << (type);
 137 }
 138
 139 static void ttm_bo_release_list(struct kref *list_kref)
 140 {
 141         struct ttm_buffer_object *bo =
 142             container_of(list_kref, struct ttm_buffer_object, list_kref);
 143         struct ttm_bo_device *bdev = bo->bdev;
 144         size_t acc_size = bo->acc_size;
 145
 146         BUG_ON(kref_read(&bo->list_kref));
 147         BUG_ON(kref_read(&bo->kref));
 148         BUG_ON(atomic_read(&bo->cpu_writers));
 149         BUG_ON(bo->mem.mm_node != NULL);
 150         BUG_ON(!list_empty(&bo->lru));
 151         BUG_ON(!list_empty(&bo->ddestroy));
 152         ttm_tt_destroy(bo->ttm);
 153         atomic_dec(&bo->bdev->glob->bo_count);
 154         dma_fence_put(bo->moving);
 155         reservation_object_fini(&bo->ttm_resv);
 156         mutex_destroy(&bo->wu_mutex);
 157         bo->destroy(bo);
 158         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
 159 }
 160
 161 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 162 {
 163         struct ttm_bo_device *bdev = bo->bdev;
 164         struct ttm_mem_type_manager *man;
 165
 166         reservation_object_assert_held(bo->resv);
 167
 168         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 169                 BUG_ON(!list_empty(&bo->lru));
 170
 171                 man = &bdev->man[bo->mem.mem_type];
 172                 list_add_tail(&bo->lru, &man->lru[bo->priority]);
 173                 kref_get(&bo->list_kref);
 174
 175                 if (bo->ttm && !(bo->ttm->page_flags &
 176                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) {
 177                         list_add_tail(&bo->swap,
 178                                       &bdev->glob->swap_lru[bo->priority]);
 179                         kref_get(&bo->list_kref);
 180                 }
 181         }
 182 }
 183 EXPORT_SYMBOL(ttm_bo_add_to_lru);
 184
 185 static void ttm_bo_ref_bug(struct kref *list_kref)
 186 {
 187         BUG();
 188 }
 189
 190 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 191 {
 192         if (!list_empty(&bo->swap)) {
 193                 list_del_init(&bo->swap);
 194                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 195         }
 196         if (!list_empty(&bo->lru)) {
 197                 list_del_init(&bo->lru);
 198                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 199         }
 200
 201         /*
 202          * TODO: Add a driver hook to delete from
 203          * driver-specific LRU's here.
 204          */
 205 }
 206
 207 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
 208 {
 209         struct ttm_bo_global *glob = bo->bdev->glob;
 210
 211         spin_lock(&glob->lru_lock);
 212         ttm_bo_del_from_lru(bo);
 213         spin_unlock(&glob->lru_lock);
 214 }
 215 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 216
 217 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
 218 {
 219         reservation_object_assert_held(bo->resv);
 220
 221         ttm_bo_del_from_lru(bo);
 222         ttm_bo_add_to_lru(bo);
 223 }
 224 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 225
 226 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 227                                   struct ttm_mem_reg *mem, bool evict,
 228                                   struct ttm_operation_ctx *ctx)
 229 {
 230         struct ttm_bo_device *bdev = bo->bdev;
 231         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 232         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 233         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 234         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 235         int ret = 0;
 236
 237         if (old_is_pci || new_is_pci ||
 238             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 239                 ret = ttm_mem_io_lock(old_man, true);
 240                 if (unlikely(ret != 0))
 241                         goto out_err;
 242                 ttm_bo_unmap_virtual_locked(bo);
 243                 ttm_mem_io_unlock(old_man);
 244         }
 245
 246         /*
 247          * Create and bind a ttm if required.
 248          */
 249
 250         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 251                 if (bo->ttm == NULL) {
 252                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 253                         ret = ttm_tt_create(bo, zero);
 254                         if (ret)
 255                                 goto out_err;
 256                 }
 257
 258                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 259                 if (ret)
 260                         goto out_err;
 261
 262                 if (mem->mem_type != TTM_PL_SYSTEM) {
 263                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
 264                         if (ret)
 265                                 goto out_err;
 266                 }
 267
 268                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 269                         if (bdev->driver->move_notify)
 270                                 bdev->driver->move_notify(bo, evict, mem);
 271                         bo->mem = *mem;
 272                         mem->mm_node = NULL;
 273                         goto moved;
 274                 }
 275         }
 276
 277         if (bdev->driver->move_notify)
 278                 bdev->driver->move_notify(bo, evict, mem);
 279
 280         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 281             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 282                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 283         else if (bdev->driver->move)
 284                 ret = bdev->driver->move(bo, evict, ctx, mem);
 285         else
 286                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 287
 288         if (ret) {
 289                 if (bdev->driver->move_notify) {
 290                         struct ttm_mem_reg tmp_mem = *mem;
 291                         *mem = bo->mem;
 292                         bo->mem = tmp_mem;
 293                         bdev->driver->move_notify(bo, false, mem);
 294                         bo->mem = *mem;
 295                         *mem = tmp_mem;
 296                 }
 297
 298                 goto out_err;
 299         }
 300
 301 moved:
 302         if (bo->evicted) {
 303                 if (bdev->driver->invalidate_caches) {
 304                         ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
 305                         if (ret)
 306                                 pr_err("Can not flush read caches\n");
 307                 }
 308                 bo->evicted = false;
 309         }
 310
 311         if (bo->mem.mm_node)
 312                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 313                     bdev->man[bo->mem.mem_type].gpu_offset;
 314         else
 315                 bo->offset = 0;
 316
 317         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 318         return 0;
 319
 320 out_err:
 321         new_man = &bdev->man[bo->mem.mem_type];
 322         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 323                 ttm_tt_destroy(bo->ttm);
 324                 bo->ttm = NULL;
 325         }
 326
 327         return ret;
 328 }
 329
 330 /**
 331  * Call bo::reserved.
 332  * Will release GPU memory type usage on destruction.
 333  * This is the place to put in driver specific hooks to release
 334  * driver private resources.
 335  * Will release the bo::reserved lock.
 336  */
 337
 338 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 339 {
 340         if (bo->bdev->driver->move_notify)
 341                 bo->bdev->driver->move_notify(bo, false, NULL);
 342
 343         ttm_tt_destroy(bo->ttm);
 344         bo->ttm = NULL;
 345         ttm_bo_mem_put(bo, &bo->mem);
 346 }
 347
 348 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 349 {
 350         int r;
 351
 352         if (bo->resv == &bo->ttm_resv)
 353                 return 0;
 354
 355         BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
 356
 357         r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
 358         if (r)
 359                 reservation_object_unlock(&bo->ttm_resv);
 360
 361         return r;
 362 }
 363
 364 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 365 {
 366         struct reservation_object_list *fobj;
 367         struct dma_fence *fence;
 368         int i;
 369
 370         fobj = reservation_object_get_list(&bo->ttm_resv);
 371         fence = reservation_object_get_excl(&bo->ttm_resv);
 372         if (fence && !fence->ops->signaled)
 373                 dma_fence_enable_sw_signaling(fence);
 374
 375         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 376                 fence = rcu_dereference_protected(fobj->shared[i],
 377                                         reservation_object_held(bo->resv));
 378
 379                 if (!fence->ops->signaled)
 380                         dma_fence_enable_sw_signaling(fence);
 381         }
 382 }
 383
 384 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 385 {
 386         struct ttm_bo_device *bdev = bo->bdev;
 387         struct ttm_bo_global *glob = bdev->glob;
 388         int ret;
 389
 390         ret = ttm_bo_individualize_resv(bo);
 391         if (ret) {
 392                 /* Last resort, if we fail to allocate memory for the
 393                  * fences block for the BO to become idle
 394                  */
 395                 reservation_object_wait_timeout_rcu(bo->resv, true, false,
 396                                                     30 * HZ);
 397                 spin_lock(&glob->lru_lock);
 398                 goto error;
 399         }
 400
 401         spin_lock(&glob->lru_lock);
 402         ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
 403         if (!ret) {
 404                 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
 405                         ttm_bo_del_from_lru(bo);
 406                         spin_unlock(&glob->lru_lock);
 407                         if (bo->resv != &bo->ttm_resv)
 408                                 reservation_object_unlock(&bo->ttm_resv);
 409
 410                         ttm_bo_cleanup_memtype_use(bo);
 411                         reservation_object_unlock(bo->resv);
 412                         return;
 413                 }
 414
 415                 ttm_bo_flush_all_fences(bo);
 416
 417                 /*
 418                  * Make NO_EVICT bos immediately available to
 419                  * shrinkers, now that they are queued for
 420                  * destruction.
 421                  */
 422                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 423                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 424                         ttm_bo_add_to_lru(bo);
 425                 }
 426
 427                 reservation_object_unlock(bo->resv);
 428         }
 429         if (bo->resv != &bo->ttm_resv)
 430                 reservation_object_unlock(&bo->ttm_resv);
 431
 432 error:
 433         kref_get(&bo->list_kref);
 434         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 435         spin_unlock(&glob->lru_lock);
 436
 437         schedule_delayed_work(&bdev->wq,
 438                               ((HZ / 100) < 1) ? 1 : HZ / 100);
 439 }
 440
 441 /**
 442  * function ttm_bo_cleanup_refs
 443  * If bo idle, remove from delayed- and lru lists, and unref.
 444  * If not idle, do nothing.
 445  *
 446  * Must be called with lru_lock and reservation held, this function
 447  * will drop the lru lock and optionally the reservation lock before returning.
 448  *
 449  * @interruptible         Any sleeps should occur interruptibly.
 450  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 451  * @unlock_resv           Unlock the reservation lock as well.
 452  */
 453
 454 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 455                                bool interruptible, bool no_wait_gpu,
 456                                bool unlock_resv)
 457 {
 458         struct ttm_bo_global *glob = bo->bdev->glob;
 459         struct reservation_object *resv;
 460         int ret;
 461
 462         if (unlikely(list_empty(&bo->ddestroy)))
 463                 resv = bo->resv;
 464         else
 465                 resv = &bo->ttm_resv;
 466
 467         if (reservation_object_test_signaled_rcu(resv, true))
 468                 ret = 0;
 469         else
 470                 ret = -EBUSY;
 471
 472         if (ret && !no_wait_gpu) {
 473                 long lret;
 474
 475                 if (unlock_resv)
 476                         reservation_object_unlock(bo->resv);
 477                 spin_unlock(&glob->lru_lock);
 478
 479                 lret = reservation_object_wait_timeout_rcu(resv, true,
 480                                                            interruptible,
 481                                                            30 * HZ);
 482
 483                 if (lret < 0)
 484                         return lret;
 485                 else if (lret == 0)
 486                         return -EBUSY;
 487
 488                 spin_lock(&glob->lru_lock);
 489                 if (unlock_resv && !reservation_object_trylock(bo->resv)) {
 490                         /*
 491                          * We raced, and lost, someone else holds the reservation now,
 492                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 493                          *
 494                          * Even if it's not the case, because we finished waiting any
 495                          * delayed destruction would succeed, so just return success
 496                          * here.
 497                          */
 498                         spin_unlock(&glob->lru_lock);
 499                         return 0;
 500                 }
 501                 ret = 0;
 502         }
 503
 504         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 505                 if (unlock_resv)
 506                         reservation_object_unlock(bo->resv);
 507                 spin_unlock(&glob->lru_lock);
 508                 return ret;
 509         }
 510
 511         ttm_bo_del_from_lru(bo);
 512         list_del_init(&bo->ddestroy);
 513         kref_put(&bo->list_kref, ttm_bo_ref_bug);
 514
 515         spin_unlock(&glob->lru_lock);
 516         ttm_bo_cleanup_memtype_use(bo);
 517
 518         if (unlock_resv)
 519                 reservation_object_unlock(bo->resv);
 520
 521         return 0;
 522 }
 523
 524 /**
 525  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 526  * encountered buffers.
 527  */
 528 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 529 {
 530         struct ttm_bo_global *glob = bdev->glob;
 531         struct list_head removed;
 532         bool empty;
 533
 534         INIT_LIST_HEAD(&removed);
 535
 536         spin_lock(&glob->lru_lock);
 537         while (!list_empty(&bdev->ddestroy)) {
 538                 struct ttm_buffer_object *bo;
 539
 540                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 541                                       ddestroy);
 542                 kref_get(&bo->list_kref);
 543                 list_move_tail(&bo->ddestroy, &removed);
 544
 545                 if (remove_all || bo->resv != &bo->ttm_resv) {
 546                         spin_unlock(&glob->lru_lock);
 547                         reservation_object_lock(bo->resv, NULL);
 548
 549                         spin_lock(&glob->lru_lock);
 550                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 551
 552                 } else if (reservation_object_trylock(bo->resv)) {
 553                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 554                 } else {
 555                         spin_unlock(&glob->lru_lock);
 556                 }
 557
 558                 kref_put(&bo->list_kref, ttm_bo_release_list);
 559                 spin_lock(&glob->lru_lock);
 560         }
 561         list_splice_tail(&removed, &bdev->ddestroy);
 562         empty = list_empty(&bdev->ddestroy);
 563         spin_unlock(&glob->lru_lock);
 564
 565         return empty;
 566 }
 567
 568 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 569 {
 570         struct ttm_bo_device *bdev =
 571             container_of(work, struct ttm_bo_device, wq.work);
 572
 573         if (!ttm_bo_delayed_delete(bdev, false))
 574                 schedule_delayed_work(&bdev->wq,
 575                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 576 }
 577
 578 static void ttm_bo_release(struct kref *kref)
 579 {
 580         struct ttm_buffer_object *bo =
 581             container_of(kref, struct ttm_buffer_object, kref);
 582         struct ttm_bo_device *bdev = bo->bdev;
 583         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 584
 585         drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
 586         ttm_mem_io_lock(man, false);
 587         ttm_mem_io_free_vm(bo);
 588         ttm_mem_io_unlock(man);
 589         ttm_bo_cleanup_refs_or_queue(bo);
 590         kref_put(&bo->list_kref, ttm_bo_release_list);
 591 }
 592
 593 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 594 {
 595         struct ttm_buffer_object *bo = *p_bo;
 596
 597         *p_bo = NULL;
 598         kref_put(&bo->kref, ttm_bo_release);
 599 }
 600 EXPORT_SYMBOL(ttm_bo_unref);
 601
 602 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 603 {
 604         return cancel_delayed_work_sync(&bdev->wq);
 605 }
 606 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 607
 608 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 609 {
 610         if (resched)
 611                 schedule_delayed_work(&bdev->wq,
 612                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 613 }
 614 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 615
 616 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 617                         struct ttm_operation_ctx *ctx)
 618 {
 619         struct ttm_bo_device *bdev = bo->bdev;
 620         struct ttm_mem_reg evict_mem;
 621         struct ttm_placement placement;
 622         int ret = 0;
 623
 624         reservation_object_assert_held(bo->resv);
 625
 626         placement.num_placement = 0;
 627         placement.num_busy_placement = 0;
 628         bdev->driver->evict_flags(bo, &placement);
 629
 630         if (!placement.num_placement && !placement.num_busy_placement) {
 631                 ret = ttm_bo_pipeline_gutting(bo);
 632                 if (ret)
 633                         return ret;
 634
 635                 return ttm_tt_create(bo, false);
 636         }
 637
 638         evict_mem = bo->mem;
 639         evict_mem.mm_node = NULL;
 640         evict_mem.bus.io_reserved_vm = false;
 641         evict_mem.bus.io_reserved_count = 0;
 642
 643         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 644         if (ret) {
 645                 if (ret != -ERESTARTSYS) {
 646                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 647                                bo);
 648                         ttm_bo_mem_space_debug(bo, &placement);
 649                 }
 650                 goto out;
 651         }
 652
 653         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 654         if (unlikely(ret)) {
 655                 if (ret != -ERESTARTSYS)
 656                         pr_err("Buffer eviction failed\n");
 657                 ttm_bo_mem_put(bo, &evict_mem);
 658                 goto out;
 659         }
 660         bo->evicted = true;
 661 out:
 662         return ret;
 663 }
 664
 665 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 666                               const struct ttm_place *place)
 667 {
 668         /* Don't evict this BO if it's outside of the
 669          * requested placement range
 670          */
 671         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 672             (place->lpfn && place->lpfn <= bo->mem.start))
 673                 return false;
 674
 675         return true;
 676 }
 677 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 678
 679 /**
 680  * Check the target bo is allowable to be evicted or swapout, including cases:
 681  *
 682  * a. if share same reservation object with ctx->resv, have assumption
 683  * reservation objects should already be locked, so not lock again and
 684  * return true directly when either the opreation allow_reserved_eviction
 685  * or the target bo already is in delayed free list;
 686  *
 687  * b. Otherwise, trylock it.
 688  */
 689 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 690                         struct ttm_operation_ctx *ctx, bool *locked)
 691 {
 692         bool ret = false;
 693
 694         *locked = false;
 695         if (bo->resv == ctx->resv) {
 696                 reservation_object_assert_held(bo->resv);
 697                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
 698                     || !list_empty(&bo->ddestroy))
 699                         ret = true;
 700         } else {
 701                 *locked = reservation_object_trylock(bo->resv);
 702                 ret = *locked;
 703         }
 704
 705         return ret;
 706 }
 707
 708 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 709                                uint32_t mem_type,
 710                                const struct ttm_place *place,
 711                                struct ttm_operation_ctx *ctx)
 712 {
 713         struct ttm_bo_global *glob = bdev->glob;
 714         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 715         struct ttm_buffer_object *bo = NULL;
 716         bool locked = false;
 717         unsigned i;
 718         int ret;
 719
 720         spin_lock(&glob->lru_lock);
 721         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 722                 list_for_each_entry(bo, &man->lru[i], lru) {
 723                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked))
 724                                 continue;
 725
 726                         if (place && !bdev->driver->eviction_valuable(bo,
 727                                                                       place)) {
 728                                 if (locked)
 729                                         reservation_object_unlock(bo->resv);
 730                                 continue;
 731                         }
 732                         break;
 733                 }
 734
 735                 /* If the inner loop terminated early, we have our candidate */
 736                 if (&bo->lru != &man->lru[i])
 737                         break;
 738
 739                 bo = NULL;
 740         }
 741
 742         if (!bo) {
 743                 spin_unlock(&glob->lru_lock);
 744                 return -EBUSY;
 745         }
 746
 747         kref_get(&bo->list_kref);
 748
 749         if (!list_empty(&bo->ddestroy)) {
 750                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 751                                           ctx->no_wait_gpu, locked);
 752                 kref_put(&bo->list_kref, ttm_bo_release_list);
 753                 return ret;
 754         }
 755
 756         ttm_bo_del_from_lru(bo);
 757         spin_unlock(&glob->lru_lock);
 758
 759         ret = ttm_bo_evict(bo, ctx);
 760         if (locked) {
 761                 ttm_bo_unreserve(bo);
 762         } else {
 763                 spin_lock(&glob->lru_lock);
 764                 ttm_bo_add_to_lru(bo);
 765                 spin_unlock(&glob->lru_lock);
 766         }
 767
 768         kref_put(&bo->list_kref, ttm_bo_release_list);
 769         return ret;
 770 }
 771
 772 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 773 {
 774         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 775
 776         if (mem->mm_node)
 777                 (*man->func->put_node)(man, mem);
 778 }
 779 EXPORT_SYMBOL(ttm_bo_mem_put);
 780
 781 /**
 782  * Add the last move fence to the BO and reserve a new shared slot.
 783  */
 784 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 785                                  struct ttm_mem_type_manager *man,
 786                                  struct ttm_mem_reg *mem)
 787 {
 788         struct dma_fence *fence;
 789         int ret;
 790
 791         spin_lock(&man->move_lock);
 792         fence = dma_fence_get(man->move);
 793         spin_unlock(&man->move_lock);
 794
 795         if (fence) {
 796                 reservation_object_add_shared_fence(bo->resv, fence);
 797
 798                 ret = reservation_object_reserve_shared(bo->resv);
 799                 if (unlikely(ret))
 800                         return ret;
 801
 802                 dma_fence_put(bo->moving);
 803                 bo->moving = fence;
 804         }
 805
 806         return 0;
 807 }
 808
 809 /**
 810  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 811  * space, or we've evicted everything and there isn't enough space.
 812  */
 813 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 814                                         uint32_t mem_type,
 815                                         const struct ttm_place *place,
 816                                         struct ttm_mem_reg *mem,
 817                                         struct ttm_operation_ctx *ctx)
 818 {
 819         struct ttm_bo_device *bdev = bo->bdev;
 820         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 821         int ret;
 822
 823         do {
 824                 ret = (*man->func->get_node)(man, bo, place, mem);
 825                 if (unlikely(ret != 0))
 826                         return ret;
 827                 if (mem->mm_node)
 828                         break;
 829                 ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);
 830                 if (unlikely(ret != 0))
 831                         return ret;
 832         } while (1);
 833         mem->mem_type = mem_type;
 834         return ttm_bo_add_move_fence(bo, man, mem);
 835 }
 836
 837 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 838                                       uint32_t cur_placement,
 839                                       uint32_t proposed_placement)
 840 {
 841         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 842         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 843
 844         /**
 845          * Keep current caching if possible.
 846          */
 847
 848         if ((cur_placement & caching) != 0)
 849                 result |= (cur_placement & caching);
 850         else if ((man->default_caching & caching) != 0)
 851                 result |= man->default_caching;
 852         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 853                 result |= TTM_PL_FLAG_CACHED;
 854         else if ((TTM_PL_FLAG_WC & caching) != 0)
 855                 result |= TTM_PL_FLAG_WC;
 856         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 857                 result |= TTM_PL_FLAG_UNCACHED;
 858
 859         return result;
 860 }
 861
 862 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 863                                  uint32_t mem_type,
 864                                  const struct ttm_place *place,
 865                                  uint32_t *masked_placement)
 866 {
 867         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 868
 869         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 870                 return false;
 871
 872         if ((place->flags & man->available_caching) == 0)
 873                 return false;
 874
 875         cur_flags |= (place->flags & man->available_caching);
 876
 877         *masked_placement = cur_flags;
 878         return true;
 879 }
 880
 881 /**
 882  * Creates space for memory region @mem according to its type.
 883  *
 884  * This function first searches for free space in compatible memory types in
 885  * the priority order defined by the driver.  If free space isn't found, then
 886  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 887  * space.
 888  */
 889 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 890                         struct ttm_placement *placement,
 891                         struct ttm_mem_reg *mem,
 892                         struct ttm_operation_ctx *ctx)
 893 {
 894         struct ttm_bo_device *bdev = bo->bdev;
 895         struct ttm_mem_type_manager *man;
 896         uint32_t mem_type = TTM_PL_SYSTEM;
 897         uint32_t cur_flags = 0;
 898         bool type_found = false;
 899         bool type_ok = false;
 900         bool has_erestartsys = false;
 901         int i, ret;
 902
 903         ret = reservation_object_reserve_shared(bo->resv);
 904         if (unlikely(ret))
 905                 return ret;
 906
 907         mem->mm_node = NULL;
 908         for (i = 0; i < placement->num_placement; ++i) {
 909                 const struct ttm_place *place = &placement->placement[i];
 910
 911                 ret = ttm_mem_type_from_place(place, &mem_type);
 912                 if (ret)
 913                         return ret;
 914                 man = &bdev->man[mem_type];
 915                 if (!man->has_type || !man->use_type)
 916                         continue;
 917
 918                 type_ok = ttm_bo_mt_compatible(man, mem_type, place,
 919                                                 &cur_flags);
 920
 921                 if (!type_ok)
 922                         continue;
 923
 924                 type_found = true;
 925                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 926                                                   cur_flags);
 927                 /*
 928                  * Use the access and other non-mapping-related flag bits from
 929                  * the memory placement flags to the current flags
 930                  */
 931                 ttm_flag_masked(&cur_flags, place->flags,
 932                                 ~TTM_PL_MASK_MEMTYPE);
 933
 934                 if (mem_type == TTM_PL_SYSTEM)
 935                         break;
 936
 937                 ret = (*man->func->get_node)(man, bo, place, mem);
 938                 if (unlikely(ret))
 939                         return ret;
 940
 941                 if (mem->mm_node) {
 942                         ret = ttm_bo_add_move_fence(bo, man, mem);
 943                         if (unlikely(ret)) {
 944                                 (*man->func->put_node)(man, mem);
 945                                 return ret;
 946                         }
 947                         break;
 948                 }
 949         }
 950
 951         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
 952                 mem->mem_type = mem_type;
 953                 mem->placement = cur_flags;
 954                 return 0;
 955         }
 956
 957         for (i = 0; i < placement->num_busy_placement; ++i) {
 958                 const struct ttm_place *place = &placement->busy_placement[i];
 959
 960                 ret = ttm_mem_type_from_place(place, &mem_type);
 961                 if (ret)
 962                         return ret;
 963                 man = &bdev->man[mem_type];
 964                 if (!man->has_type || !man->use_type)
 965                         continue;
 966                 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
 967                         continue;
 968
 969                 type_found = true;
 970                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 971                                                   cur_flags);
 972                 /*
 973                  * Use the access and other non-mapping-related flag bits from
 974                  * the memory placement flags to the current flags
 975                  */
 976                 ttm_flag_masked(&cur_flags, place->flags,
 977                                 ~TTM_PL_MASK_MEMTYPE);
 978
 979                 if (mem_type == TTM_PL_SYSTEM) {
 980                         mem->mem_type = mem_type;
 981                         mem->placement = cur_flags;
 982                         mem->mm_node = NULL;
 983                         return 0;
 984                 }
 985
 986                 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx);
 987                 if (ret == 0 && mem->mm_node) {
 988                         mem->placement = cur_flags;
 989                         return 0;
 990                 }
 991                 if (ret == -ERESTARTSYS)
 992                         has_erestartsys = true;
 993         }
 994
 995         if (!type_found) {
 996                 pr_err(TTM_PFX "No compatible memory type found\n");
 997                 return -EINVAL;
 998         }
 999
1000         return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
1001 }
1002 EXPORT_SYMBOL(ttm_bo_mem_space);
1003
1004 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1005                               struct ttm_placement *placement,
1006                               struct ttm_operation_ctx *ctx)
1007 {
1008         int ret = 0;
1009         struct ttm_mem_reg mem;
1010
1011         reservation_object_assert_held(bo->resv);
1012
1013         mem.num_pages = bo->num_pages;
1014         mem.size = mem.num_pages << PAGE_SHIFT;
1015         mem.page_alignment = bo->mem.page_alignment;
1016         mem.bus.io_reserved_vm = false;
1017         mem.bus.io_reserved_count = 0;
1018         /*
1019          * Determine where to move the buffer.
1020          */
1021         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1022         if (ret)
1023                 goto out_unlock;
1024         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1025 out_unlock:
1026         if (ret && mem.mm_node)
1027                 ttm_bo_mem_put(bo, &mem);
1028         return ret;
1029 }
1030
1031 static bool ttm_bo_places_compat(const struct ttm_place *places,
1032                                  unsigned num_placement,
1033                                  struct ttm_mem_reg *mem,
1034                                  uint32_t *new_flags)
1035 {
1036         unsigned i;
1037
1038         for (i = 0; i < num_placement; i++) {
1039                 const struct ttm_place *heap = &places[i];
1040
1041                 if (mem->mm_node && (mem->start < heap->fpfn ||
1042                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1043                         continue;
1044
1045                 *new_flags = heap->flags;
1046                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1047                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1048                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1049                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1050                         return true;
1051         }
1052         return false;
1053 }
1054
1055 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1056                        struct ttm_mem_reg *mem,
1057                        uint32_t *new_flags)
1058 {
1059         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1060                                  mem, new_flags))
1061                 return true;
1062
1063         if ((placement->busy_placement != placement->placement ||
1064              placement->num_busy_placement > placement->num_placement) &&
1065             ttm_bo_places_compat(placement->busy_placement,
1066                                  placement->num_busy_placement,
1067                                  mem, new_flags))
1068                 return true;
1069
1070         return false;
1071 }
1072 EXPORT_SYMBOL(ttm_bo_mem_compat);
1073
1074 int ttm_bo_validate(struct ttm_buffer_object *bo,
1075                     struct ttm_placement *placement,
1076                     struct ttm_operation_ctx *ctx)
1077 {
1078         int ret;
1079         uint32_t new_flags;
1080
1081         reservation_object_assert_held(bo->resv);
1082         /*
1083          * Check whether we need to move buffer.
1084          */
1085         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1086                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1087                 if (ret)
1088                         return ret;
1089         } else {
1090                 /*
1091                  * Use the access and other non-mapping-related flag bits from
1092                  * the compatible memory placement flags to the active flags
1093                  */
1094                 ttm_flag_masked(&bo->mem.placement, new_flags,
1095                                 ~TTM_PL_MASK_MEMTYPE);
1096         }
1097         /*
1098          * We might need to add a TTM.
1099          */
1100         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1101                 ret = ttm_tt_create(bo, true);
1102                 if (ret)
1103                         return ret;
1104         }
1105         return 0;
1106 }
1107 EXPORT_SYMBOL(ttm_bo_validate);
1108
1109 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1110                          struct ttm_buffer_object *bo,
1111                          unsigned long size,
1112                          enum ttm_bo_type type,
1113                          struct ttm_placement *placement,
1114                          uint32_t page_alignment,
1115                          struct ttm_operation_ctx *ctx,
1116                          size_t acc_size,
1117                          struct sg_table *sg,
1118                          struct reservation_object *resv,
1119                          void (*destroy) (struct ttm_buffer_object *))
1120 {
1121         int ret = 0;
1122         unsigned long num_pages;
1123         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1124         bool locked;
1125
1126         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1127         if (ret) {
1128                 pr_err("Out of kernel memory\n");
1129                 if (destroy)
1130                         (*destroy)(bo);
1131                 else
1132                         kfree(bo);
1133                 return -ENOMEM;
1134         }
1135
1136         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1137         if (num_pages == 0) {
1138                 pr_err("Illegal buffer object size\n");
1139                 if (destroy)
1140                         (*destroy)(bo);
1141                 else
1142                         kfree(bo);
1143                 ttm_mem_global_free(mem_glob, acc_size);
1144                 return -EINVAL;
1145         }
1146         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1147
1148         kref_init(&bo->kref);
1149         kref_init(&bo->list_kref);
1150         atomic_set(&bo->cpu_writers, 0);
1151         INIT_LIST_HEAD(&bo->lru);
1152         INIT_LIST_HEAD(&bo->ddestroy);
1153         INIT_LIST_HEAD(&bo->swap);
1154         INIT_LIST_HEAD(&bo->io_reserve_lru);
1155         mutex_init(&bo->wu_mutex);
1156         bo->bdev = bdev;
1157         bo->type = type;
1158         bo->num_pages = num_pages;
1159         bo->mem.size = num_pages << PAGE_SHIFT;
1160         bo->mem.mem_type = TTM_PL_SYSTEM;
1161         bo->mem.num_pages = bo->num_pages;
1162         bo->mem.mm_node = NULL;
1163         bo->mem.page_alignment = page_alignment;
1164         bo->mem.bus.io_reserved_vm = false;
1165         bo->mem.bus.io_reserved_count = 0;
1166         bo->moving = NULL;
1167         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1168         bo->acc_size = acc_size;
1169         bo->sg = sg;
1170         if (resv) {
1171                 bo->resv = resv;
1172                 reservation_object_assert_held(bo->resv);
1173         } else {
1174                 bo->resv = &bo->ttm_resv;
1175         }
1176         reservation_object_init(&bo->ttm_resv);
1177         atomic_inc(&bo->bdev->glob->bo_count);
1178         drm_vma_node_reset(&bo->vma_node);
1179
1180         /*
1181          * For ttm_bo_type_device buffers, allocate
1182          * address space from the device.
1183          */
1184         if (bo->type == ttm_bo_type_device ||
1185             bo->type == ttm_bo_type_sg)
1186                 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
1187                                          bo->mem.num_pages);
1188
1189         /* passed reservation objects should already be locked,
1190          * since otherwise lockdep will be angered in radeon.
1191          */
1192         if (!resv) {
1193                 locked = reservation_object_trylock(bo->resv);
1194                 WARN_ON(!locked);
1195         }
1196
1197         if (likely(!ret))
1198                 ret = ttm_bo_validate(bo, placement, ctx);
1199
1200         if (unlikely(ret)) {
1201                 if (!resv)
1202                         ttm_bo_unreserve(bo);
1203
1204                 ttm_bo_unref(&bo);
1205                 return ret;
1206         }
1207
1208         if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
1209                 spin_lock(&bdev->glob->lru_lock);
1210                 ttm_bo_add_to_lru(bo);
1211                 spin_unlock(&bdev->glob->lru_lock);
1212         }
1213
1214         return ret;
1215 }
1216 EXPORT_SYMBOL(ttm_bo_init_reserved);
1217
1218 int ttm_bo_init(struct ttm_bo_device *bdev,
1219                 struct ttm_buffer_object *bo,
1220                 unsigned long size,
1221                 enum ttm_bo_type type,
1222                 struct ttm_placement *placement,
1223                 uint32_t page_alignment,
1224                 bool interruptible,
1225                 size_t acc_size,
1226                 struct sg_table *sg,
1227                 struct reservation_object *resv,
1228                 void (*destroy) (struct ttm_buffer_object *))
1229 {
1230         struct ttm_operation_ctx ctx = { interruptible, false };
1231         int ret;
1232
1233         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1234                                    page_alignment, &ctx, acc_size,
1235                                    sg, resv, destroy);
1236         if (ret)
1237                 return ret;
1238
1239         if (!resv)
1240                 ttm_bo_unreserve(bo);
1241
1242         return 0;
1243 }
1244 EXPORT_SYMBOL(ttm_bo_init);
1245
1246 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1247                        unsigned long bo_size,
1248                        unsigned struct_size)
1249 {
1250         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1251         size_t size = 0;
1252
1253         size += ttm_round_pot(struct_size);
1254         size += ttm_round_pot(npages * sizeof(void *));
1255         size += ttm_round_pot(sizeof(struct ttm_tt));
1256         return size;
1257 }
1258 EXPORT_SYMBOL(ttm_bo_acc_size);
1259
1260 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1261                            unsigned long bo_size,
1262                            unsigned struct_size)
1263 {
1264         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1265         size_t size = 0;
1266
1267         size += ttm_round_pot(struct_size);
1268         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1269         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1270         return size;
1271 }
1272 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1273
1274 int ttm_bo_create(struct ttm_bo_device *bdev,
1275                         unsigned long size,
1276                         enum ttm_bo_type type,
1277                         struct ttm_placement *placement,
1278                         uint32_t page_alignment,
1279                         bool interruptible,
1280                         struct ttm_buffer_object **p_bo)
1281 {
1282         struct ttm_buffer_object *bo;
1283         size_t acc_size;
1284         int ret;
1285
1286         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1287         if (unlikely(bo == NULL))
1288                 return -ENOMEM;
1289
1290         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1291         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1292                           interruptible, acc_size,
1293                           NULL, NULL, NULL);
1294         if (likely(ret == 0))
1295                 *p_bo = bo;
1296
1297         return ret;
1298 }
1299 EXPORT_SYMBOL(ttm_bo_create);
1300
1301 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1302                                    unsigned mem_type)
1303 {
1304         struct ttm_operation_ctx ctx = {
1305                 .interruptible = false,
1306                 .no_wait_gpu = false,
1307                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1308         };
1309         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1310         struct ttm_bo_global *glob = bdev->glob;
1311         struct dma_fence *fence;
1312         int ret;
1313         unsigned i;
1314
1315         /*
1316          * Can't use standard list traversal since we're unlocking.
1317          */
1318
1319         spin_lock(&glob->lru_lock);
1320         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1321                 while (!list_empty(&man->lru[i])) {
1322                         spin_unlock(&glob->lru_lock);
1323                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
1324                         if (ret)
1325                                 return ret;
1326                         spin_lock(&glob->lru_lock);
1327                 }
1328         }
1329         spin_unlock(&glob->lru_lock);
1330
1331         spin_lock(&man->move_lock);
1332         fence = dma_fence_get(man->move);
1333         spin_unlock(&man->move_lock);
1334
1335         if (fence) {
1336                 ret = dma_fence_wait(fence, false);
1337                 dma_fence_put(fence);
1338                 if (ret)
1339                         return ret;
1340         }
1341
1342         return 0;
1343 }
1344
1345 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1346 {
1347         struct ttm_mem_type_manager *man;
1348         int ret = -EINVAL;
1349
1350         if (mem_type >= TTM_NUM_MEM_TYPES) {
1351                 pr_err("Illegal memory type %d\n", mem_type);
1352                 return ret;
1353         }
1354         man = &bdev->man[mem_type];
1355
1356         if (!man->has_type) {
1357                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1358                        mem_type);
1359                 return ret;
1360         }
1361
1362         man->use_type = false;
1363         man->has_type = false;
1364
1365         ret = 0;
1366         if (mem_type > 0) {
1367                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1368                 if (ret) {
1369                         pr_err("Cleanup eviction failed\n");
1370                         return ret;
1371                 }
1372
1373                 ret = (*man->func->takedown)(man);
1374         }
1375
1376         dma_fence_put(man->move);
1377         man->move = NULL;
1378
1379         return ret;
1380 }
1381 EXPORT_SYMBOL(ttm_bo_clean_mm);
1382
1383 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1384 {
1385         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1386
1387         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1388                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1389                 return -EINVAL;
1390         }
1391
1392         if (!man->has_type) {
1393                 pr_err("Memory type %u has not been initialized\n", mem_type);
1394                 return 0;
1395         }
1396
1397         return ttm_bo_force_list_clean(bdev, mem_type);
1398 }
1399 EXPORT_SYMBOL(ttm_bo_evict_mm);
1400
1401 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1402                         unsigned long p_size)
1403 {
1404         int ret;
1405         struct ttm_mem_type_manager *man;
1406         unsigned i;
1407
1408         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1409         man = &bdev->man[type];
1410         BUG_ON(man->has_type);
1411         man->io_reserve_fastpath = true;
1412         man->use_io_reserve_lru = false;
1413         mutex_init(&man->io_reserve_mutex);
1414         spin_lock_init(&man->move_lock);
1415         INIT_LIST_HEAD(&man->io_reserve_lru);
1416
1417         ret = bdev->driver->init_mem_type(bdev, type, man);
1418         if (ret)
1419                 return ret;
1420         man->bdev = bdev;
1421
1422         if (type != TTM_PL_SYSTEM) {
1423                 ret = (*man->func->init)(man, p_size);
1424                 if (ret)
1425                         return ret;
1426         }
1427         man->has_type = true;
1428         man->use_type = true;
1429         man->size = p_size;
1430
1431         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1432                 INIT_LIST_HEAD(&man->lru[i]);
1433         man->move = NULL;
1434
1435         return 0;
1436 }
1437 EXPORT_SYMBOL(ttm_bo_init_mm);
1438
1439 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1440 {
1441         struct ttm_bo_global *glob =
1442                 container_of(kobj, struct ttm_bo_global, kobj);
1443
1444         __free_page(glob->dummy_read_page);
1445         kfree(glob);
1446 }
1447
1448 void ttm_bo_global_release(struct drm_global_reference *ref)
1449 {
1450         struct ttm_bo_global *glob = ref->object;
1451
1452         kobject_del(&glob->kobj);
1453         kobject_put(&glob->kobj);
1454 }
1455 EXPORT_SYMBOL(ttm_bo_global_release);
1456
1457 int ttm_bo_global_init(struct drm_global_reference *ref)
1458 {
1459         struct ttm_bo_global_ref *bo_ref =
1460                 container_of(ref, struct ttm_bo_global_ref, ref);
1461         struct ttm_bo_global *glob = ref->object;
1462         int ret;
1463         unsigned i;
1464
1465         mutex_init(&glob->device_list_mutex);
1466         spin_lock_init(&glob->lru_lock);
1467         glob->mem_glob = bo_ref->mem_glob;
1468         glob->mem_glob->bo_glob = glob;
1469         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1470
1471         if (unlikely(glob->dummy_read_page == NULL)) {
1472                 ret = -ENOMEM;
1473                 goto out_no_drp;
1474         }
1475
1476         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1477                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1478         INIT_LIST_HEAD(&glob->device_list);
1479         atomic_set(&glob->bo_count, 0);
1480
1481         ret = kobject_init_and_add(
1482                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1483         if (unlikely(ret != 0))
1484                 kobject_put(&glob->kobj);
1485         return ret;
1486 out_no_drp:
1487         kfree(glob);
1488         return ret;
1489 }
1490 EXPORT_SYMBOL(ttm_bo_global_init);
1491
1492
1493 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1494 {
1495         int ret = 0;
1496         unsigned i = TTM_NUM_MEM_TYPES;
1497         struct ttm_mem_type_manager *man;
1498         struct ttm_bo_global *glob = bdev->glob;
1499
1500         while (i--) {
1501                 man = &bdev->man[i];
1502                 if (man->has_type) {
1503                         man->use_type = false;
1504                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1505                                 ret = -EBUSY;
1506                                 pr_err("DRM memory manager type %d is not clean\n",
1507                                        i);
1508                         }
1509                         man->has_type = false;
1510                 }
1511         }
1512
1513         mutex_lock(&glob->device_list_mutex);
1514         list_del(&bdev->device_list);
1515         mutex_unlock(&glob->device_list_mutex);
1516
1517         cancel_delayed_work_sync(&bdev->wq);
1518
1519         if (ttm_bo_delayed_delete(bdev, true))
1520                 pr_debug("Delayed destroy list was clean\n");
1521
1522         spin_lock(&glob->lru_lock);
1523         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1524                 if (list_empty(&bdev->man[0].lru[0]))
1525                         pr_debug("Swap list %d was clean\n", i);
1526         spin_unlock(&glob->lru_lock);
1527
1528         drm_vma_offset_manager_destroy(&bdev->vma_manager);
1529
1530         return ret;
1531 }
1532 EXPORT_SYMBOL(ttm_bo_device_release);
1533
1534 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1535                        struct ttm_bo_global *glob,
1536                        struct ttm_bo_driver *driver,
1537                        struct address_space *mapping,
1538                        uint64_t file_page_offset,
1539                        bool need_dma32)
1540 {
1541         int ret = -EINVAL;
1542
1543         bdev->driver = driver;
1544
1545         memset(bdev->man, 0, sizeof(bdev->man));
1546
1547         /*
1548          * Initialize the system memory buffer type.
1549          * Other types need to be driver / IOCTL initialized.
1550          */
1551         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1552         if (unlikely(ret != 0))
1553                 goto out_no_sys;
1554
1555         drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
1556                                     0x10000000);
1557         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1558         INIT_LIST_HEAD(&bdev->ddestroy);
1559         bdev->dev_mapping = mapping;
1560         bdev->glob = glob;
1561         bdev->need_dma32 = need_dma32;
1562         mutex_lock(&glob->device_list_mutex);
1563         list_add_tail(&bdev->device_list, &glob->device_list);
1564         mutex_unlock(&glob->device_list_mutex);
1565
1566         return 0;
1567 out_no_sys:
1568         return ret;
1569 }
1570 EXPORT_SYMBOL(ttm_bo_device_init);
1571
1572 /*
1573  * buffer object vm functions.
1574  */
1575
1576 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1577 {
1578         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1579
1580         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1581                 if (mem->mem_type == TTM_PL_SYSTEM)
1582                         return false;
1583
1584                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1585                         return false;
1586
1587                 if (mem->placement & TTM_PL_FLAG_CACHED)
1588                         return false;
1589         }
1590         return true;
1591 }
1592
1593 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1594 {
1595         struct ttm_bo_device *bdev = bo->bdev;
1596
1597         drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1598         ttm_mem_io_free_vm(bo);
1599 }
1600
1601 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1602 {
1603         struct ttm_bo_device *bdev = bo->bdev;
1604         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1605
1606         ttm_mem_io_lock(man, false);
1607         ttm_bo_unmap_virtual_locked(bo);
1608         ttm_mem_io_unlock(man);
1609 }
1610
1611
1612 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1613
1614 int ttm_bo_wait(struct ttm_buffer_object *bo,
1615                 bool interruptible, bool no_wait)
1616 {
1617         long timeout = 15 * HZ;
1618
1619         if (no_wait) {
1620                 if (reservation_object_test_signaled_rcu(bo->resv, true))
1621                         return 0;
1622                 else
1623                         return -EBUSY;
1624         }
1625
1626         timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
1627                                                       interruptible, timeout);
1628         if (timeout < 0)
1629                 return timeout;
1630
1631         if (timeout == 0)
1632                 return -EBUSY;
1633
1634         reservation_object_add_excl_fence(bo->resv, NULL);
1635         return 0;
1636 }
1637 EXPORT_SYMBOL(ttm_bo_wait);
1638
1639 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1640 {
1641         int ret = 0;
1642
1643         /*
1644          * Using ttm_bo_reserve makes sure the lru lists are updated.
1645          */
1646
1647         ret = ttm_bo_reserve(bo, true, no_wait, NULL);
1648         if (unlikely(ret != 0))
1649                 return ret;
1650         ret = ttm_bo_wait(bo, true, no_wait);
1651         if (likely(ret == 0))
1652                 atomic_inc(&bo->cpu_writers);
1653         ttm_bo_unreserve(bo);
1654         return ret;
1655 }
1656 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1657
1658 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1659 {
1660         atomic_dec(&bo->cpu_writers);
1661 }
1662 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1663
1664 /**
1665  * A buffer object shrink method that tries to swap out the first
1666  * buffer object on the bo_global::swap_lru list.
1667  */
1668 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1669 {
1670         struct ttm_buffer_object *bo;
1671         int ret = -EBUSY;
1672         bool locked;
1673         unsigned i;
1674
1675         spin_lock(&glob->lru_lock);
1676         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1677                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1678                         if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) {
1679                                 ret = 0;
1680                                 break;
1681                         }
1682                 }
1683                 if (!ret)
1684                         break;
1685         }
1686
1687         if (ret) {
1688                 spin_unlock(&glob->lru_lock);
1689                 return ret;
1690         }
1691
1692         kref_get(&bo->list_kref);
1693
1694         if (!list_empty(&bo->ddestroy)) {
1695                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1696                 kref_put(&bo->list_kref, ttm_bo_release_list);
1697                 return ret;
1698         }
1699
1700         ttm_bo_del_from_lru(bo);
1701         spin_unlock(&glob->lru_lock);
1702
1703         /**
1704          * Move to system cached
1705          */
1706
1707         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1708             bo->ttm->caching_state != tt_cached) {
1709                 struct ttm_operation_ctx ctx = { false, false };
1710                 struct ttm_mem_reg evict_mem;
1711
1712                 evict_mem = bo->mem;
1713                 evict_mem.mm_node = NULL;
1714                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1715                 evict_mem.mem_type = TTM_PL_SYSTEM;
1716
1717                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1718                 if (unlikely(ret != 0))
1719                         goto out;
1720         }
1721
1722         /**
1723          * Make sure BO is idle.
1724          */
1725
1726         ret = ttm_bo_wait(bo, false, false);
1727         if (unlikely(ret != 0))
1728                 goto out;
1729
1730         ttm_bo_unmap_virtual(bo);
1731
1732         /**
1733          * Swap out. Buffer will be swapped in again as soon as
1734          * anyone tries to access a ttm page.
1735          */
1736
1737         if (bo->bdev->driver->swap_notify)
1738                 bo->bdev->driver->swap_notify(bo);
1739
1740         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1741 out:
1742
1743         /**
1744          *
1745          * Unreserve without putting on LRU to avoid swapping out an
1746          * already swapped buffer.
1747          */
1748         if (locked)
1749                 reservation_object_unlock(bo->resv);
1750         kref_put(&bo->list_kref, ttm_bo_release_list);
1751         return ret;
1752 }
1753 EXPORT_SYMBOL(ttm_bo_swapout);
1754
1755 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1756 {
1757         struct ttm_operation_ctx ctx = {
1758                 .interruptible = false,
1759                 .no_wait_gpu = false
1760         };
1761
1762         while (ttm_bo_swapout(bdev->glob, &ctx) == 0)
1763                 ;
1764 }
1765 EXPORT_SYMBOL(ttm_bo_swapout_all);
1766
1767 /**
1768  * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
1769  * unreserved
1770  *
1771  * @bo: Pointer to buffer
1772  */
1773 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
1774 {
1775         int ret;
1776
1777         /*
1778          * In the absense of a wait_unlocked API,
1779          * Use the bo::wu_mutex to avoid triggering livelocks due to
1780          * concurrent use of this function. Note that this use of
1781          * bo::wu_mutex can go away if we change locking order to
1782          * mmap_sem -> bo::reserve.
1783          */
1784         ret = mutex_lock_interruptible(&bo->wu_mutex);
1785         if (unlikely(ret != 0))
1786                 return -ERESTARTSYS;
1787         if (!ww_mutex_is_locked(&bo->resv->lock))
1788                 goto out_unlock;
1789         ret = reservation_object_lock_interruptible(bo->resv, NULL);
1790         if (ret == -EINTR)
1791                 ret = -ERESTARTSYS;
1792         if (unlikely(ret != 0))
1793                 goto out_unlock;
1794         reservation_object_unlock(bo->resv);
1795
1796 out_unlock:
1797         mutex_unlock(&bo->wu_mutex);
1798         return ret;
1799 }