drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_module.h>
  35 #include <drm/ttm/ttm_bo_driver.h>
  36 #include <drm/ttm/ttm_placement.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40 #include <linux/mm.h>
  41 #include <linux/file.h>
  42 #include <linux/module.h>
  43 #include <linux/atomic.h>
  44 #include <linux/reservation.h>
  45
  46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  47
  48 /**
  49  * ttm_global_mutex - protecting the global BO state
  50  */
  51 DEFINE_MUTEX(ttm_global_mutex);
  52 struct ttm_bo_global ttm_bo_glob = {
  53         .use_count = 0
  54 };
  55
  56 static struct attribute ttm_bo_count = {
  57         .name = "bo_count",
  58         .mode = S_IRUGO
  59 };
  60
  61 /* default destructor */
  62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  63 {
  64         kfree(bo);
  65 }
  66
  67 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
  68                                           uint32_t *mem_type)
  69 {
  70         int pos;
  71
  72         pos = ffs(place->flags & TTM_PL_MASK_MEM);
  73         if (unlikely(!pos))
  74                 return -EINVAL;
  75
  76         *mem_type = pos - 1;
  77         return 0;
  78 }
  79
  80 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
  81 {
  82         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  83         struct drm_printer p = drm_debug_printer(TTM_PFX);
  84
  85         pr_err("    has_type: %d\n", man->has_type);
  86         pr_err("    use_type: %d\n", man->use_type);
  87         pr_err("    flags: 0x%08X\n", man->flags);
  88         pr_err("    gpu_offset: 0x%08llX\n", man->gpu_offset);
  89         pr_err("    size: %llu\n", man->size);
  90         pr_err("    available_caching: 0x%08X\n", man->available_caching);
  91         pr_err("    default_caching: 0x%08X\n", man->default_caching);
  92         if (mem_type != TTM_PL_SYSTEM)
  93                 (*man->func->debug)(man, &p);
  94 }
  95
  96 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  97                                         struct ttm_placement *placement)
  98 {
  99         int i, ret, mem_type;
 100
 101         pr_err("No space for %p (%lu pages, %luK, %luM)\n",
 102                bo, bo->mem.num_pages, bo->mem.size >> 10,
 103                bo->mem.size >> 20);
 104         for (i = 0; i < placement->num_placement; i++) {
 105                 ret = ttm_mem_type_from_place(&placement->placement[i],
 106                                                 &mem_type);
 107                 if (ret)
 108                         return;
 109                 pr_err("  placement[%d]=0x%08X (%d)\n",
 110                        i, placement->placement[i].flags, mem_type);
 111                 ttm_mem_type_debug(bo->bdev, mem_type);
 112         }
 113 }
 114
 115 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 116                                   struct attribute *attr,
 117                                   char *buffer)
 118 {
 119         struct ttm_bo_global *glob =
 120                 container_of(kobj, struct ttm_bo_global, kobj);
 121
 122         return snprintf(buffer, PAGE_SIZE, "%d\n",
 123                                 atomic_read(&glob->bo_count));
 124 }
 125
 126 static struct attribute *ttm_bo_global_attrs[] = {
 127         &ttm_bo_count,
 128         NULL
 129 };
 130
 131 static const struct sysfs_ops ttm_bo_global_ops = {
 132         .show = &ttm_bo_global_show
 133 };
 134
 135 static struct kobj_type ttm_bo_glob_kobj_type  = {
 136         .release = &ttm_bo_global_kobj_release,
 137         .sysfs_ops = &ttm_bo_global_ops,
 138         .default_attrs = ttm_bo_global_attrs
 139 };
 140
 141
 142 static inline uint32_t ttm_bo_type_flags(unsigned type)
 143 {
 144         return 1 << (type);
 145 }
 146
 147 static void ttm_bo_release_list(struct kref *list_kref)
 148 {
 149         struct ttm_buffer_object *bo =
 150             container_of(list_kref, struct ttm_buffer_object, list_kref);
 151         struct ttm_bo_device *bdev = bo->bdev;
 152         size_t acc_size = bo->acc_size;
 153
 154         BUG_ON(kref_read(&bo->list_kref));
 155         BUG_ON(kref_read(&bo->kref));
 156         BUG_ON(atomic_read(&bo->cpu_writers));
 157         BUG_ON(bo->mem.mm_node != NULL);
 158         BUG_ON(!list_empty(&bo->lru));
 159         BUG_ON(!list_empty(&bo->ddestroy));
 160         ttm_tt_destroy(bo->ttm);
 161         atomic_dec(&bo->bdev->glob->bo_count);
 162         dma_fence_put(bo->moving);
 163         reservation_object_fini(&bo->ttm_resv);
 164         mutex_destroy(&bo->wu_mutex);
 165         bo->destroy(bo);
 166         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
 167 }
 168
 169 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 170 {
 171         struct ttm_bo_device *bdev = bo->bdev;
 172         struct ttm_mem_type_manager *man;
 173
 174         reservation_object_assert_held(bo->resv);
 175
 176         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 177                 BUG_ON(!list_empty(&bo->lru));
 178
 179                 man = &bdev->man[bo->mem.mem_type];
 180                 list_add_tail(&bo->lru, &man->lru[bo->priority]);
 181                 kref_get(&bo->list_kref);
 182
 183                 if (bo->ttm && !(bo->ttm->page_flags &
 184                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) {
 185                         list_add_tail(&bo->swap,
 186                                       &bdev->glob->swap_lru[bo->priority]);
 187                         kref_get(&bo->list_kref);
 188                 }
 189         }
 190 }
 191 EXPORT_SYMBOL(ttm_bo_add_to_lru);
 192
 193 static void ttm_bo_ref_bug(struct kref *list_kref)
 194 {
 195         BUG();
 196 }
 197
 198 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 199 {
 200         if (!list_empty(&bo->swap)) {
 201                 list_del_init(&bo->swap);
 202                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 203         }
 204         if (!list_empty(&bo->lru)) {
 205                 list_del_init(&bo->lru);
 206                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 207         }
 208
 209         /*
 210          * TODO: Add a driver hook to delete from
 211          * driver-specific LRU's here.
 212          */
 213 }
 214
 215 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
 216 {
 217         struct ttm_bo_global *glob = bo->bdev->glob;
 218
 219         spin_lock(&glob->lru_lock);
 220         ttm_bo_del_from_lru(bo);
 221         spin_unlock(&glob->lru_lock);
 222 }
 223 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 224
 225 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
 226                                      struct ttm_buffer_object *bo)
 227 {
 228         if (!pos->first)
 229                 pos->first = bo;
 230         pos->last = bo;
 231 }
 232
 233 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
 234                              struct ttm_lru_bulk_move *bulk)
 235 {
 236         reservation_object_assert_held(bo->resv);
 237
 238         ttm_bo_del_from_lru(bo);
 239         ttm_bo_add_to_lru(bo);
 240
 241         if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 242                 switch (bo->mem.mem_type) {
 243                 case TTM_PL_TT:
 244                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 245                         break;
 246
 247                 case TTM_PL_VRAM:
 248                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 249                         break;
 250                 }
 251                 if (bo->ttm && !(bo->ttm->page_flags &
 252                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 253                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 254         }
 255 }
 256 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 257
 258 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 259 {
 260         unsigned i;
 261
 262         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 263                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 264                 struct ttm_mem_type_manager *man;
 265
 266                 if (!pos->first)
 267                         continue;
 268
 269                 reservation_object_assert_held(pos->first->resv);
 270                 reservation_object_assert_held(pos->last->resv);
 271
 272                 man = &pos->first->bdev->man[TTM_PL_TT];
 273                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 274                                     &pos->last->lru);
 275         }
 276
 277         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 278                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 279                 struct ttm_mem_type_manager *man;
 280
 281                 if (!pos->first)
 282                         continue;
 283
 284                 reservation_object_assert_held(pos->first->resv);
 285                 reservation_object_assert_held(pos->last->resv);
 286
 287                 man = &pos->first->bdev->man[TTM_PL_VRAM];
 288                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 289                                     &pos->last->lru);
 290         }
 291
 292         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 293                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 294                 struct list_head *lru;
 295
 296                 if (!pos->first)
 297                         continue;
 298
 299                 reservation_object_assert_held(pos->first->resv);
 300                 reservation_object_assert_held(pos->last->resv);
 301
 302                 lru = &pos->first->bdev->glob->swap_lru[i];
 303                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 304         }
 305 }
 306 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 307
 308 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 309                                   struct ttm_mem_reg *mem, bool evict,
 310                                   struct ttm_operation_ctx *ctx)
 311 {
 312         struct ttm_bo_device *bdev = bo->bdev;
 313         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 314         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 315         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 316         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 317         int ret = 0;
 318
 319         if (old_is_pci || new_is_pci ||
 320             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 321                 ret = ttm_mem_io_lock(old_man, true);
 322                 if (unlikely(ret != 0))
 323                         goto out_err;
 324                 ttm_bo_unmap_virtual_locked(bo);
 325                 ttm_mem_io_unlock(old_man);
 326         }
 327
 328         /*
 329          * Create and bind a ttm if required.
 330          */
 331
 332         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 333                 if (bo->ttm == NULL) {
 334                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 335                         ret = ttm_tt_create(bo, zero);
 336                         if (ret)
 337                                 goto out_err;
 338                 }
 339
 340                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 341                 if (ret)
 342                         goto out_err;
 343
 344                 if (mem->mem_type != TTM_PL_SYSTEM) {
 345                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
 346                         if (ret)
 347                                 goto out_err;
 348                 }
 349
 350                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 351                         if (bdev->driver->move_notify)
 352                                 bdev->driver->move_notify(bo, evict, mem);
 353                         bo->mem = *mem;
 354                         mem->mm_node = NULL;
 355                         goto moved;
 356                 }
 357         }
 358
 359         if (bdev->driver->move_notify)
 360                 bdev->driver->move_notify(bo, evict, mem);
 361
 362         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 363             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 364                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 365         else if (bdev->driver->move)
 366                 ret = bdev->driver->move(bo, evict, ctx, mem);
 367         else
 368                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 369
 370         if (ret) {
 371                 if (bdev->driver->move_notify) {
 372                         swap(*mem, bo->mem);
 373                         bdev->driver->move_notify(bo, false, mem);
 374                         swap(*mem, bo->mem);
 375                 }
 376
 377                 goto out_err;
 378         }
 379
 380 moved:
 381         if (bo->evicted) {
 382                 if (bdev->driver->invalidate_caches) {
 383                         ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
 384                         if (ret)
 385                                 pr_err("Can not flush read caches\n");
 386                 }
 387                 bo->evicted = false;
 388         }
 389
 390         if (bo->mem.mm_node)
 391                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 392                     bdev->man[bo->mem.mem_type].gpu_offset;
 393         else
 394                 bo->offset = 0;
 395
 396         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 397         return 0;
 398
 399 out_err:
 400         new_man = &bdev->man[bo->mem.mem_type];
 401         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 402                 ttm_tt_destroy(bo->ttm);
 403                 bo->ttm = NULL;
 404         }
 405
 406         return ret;
 407 }
 408
 409 /**
 410  * Call bo::reserved.
 411  * Will release GPU memory type usage on destruction.
 412  * This is the place to put in driver specific hooks to release
 413  * driver private resources.
 414  * Will release the bo::reserved lock.
 415  */
 416
 417 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 418 {
 419         if (bo->bdev->driver->move_notify)
 420                 bo->bdev->driver->move_notify(bo, false, NULL);
 421
 422         ttm_tt_destroy(bo->ttm);
 423         bo->ttm = NULL;
 424         ttm_bo_mem_put(bo, &bo->mem);
 425 }
 426
 427 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 428 {
 429         int r;
 430
 431         if (bo->resv == &bo->ttm_resv)
 432                 return 0;
 433
 434         BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
 435
 436         r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
 437         if (r)
 438                 reservation_object_unlock(&bo->ttm_resv);
 439
 440         return r;
 441 }
 442
 443 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 444 {
 445         struct reservation_object_list *fobj;
 446         struct dma_fence *fence;
 447         int i;
 448
 449         fobj = reservation_object_get_list(&bo->ttm_resv);
 450         fence = reservation_object_get_excl(&bo->ttm_resv);
 451         if (fence && !fence->ops->signaled)
 452                 dma_fence_enable_sw_signaling(fence);
 453
 454         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 455                 fence = rcu_dereference_protected(fobj->shared[i],
 456                                         reservation_object_held(bo->resv));
 457
 458                 if (!fence->ops->signaled)
 459                         dma_fence_enable_sw_signaling(fence);
 460         }
 461 }
 462
 463 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 464 {
 465         struct ttm_bo_device *bdev = bo->bdev;
 466         struct ttm_bo_global *glob = bdev->glob;
 467         int ret;
 468
 469         ret = ttm_bo_individualize_resv(bo);
 470         if (ret) {
 471                 /* Last resort, if we fail to allocate memory for the
 472                  * fences block for the BO to become idle
 473                  */
 474                 reservation_object_wait_timeout_rcu(bo->resv, true, false,
 475                                                     30 * HZ);
 476                 spin_lock(&glob->lru_lock);
 477                 goto error;
 478         }
 479
 480         spin_lock(&glob->lru_lock);
 481         ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
 482         if (!ret) {
 483                 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
 484                         ttm_bo_del_from_lru(bo);
 485                         spin_unlock(&glob->lru_lock);
 486                         if (bo->resv != &bo->ttm_resv)
 487                                 reservation_object_unlock(&bo->ttm_resv);
 488
 489                         ttm_bo_cleanup_memtype_use(bo);
 490                         reservation_object_unlock(bo->resv);
 491                         return;
 492                 }
 493
 494                 ttm_bo_flush_all_fences(bo);
 495
 496                 /*
 497                  * Make NO_EVICT bos immediately available to
 498                  * shrinkers, now that they are queued for
 499                  * destruction.
 500                  */
 501                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 502                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 503                         ttm_bo_add_to_lru(bo);
 504                 }
 505
 506                 reservation_object_unlock(bo->resv);
 507         }
 508         if (bo->resv != &bo->ttm_resv)
 509                 reservation_object_unlock(&bo->ttm_resv);
 510
 511 error:
 512         kref_get(&bo->list_kref);
 513         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 514         spin_unlock(&glob->lru_lock);
 515
 516         schedule_delayed_work(&bdev->wq,
 517                               ((HZ / 100) < 1) ? 1 : HZ / 100);
 518 }
 519
 520 /**
 521  * function ttm_bo_cleanup_refs
 522  * If bo idle, remove from delayed- and lru lists, and unref.
 523  * If not idle, do nothing.
 524  *
 525  * Must be called with lru_lock and reservation held, this function
 526  * will drop the lru lock and optionally the reservation lock before returning.
 527  *
 528  * @interruptible         Any sleeps should occur interruptibly.
 529  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 530  * @unlock_resv           Unlock the reservation lock as well.
 531  */
 532
 533 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 534                                bool interruptible, bool no_wait_gpu,
 535                                bool unlock_resv)
 536 {
 537         struct ttm_bo_global *glob = bo->bdev->glob;
 538         struct reservation_object *resv;
 539         int ret;
 540
 541         if (unlikely(list_empty(&bo->ddestroy)))
 542                 resv = bo->resv;
 543         else
 544                 resv = &bo->ttm_resv;
 545
 546         if (reservation_object_test_signaled_rcu(resv, true))
 547                 ret = 0;
 548         else
 549                 ret = -EBUSY;
 550
 551         if (ret && !no_wait_gpu) {
 552                 long lret;
 553
 554                 if (unlock_resv)
 555                         reservation_object_unlock(bo->resv);
 556                 spin_unlock(&glob->lru_lock);
 557
 558                 lret = reservation_object_wait_timeout_rcu(resv, true,
 559                                                            interruptible,
 560                                                            30 * HZ);
 561
 562                 if (lret < 0)
 563                         return lret;
 564                 else if (lret == 0)
 565                         return -EBUSY;
 566
 567                 spin_lock(&glob->lru_lock);
 568                 if (unlock_resv && !reservation_object_trylock(bo->resv)) {
 569                         /*
 570                          * We raced, and lost, someone else holds the reservation now,
 571                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 572                          *
 573                          * Even if it's not the case, because we finished waiting any
 574                          * delayed destruction would succeed, so just return success
 575                          * here.
 576                          */
 577                         spin_unlock(&glob->lru_lock);
 578                         return 0;
 579                 }
 580                 ret = 0;
 581         }
 582
 583         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 584                 if (unlock_resv)
 585                         reservation_object_unlock(bo->resv);
 586                 spin_unlock(&glob->lru_lock);
 587                 return ret;
 588         }
 589
 590         ttm_bo_del_from_lru(bo);
 591         list_del_init(&bo->ddestroy);
 592         kref_put(&bo->list_kref, ttm_bo_ref_bug);
 593
 594         spin_unlock(&glob->lru_lock);
 595         ttm_bo_cleanup_memtype_use(bo);
 596
 597         if (unlock_resv)
 598                 reservation_object_unlock(bo->resv);
 599
 600         return 0;
 601 }
 602
 603 /**
 604  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 605  * encountered buffers.
 606  */
 607 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 608 {
 609         struct ttm_bo_global *glob = bdev->glob;
 610         struct list_head removed;
 611         bool empty;
 612
 613         INIT_LIST_HEAD(&removed);
 614
 615         spin_lock(&glob->lru_lock);
 616         while (!list_empty(&bdev->ddestroy)) {
 617                 struct ttm_buffer_object *bo;
 618
 619                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 620                                       ddestroy);
 621                 kref_get(&bo->list_kref);
 622                 list_move_tail(&bo->ddestroy, &removed);
 623
 624                 if (remove_all || bo->resv != &bo->ttm_resv) {
 625                         spin_unlock(&glob->lru_lock);
 626                         reservation_object_lock(bo->resv, NULL);
 627
 628                         spin_lock(&glob->lru_lock);
 629                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 630
 631                 } else if (reservation_object_trylock(bo->resv)) {
 632                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 633                 } else {
 634                         spin_unlock(&glob->lru_lock);
 635                 }
 636
 637                 kref_put(&bo->list_kref, ttm_bo_release_list);
 638                 spin_lock(&glob->lru_lock);
 639         }
 640         list_splice_tail(&removed, &bdev->ddestroy);
 641         empty = list_empty(&bdev->ddestroy);
 642         spin_unlock(&glob->lru_lock);
 643
 644         return empty;
 645 }
 646
 647 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 648 {
 649         struct ttm_bo_device *bdev =
 650             container_of(work, struct ttm_bo_device, wq.work);
 651
 652         if (!ttm_bo_delayed_delete(bdev, false))
 653                 schedule_delayed_work(&bdev->wq,
 654                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 655 }
 656
 657 static void ttm_bo_release(struct kref *kref)
 658 {
 659         struct ttm_buffer_object *bo =
 660             container_of(kref, struct ttm_buffer_object, kref);
 661         struct ttm_bo_device *bdev = bo->bdev;
 662         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 663
 664         drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
 665         ttm_mem_io_lock(man, false);
 666         ttm_mem_io_free_vm(bo);
 667         ttm_mem_io_unlock(man);
 668         ttm_bo_cleanup_refs_or_queue(bo);
 669         kref_put(&bo->list_kref, ttm_bo_release_list);
 670 }
 671
 672 void ttm_bo_put(struct ttm_buffer_object *bo)
 673 {
 674         kref_put(&bo->kref, ttm_bo_release);
 675 }
 676 EXPORT_SYMBOL(ttm_bo_put);
 677
 678 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 679 {
 680         struct ttm_buffer_object *bo = *p_bo;
 681
 682         *p_bo = NULL;
 683         ttm_bo_put(bo);
 684 }
 685 EXPORT_SYMBOL(ttm_bo_unref);
 686
 687 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 688 {
 689         return cancel_delayed_work_sync(&bdev->wq);
 690 }
 691 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 692
 693 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 694 {
 695         if (resched)
 696                 schedule_delayed_work(&bdev->wq,
 697                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 698 }
 699 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 700
 701 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 702                         struct ttm_operation_ctx *ctx)
 703 {
 704         struct ttm_bo_device *bdev = bo->bdev;
 705         struct ttm_mem_reg evict_mem;
 706         struct ttm_placement placement;
 707         int ret = 0;
 708
 709         reservation_object_assert_held(bo->resv);
 710
 711         placement.num_placement = 0;
 712         placement.num_busy_placement = 0;
 713         bdev->driver->evict_flags(bo, &placement);
 714
 715         if (!placement.num_placement && !placement.num_busy_placement) {
 716                 ret = ttm_bo_pipeline_gutting(bo);
 717                 if (ret)
 718                         return ret;
 719
 720                 return ttm_tt_create(bo, false);
 721         }
 722
 723         evict_mem = bo->mem;
 724         evict_mem.mm_node = NULL;
 725         evict_mem.bus.io_reserved_vm = false;
 726         evict_mem.bus.io_reserved_count = 0;
 727
 728         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 729         if (ret) {
 730                 if (ret != -ERESTARTSYS) {
 731                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 732                                bo);
 733                         ttm_bo_mem_space_debug(bo, &placement);
 734                 }
 735                 goto out;
 736         }
 737
 738         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 739         if (unlikely(ret)) {
 740                 if (ret != -ERESTARTSYS)
 741                         pr_err("Buffer eviction failed\n");
 742                 ttm_bo_mem_put(bo, &evict_mem);
 743                 goto out;
 744         }
 745         bo->evicted = true;
 746 out:
 747         return ret;
 748 }
 749
 750 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 751                               const struct ttm_place *place)
 752 {
 753         /* Don't evict this BO if it's outside of the
 754          * requested placement range
 755          */
 756         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 757             (place->lpfn && place->lpfn <= bo->mem.start))
 758                 return false;
 759
 760         return true;
 761 }
 762 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 763
 764 /**
 765  * Check the target bo is allowable to be evicted or swapout, including cases:
 766  *
 767  * a. if share same reservation object with ctx->resv, have assumption
 768  * reservation objects should already be locked, so not lock again and
 769  * return true directly when either the opreation allow_reserved_eviction
 770  * or the target bo already is in delayed free list;
 771  *
 772  * b. Otherwise, trylock it.
 773  */
 774 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 775                         struct ttm_operation_ctx *ctx, bool *locked)
 776 {
 777         bool ret = false;
 778
 779         *locked = false;
 780         if (bo->resv == ctx->resv) {
 781                 reservation_object_assert_held(bo->resv);
 782                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
 783                     || !list_empty(&bo->ddestroy))
 784                         ret = true;
 785         } else {
 786                 *locked = reservation_object_trylock(bo->resv);
 787                 ret = *locked;
 788         }
 789
 790         return ret;
 791 }
 792
 793 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 794                                uint32_t mem_type,
 795                                const struct ttm_place *place,
 796                                struct ttm_operation_ctx *ctx)
 797 {
 798         struct ttm_bo_global *glob = bdev->glob;
 799         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 800         struct ttm_buffer_object *bo = NULL;
 801         bool locked = false;
 802         unsigned i;
 803         int ret;
 804
 805         spin_lock(&glob->lru_lock);
 806         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 807                 list_for_each_entry(bo, &man->lru[i], lru) {
 808                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked))
 809                                 continue;
 810
 811                         if (place && !bdev->driver->eviction_valuable(bo,
 812                                                                       place)) {
 813                                 if (locked)
 814                                         reservation_object_unlock(bo->resv);
 815                                 continue;
 816                         }
 817                         break;
 818                 }
 819
 820                 /* If the inner loop terminated early, we have our candidate */
 821                 if (&bo->lru != &man->lru[i])
 822                         break;
 823
 824                 bo = NULL;
 825         }
 826
 827         if (!bo) {
 828                 spin_unlock(&glob->lru_lock);
 829                 return -EBUSY;
 830         }
 831
 832         kref_get(&bo->list_kref);
 833
 834         if (!list_empty(&bo->ddestroy)) {
 835                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 836                                           ctx->no_wait_gpu, locked);
 837                 kref_put(&bo->list_kref, ttm_bo_release_list);
 838                 return ret;
 839         }
 840
 841         ttm_bo_del_from_lru(bo);
 842         spin_unlock(&glob->lru_lock);
 843
 844         ret = ttm_bo_evict(bo, ctx);
 845         if (locked) {
 846                 ttm_bo_unreserve(bo);
 847         } else {
 848                 spin_lock(&glob->lru_lock);
 849                 ttm_bo_add_to_lru(bo);
 850                 spin_unlock(&glob->lru_lock);
 851         }
 852
 853         kref_put(&bo->list_kref, ttm_bo_release_list);
 854         return ret;
 855 }
 856
 857 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 858 {
 859         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 860
 861         if (mem->mm_node)
 862                 (*man->func->put_node)(man, mem);
 863 }
 864 EXPORT_SYMBOL(ttm_bo_mem_put);
 865
 866 /**
 867  * Add the last move fence to the BO and reserve a new shared slot.
 868  */
 869 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 870                                  struct ttm_mem_type_manager *man,
 871                                  struct ttm_mem_reg *mem)
 872 {
 873         struct dma_fence *fence;
 874         int ret;
 875
 876         spin_lock(&man->move_lock);
 877         fence = dma_fence_get(man->move);
 878         spin_unlock(&man->move_lock);
 879
 880         if (fence) {
 881                 reservation_object_add_shared_fence(bo->resv, fence);
 882
 883                 ret = reservation_object_reserve_shared(bo->resv, 1);
 884                 if (unlikely(ret))
 885                         return ret;
 886
 887                 dma_fence_put(bo->moving);
 888                 bo->moving = fence;
 889         }
 890
 891         return 0;
 892 }
 893
 894 /**
 895  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 896  * space, or we've evicted everything and there isn't enough space.
 897  */
 898 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 899                                         uint32_t mem_type,
 900                                         const struct ttm_place *place,
 901                                         struct ttm_mem_reg *mem,
 902                                         struct ttm_operation_ctx *ctx)
 903 {
 904         struct ttm_bo_device *bdev = bo->bdev;
 905         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 906         int ret;
 907
 908         do {
 909                 ret = (*man->func->get_node)(man, bo, place, mem);
 910                 if (unlikely(ret != 0))
 911                         return ret;
 912                 if (mem->mm_node)
 913                         break;
 914                 ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);
 915                 if (unlikely(ret != 0))
 916                         return ret;
 917         } while (1);
 918         mem->mem_type = mem_type;
 919         return ttm_bo_add_move_fence(bo, man, mem);
 920 }
 921
 922 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 923                                       uint32_t cur_placement,
 924                                       uint32_t proposed_placement)
 925 {
 926         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 927         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 928
 929         /**
 930          * Keep current caching if possible.
 931          */
 932
 933         if ((cur_placement & caching) != 0)
 934                 result |= (cur_placement & caching);
 935         else if ((man->default_caching & caching) != 0)
 936                 result |= man->default_caching;
 937         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 938                 result |= TTM_PL_FLAG_CACHED;
 939         else if ((TTM_PL_FLAG_WC & caching) != 0)
 940                 result |= TTM_PL_FLAG_WC;
 941         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 942                 result |= TTM_PL_FLAG_UNCACHED;
 943
 944         return result;
 945 }
 946
 947 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 948                                  uint32_t mem_type,
 949                                  const struct ttm_place *place,
 950                                  uint32_t *masked_placement)
 951 {
 952         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 953
 954         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 955                 return false;
 956
 957         if ((place->flags & man->available_caching) == 0)
 958                 return false;
 959
 960         cur_flags |= (place->flags & man->available_caching);
 961
 962         *masked_placement = cur_flags;
 963         return true;
 964 }
 965
 966 /**
 967  * Creates space for memory region @mem according to its type.
 968  *
 969  * This function first searches for free space in compatible memory types in
 970  * the priority order defined by the driver.  If free space isn't found, then
 971  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 972  * space.
 973  */
 974 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 975                         struct ttm_placement *placement,
 976                         struct ttm_mem_reg *mem,
 977                         struct ttm_operation_ctx *ctx)
 978 {
 979         struct ttm_bo_device *bdev = bo->bdev;
 980         struct ttm_mem_type_manager *man;
 981         uint32_t mem_type = TTM_PL_SYSTEM;
 982         uint32_t cur_flags = 0;
 983         bool type_found = false;
 984         bool type_ok = false;
 985         bool has_erestartsys = false;
 986         int i, ret;
 987
 988         ret = reservation_object_reserve_shared(bo->resv, 1);
 989         if (unlikely(ret))
 990                 return ret;
 991
 992         mem->mm_node = NULL;
 993         for (i = 0; i < placement->num_placement; ++i) {
 994                 const struct ttm_place *place = &placement->placement[i];
 995
 996                 ret = ttm_mem_type_from_place(place, &mem_type);
 997                 if (ret)
 998                         return ret;
 999                 man = &bdev->man[mem_type];
1000                 if (!man->has_type || !man->use_type)
1001                         continue;
1002
1003                 type_ok = ttm_bo_mt_compatible(man, mem_type, place,
1004                                                 &cur_flags);
1005
1006                 if (!type_ok)
1007                         continue;
1008
1009                 type_found = true;
1010                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1011                                                   cur_flags);
1012                 /*
1013                  * Use the access and other non-mapping-related flag bits from
1014                  * the memory placement flags to the current flags
1015                  */
1016                 ttm_flag_masked(&cur_flags, place->flags,
1017                                 ~TTM_PL_MASK_MEMTYPE);
1018
1019                 if (mem_type == TTM_PL_SYSTEM)
1020                         break;
1021
1022                 ret = (*man->func->get_node)(man, bo, place, mem);
1023                 if (unlikely(ret))
1024                         return ret;
1025
1026                 if (mem->mm_node) {
1027                         ret = ttm_bo_add_move_fence(bo, man, mem);
1028                         if (unlikely(ret)) {
1029                                 (*man->func->put_node)(man, mem);
1030                                 return ret;
1031                         }
1032                         break;
1033                 }
1034         }
1035
1036         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
1037                 mem->mem_type = mem_type;
1038                 mem->placement = cur_flags;
1039                 return 0;
1040         }
1041
1042         for (i = 0; i < placement->num_busy_placement; ++i) {
1043                 const struct ttm_place *place = &placement->busy_placement[i];
1044
1045                 ret = ttm_mem_type_from_place(place, &mem_type);
1046                 if (ret)
1047                         return ret;
1048                 man = &bdev->man[mem_type];
1049                 if (!man->has_type || !man->use_type)
1050                         continue;
1051                 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
1052                         continue;
1053
1054                 type_found = true;
1055                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1056                                                   cur_flags);
1057                 /*
1058                  * Use the access and other non-mapping-related flag bits from
1059                  * the memory placement flags to the current flags
1060                  */
1061                 ttm_flag_masked(&cur_flags, place->flags,
1062                                 ~TTM_PL_MASK_MEMTYPE);
1063
1064                 if (mem_type == TTM_PL_SYSTEM) {
1065                         mem->mem_type = mem_type;
1066                         mem->placement = cur_flags;
1067                         mem->mm_node = NULL;
1068                         return 0;
1069                 }
1070
1071                 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx);
1072                 if (ret == 0 && mem->mm_node) {
1073                         mem->placement = cur_flags;
1074                         return 0;
1075                 }
1076                 if (ret == -ERESTARTSYS)
1077                         has_erestartsys = true;
1078         }
1079
1080         if (!type_found) {
1081                 pr_err(TTM_PFX "No compatible memory type found\n");
1082                 return -EINVAL;
1083         }
1084
1085         return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
1086 }
1087 EXPORT_SYMBOL(ttm_bo_mem_space);
1088
1089 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1090                               struct ttm_placement *placement,
1091                               struct ttm_operation_ctx *ctx)
1092 {
1093         int ret = 0;
1094         struct ttm_mem_reg mem;
1095
1096         reservation_object_assert_held(bo->resv);
1097
1098         mem.num_pages = bo->num_pages;
1099         mem.size = mem.num_pages << PAGE_SHIFT;
1100         mem.page_alignment = bo->mem.page_alignment;
1101         mem.bus.io_reserved_vm = false;
1102         mem.bus.io_reserved_count = 0;
1103         /*
1104          * Determine where to move the buffer.
1105          */
1106         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1107         if (ret)
1108                 goto out_unlock;
1109         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1110 out_unlock:
1111         if (ret && mem.mm_node)
1112                 ttm_bo_mem_put(bo, &mem);
1113         return ret;
1114 }
1115
1116 static bool ttm_bo_places_compat(const struct ttm_place *places,
1117                                  unsigned num_placement,
1118                                  struct ttm_mem_reg *mem,
1119                                  uint32_t *new_flags)
1120 {
1121         unsigned i;
1122
1123         for (i = 0; i < num_placement; i++) {
1124                 const struct ttm_place *heap = &places[i];
1125
1126                 if (mem->mm_node && (mem->start < heap->fpfn ||
1127                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1128                         continue;
1129
1130                 *new_flags = heap->flags;
1131                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1132                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1133                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1134                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1135                         return true;
1136         }
1137         return false;
1138 }
1139
1140 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1141                        struct ttm_mem_reg *mem,
1142                        uint32_t *new_flags)
1143 {
1144         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1145                                  mem, new_flags))
1146                 return true;
1147
1148         if ((placement->busy_placement != placement->placement ||
1149              placement->num_busy_placement > placement->num_placement) &&
1150             ttm_bo_places_compat(placement->busy_placement,
1151                                  placement->num_busy_placement,
1152                                  mem, new_flags))
1153                 return true;
1154
1155         return false;
1156 }
1157 EXPORT_SYMBOL(ttm_bo_mem_compat);
1158
1159 int ttm_bo_validate(struct ttm_buffer_object *bo,
1160                     struct ttm_placement *placement,
1161                     struct ttm_operation_ctx *ctx)
1162 {
1163         int ret;
1164         uint32_t new_flags;
1165
1166         reservation_object_assert_held(bo->resv);
1167         /*
1168          * Check whether we need to move buffer.
1169          */
1170         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1171                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1172                 if (ret)
1173                         return ret;
1174         } else {
1175                 /*
1176                  * Use the access and other non-mapping-related flag bits from
1177                  * the compatible memory placement flags to the active flags
1178                  */
1179                 ttm_flag_masked(&bo->mem.placement, new_flags,
1180                                 ~TTM_PL_MASK_MEMTYPE);
1181         }
1182         /*
1183          * We might need to add a TTM.
1184          */
1185         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1186                 ret = ttm_tt_create(bo, true);
1187                 if (ret)
1188                         return ret;
1189         }
1190         return 0;
1191 }
1192 EXPORT_SYMBOL(ttm_bo_validate);
1193
1194 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1195                          struct ttm_buffer_object *bo,
1196                          unsigned long size,
1197                          enum ttm_bo_type type,
1198                          struct ttm_placement *placement,
1199                          uint32_t page_alignment,
1200                          struct ttm_operation_ctx *ctx,
1201                          size_t acc_size,
1202                          struct sg_table *sg,
1203                          struct reservation_object *resv,
1204                          void (*destroy) (struct ttm_buffer_object *))
1205 {
1206         int ret = 0;
1207         unsigned long num_pages;
1208         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1209         bool locked;
1210
1211         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1212         if (ret) {
1213                 pr_err("Out of kernel memory\n");
1214                 if (destroy)
1215                         (*destroy)(bo);
1216                 else
1217                         kfree(bo);
1218                 return -ENOMEM;
1219         }
1220
1221         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1222         if (num_pages == 0) {
1223                 pr_err("Illegal buffer object size\n");
1224                 if (destroy)
1225                         (*destroy)(bo);
1226                 else
1227                         kfree(bo);
1228                 ttm_mem_global_free(mem_glob, acc_size);
1229                 return -EINVAL;
1230         }
1231         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1232
1233         kref_init(&bo->kref);
1234         kref_init(&bo->list_kref);
1235         atomic_set(&bo->cpu_writers, 0);
1236         INIT_LIST_HEAD(&bo->lru);
1237         INIT_LIST_HEAD(&bo->ddestroy);
1238         INIT_LIST_HEAD(&bo->swap);
1239         INIT_LIST_HEAD(&bo->io_reserve_lru);
1240         mutex_init(&bo->wu_mutex);
1241         bo->bdev = bdev;
1242         bo->type = type;
1243         bo->num_pages = num_pages;
1244         bo->mem.size = num_pages << PAGE_SHIFT;
1245         bo->mem.mem_type = TTM_PL_SYSTEM;
1246         bo->mem.num_pages = bo->num_pages;
1247         bo->mem.mm_node = NULL;
1248         bo->mem.page_alignment = page_alignment;
1249         bo->mem.bus.io_reserved_vm = false;
1250         bo->mem.bus.io_reserved_count = 0;
1251         bo->moving = NULL;
1252         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1253         bo->acc_size = acc_size;
1254         bo->sg = sg;
1255         if (resv) {
1256                 bo->resv = resv;
1257                 reservation_object_assert_held(bo->resv);
1258         } else {
1259                 bo->resv = &bo->ttm_resv;
1260         }
1261         reservation_object_init(&bo->ttm_resv);
1262         atomic_inc(&bo->bdev->glob->bo_count);
1263         drm_vma_node_reset(&bo->vma_node);
1264
1265         /*
1266          * For ttm_bo_type_device buffers, allocate
1267          * address space from the device.
1268          */
1269         if (bo->type == ttm_bo_type_device ||
1270             bo->type == ttm_bo_type_sg)
1271                 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
1272                                          bo->mem.num_pages);
1273
1274         /* passed reservation objects should already be locked,
1275          * since otherwise lockdep will be angered in radeon.
1276          */
1277         if (!resv) {
1278                 locked = reservation_object_trylock(bo->resv);
1279                 WARN_ON(!locked);
1280         }
1281
1282         if (likely(!ret))
1283                 ret = ttm_bo_validate(bo, placement, ctx);
1284
1285         if (unlikely(ret)) {
1286                 if (!resv)
1287                         ttm_bo_unreserve(bo);
1288
1289                 ttm_bo_put(bo);
1290                 return ret;
1291         }
1292
1293         if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
1294                 spin_lock(&bdev->glob->lru_lock);
1295                 ttm_bo_add_to_lru(bo);
1296                 spin_unlock(&bdev->glob->lru_lock);
1297         }
1298
1299         return ret;
1300 }
1301 EXPORT_SYMBOL(ttm_bo_init_reserved);
1302
1303 int ttm_bo_init(struct ttm_bo_device *bdev,
1304                 struct ttm_buffer_object *bo,
1305                 unsigned long size,
1306                 enum ttm_bo_type type,
1307                 struct ttm_placement *placement,
1308                 uint32_t page_alignment,
1309                 bool interruptible,
1310                 size_t acc_size,
1311                 struct sg_table *sg,
1312                 struct reservation_object *resv,
1313                 void (*destroy) (struct ttm_buffer_object *))
1314 {
1315         struct ttm_operation_ctx ctx = { interruptible, false };
1316         int ret;
1317
1318         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1319                                    page_alignment, &ctx, acc_size,
1320                                    sg, resv, destroy);
1321         if (ret)
1322                 return ret;
1323
1324         if (!resv)
1325                 ttm_bo_unreserve(bo);
1326
1327         return 0;
1328 }
1329 EXPORT_SYMBOL(ttm_bo_init);
1330
1331 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1332                        unsigned long bo_size,
1333                        unsigned struct_size)
1334 {
1335         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1336         size_t size = 0;
1337
1338         size += ttm_round_pot(struct_size);
1339         size += ttm_round_pot(npages * sizeof(void *));
1340         size += ttm_round_pot(sizeof(struct ttm_tt));
1341         return size;
1342 }
1343 EXPORT_SYMBOL(ttm_bo_acc_size);
1344
1345 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1346                            unsigned long bo_size,
1347                            unsigned struct_size)
1348 {
1349         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1350         size_t size = 0;
1351
1352         size += ttm_round_pot(struct_size);
1353         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1354         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1355         return size;
1356 }
1357 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1358
1359 int ttm_bo_create(struct ttm_bo_device *bdev,
1360                         unsigned long size,
1361                         enum ttm_bo_type type,
1362                         struct ttm_placement *placement,
1363                         uint32_t page_alignment,
1364                         bool interruptible,
1365                         struct ttm_buffer_object **p_bo)
1366 {
1367         struct ttm_buffer_object *bo;
1368         size_t acc_size;
1369         int ret;
1370
1371         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1372         if (unlikely(bo == NULL))
1373                 return -ENOMEM;
1374
1375         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1376         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1377                           interruptible, acc_size,
1378                           NULL, NULL, NULL);
1379         if (likely(ret == 0))
1380                 *p_bo = bo;
1381
1382         return ret;
1383 }
1384 EXPORT_SYMBOL(ttm_bo_create);
1385
1386 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1387                                    unsigned mem_type)
1388 {
1389         struct ttm_operation_ctx ctx = {
1390                 .interruptible = false,
1391                 .no_wait_gpu = false,
1392                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1393         };
1394         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1395         struct ttm_bo_global *glob = bdev->glob;
1396         struct dma_fence *fence;
1397         int ret;
1398         unsigned i;
1399
1400         /*
1401          * Can't use standard list traversal since we're unlocking.
1402          */
1403
1404         spin_lock(&glob->lru_lock);
1405         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1406                 while (!list_empty(&man->lru[i])) {
1407                         spin_unlock(&glob->lru_lock);
1408                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
1409                         if (ret)
1410                                 return ret;
1411                         spin_lock(&glob->lru_lock);
1412                 }
1413         }
1414         spin_unlock(&glob->lru_lock);
1415
1416         spin_lock(&man->move_lock);
1417         fence = dma_fence_get(man->move);
1418         spin_unlock(&man->move_lock);
1419
1420         if (fence) {
1421                 ret = dma_fence_wait(fence, false);
1422                 dma_fence_put(fence);
1423                 if (ret)
1424                         return ret;
1425         }
1426
1427         return 0;
1428 }
1429
1430 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1431 {
1432         struct ttm_mem_type_manager *man;
1433         int ret = -EINVAL;
1434
1435         if (mem_type >= TTM_NUM_MEM_TYPES) {
1436                 pr_err("Illegal memory type %d\n", mem_type);
1437                 return ret;
1438         }
1439         man = &bdev->man[mem_type];
1440
1441         if (!man->has_type) {
1442                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1443                        mem_type);
1444                 return ret;
1445         }
1446
1447         man->use_type = false;
1448         man->has_type = false;
1449
1450         ret = 0;
1451         if (mem_type > 0) {
1452                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1453                 if (ret) {
1454                         pr_err("Cleanup eviction failed\n");
1455                         return ret;
1456                 }
1457
1458                 ret = (*man->func->takedown)(man);
1459         }
1460
1461         dma_fence_put(man->move);
1462         man->move = NULL;
1463
1464         return ret;
1465 }
1466 EXPORT_SYMBOL(ttm_bo_clean_mm);
1467
1468 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1469 {
1470         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1471
1472         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1473                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1474                 return -EINVAL;
1475         }
1476
1477         if (!man->has_type) {
1478                 pr_err("Memory type %u has not been initialized\n", mem_type);
1479                 return 0;
1480         }
1481
1482         return ttm_bo_force_list_clean(bdev, mem_type);
1483 }
1484 EXPORT_SYMBOL(ttm_bo_evict_mm);
1485
1486 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1487                         unsigned long p_size)
1488 {
1489         int ret;
1490         struct ttm_mem_type_manager *man;
1491         unsigned i;
1492
1493         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1494         man = &bdev->man[type];
1495         BUG_ON(man->has_type);
1496         man->io_reserve_fastpath = true;
1497         man->use_io_reserve_lru = false;
1498         mutex_init(&man->io_reserve_mutex);
1499         spin_lock_init(&man->move_lock);
1500         INIT_LIST_HEAD(&man->io_reserve_lru);
1501
1502         ret = bdev->driver->init_mem_type(bdev, type, man);
1503         if (ret)
1504                 return ret;
1505         man->bdev = bdev;
1506
1507         if (type != TTM_PL_SYSTEM) {
1508                 ret = (*man->func->init)(man, p_size);
1509                 if (ret)
1510                         return ret;
1511         }
1512         man->has_type = true;
1513         man->use_type = true;
1514         man->size = p_size;
1515
1516         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1517                 INIT_LIST_HEAD(&man->lru[i]);
1518         man->move = NULL;
1519
1520         return 0;
1521 }
1522 EXPORT_SYMBOL(ttm_bo_init_mm);
1523
1524 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1525 {
1526         struct ttm_bo_global *glob =
1527                 container_of(kobj, struct ttm_bo_global, kobj);
1528
1529         __free_page(glob->dummy_read_page);
1530 }
1531
1532 static void ttm_bo_global_release(void)
1533 {
1534         struct ttm_bo_global *glob = &ttm_bo_glob;
1535
1536         mutex_lock(&ttm_global_mutex);
1537         if (--glob->use_count > 0)
1538                 goto out;
1539
1540         kobject_del(&glob->kobj);
1541         kobject_put(&glob->kobj);
1542         ttm_mem_global_release(&ttm_mem_glob);
1543 out:
1544         mutex_unlock(&ttm_global_mutex);
1545 }
1546
1547 static int ttm_bo_global_init(void)
1548 {
1549         struct ttm_bo_global *glob = &ttm_bo_glob;
1550         int ret = 0;
1551         unsigned i;
1552
1553         mutex_lock(&ttm_global_mutex);
1554         if (++glob->use_count > 1)
1555                 goto out;
1556
1557         ret = ttm_mem_global_init(&ttm_mem_glob);
1558         if (ret)
1559                 goto out;
1560
1561         spin_lock_init(&glob->lru_lock);
1562         glob->mem_glob = &ttm_mem_glob;
1563         glob->mem_glob->bo_glob = glob;
1564         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1565
1566         if (unlikely(glob->dummy_read_page == NULL)) {
1567                 ret = -ENOMEM;
1568                 goto out;
1569         }
1570
1571         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1572                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1573         INIT_LIST_HEAD(&glob->device_list);
1574         atomic_set(&glob->bo_count, 0);
1575
1576         ret = kobject_init_and_add(
1577                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1578         if (unlikely(ret != 0))
1579                 kobject_put(&glob->kobj);
1580 out:
1581         mutex_unlock(&ttm_global_mutex);
1582         return ret;
1583 }
1584
1585 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1586 {
1587         int ret = 0;
1588         unsigned i = TTM_NUM_MEM_TYPES;
1589         struct ttm_mem_type_manager *man;
1590         struct ttm_bo_global *glob = bdev->glob;
1591
1592         while (i--) {
1593                 man = &bdev->man[i];
1594                 if (man->has_type) {
1595                         man->use_type = false;
1596                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1597                                 ret = -EBUSY;
1598                                 pr_err("DRM memory manager type %d is not clean\n",
1599                                        i);
1600                         }
1601                         man->has_type = false;
1602                 }
1603         }
1604
1605         mutex_lock(&ttm_global_mutex);
1606         list_del(&bdev->device_list);
1607         mutex_unlock(&ttm_global_mutex);
1608
1609         cancel_delayed_work_sync(&bdev->wq);
1610
1611         if (ttm_bo_delayed_delete(bdev, true))
1612                 pr_debug("Delayed destroy list was clean\n");
1613
1614         spin_lock(&glob->lru_lock);
1615         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1616                 if (list_empty(&bdev->man[0].lru[0]))
1617                         pr_debug("Swap list %d was clean\n", i);
1618         spin_unlock(&glob->lru_lock);
1619
1620         drm_vma_offset_manager_destroy(&bdev->vma_manager);
1621
1622         if (!ret)
1623                 ttm_bo_global_release();
1624
1625         return ret;
1626 }
1627 EXPORT_SYMBOL(ttm_bo_device_release);
1628
1629 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1630                        struct ttm_bo_driver *driver,
1631                        struct address_space *mapping,
1632                        uint64_t file_page_offset,
1633                        bool need_dma32)
1634 {
1635         struct ttm_bo_global *glob = &ttm_bo_glob;
1636         int ret;
1637
1638         ret = ttm_bo_global_init();
1639         if (ret)
1640                 return ret;
1641
1642         bdev->driver = driver;
1643
1644         memset(bdev->man, 0, sizeof(bdev->man));
1645
1646         /*
1647          * Initialize the system memory buffer type.
1648          * Other types need to be driver / IOCTL initialized.
1649          */
1650         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1651         if (unlikely(ret != 0))
1652                 goto out_no_sys;
1653
1654         drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
1655                                     0x10000000);
1656         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1657         INIT_LIST_HEAD(&bdev->ddestroy);
1658         bdev->dev_mapping = mapping;
1659         bdev->glob = glob;
1660         bdev->need_dma32 = need_dma32;
1661         mutex_lock(&ttm_global_mutex);
1662         list_add_tail(&bdev->device_list, &glob->device_list);
1663         mutex_unlock(&ttm_global_mutex);
1664
1665         return 0;
1666 out_no_sys:
1667         ttm_bo_global_release();
1668         return ret;
1669 }
1670 EXPORT_SYMBOL(ttm_bo_device_init);
1671
1672 /*
1673  * buffer object vm functions.
1674  */
1675
1676 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1677 {
1678         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1679
1680         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1681                 if (mem->mem_type == TTM_PL_SYSTEM)
1682                         return false;
1683
1684                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1685                         return false;
1686
1687                 if (mem->placement & TTM_PL_FLAG_CACHED)
1688                         return false;
1689         }
1690         return true;
1691 }
1692
1693 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1694 {
1695         struct ttm_bo_device *bdev = bo->bdev;
1696
1697         drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1698         ttm_mem_io_free_vm(bo);
1699 }
1700
1701 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1702 {
1703         struct ttm_bo_device *bdev = bo->bdev;
1704         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1705
1706         ttm_mem_io_lock(man, false);
1707         ttm_bo_unmap_virtual_locked(bo);
1708         ttm_mem_io_unlock(man);
1709 }
1710
1711
1712 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1713
1714 int ttm_bo_wait(struct ttm_buffer_object *bo,
1715                 bool interruptible, bool no_wait)
1716 {
1717         long timeout = 15 * HZ;
1718
1719         if (no_wait) {
1720                 if (reservation_object_test_signaled_rcu(bo->resv, true))
1721                         return 0;
1722                 else
1723                         return -EBUSY;
1724         }
1725
1726         timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
1727                                                       interruptible, timeout);
1728         if (timeout < 0)
1729                 return timeout;
1730
1731         if (timeout == 0)
1732                 return -EBUSY;
1733
1734         reservation_object_add_excl_fence(bo->resv, NULL);
1735         return 0;
1736 }
1737 EXPORT_SYMBOL(ttm_bo_wait);
1738
1739 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1740 {
1741         int ret = 0;
1742
1743         /*
1744          * Using ttm_bo_reserve makes sure the lru lists are updated.
1745          */
1746
1747         ret = ttm_bo_reserve(bo, true, no_wait, NULL);
1748         if (unlikely(ret != 0))
1749                 return ret;
1750         ret = ttm_bo_wait(bo, true, no_wait);
1751         if (likely(ret == 0))
1752                 atomic_inc(&bo->cpu_writers);
1753         ttm_bo_unreserve(bo);
1754         return ret;
1755 }
1756 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1757
1758 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1759 {
1760         atomic_dec(&bo->cpu_writers);
1761 }
1762 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1763
1764 /**
1765  * A buffer object shrink method that tries to swap out the first
1766  * buffer object on the bo_global::swap_lru list.
1767  */
1768 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1769 {
1770         struct ttm_buffer_object *bo;
1771         int ret = -EBUSY;
1772         bool locked;
1773         unsigned i;
1774
1775         spin_lock(&glob->lru_lock);
1776         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1777                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1778                         if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) {
1779                                 ret = 0;
1780                                 break;
1781                         }
1782                 }
1783                 if (!ret)
1784                         break;
1785         }
1786
1787         if (ret) {
1788                 spin_unlock(&glob->lru_lock);
1789                 return ret;
1790         }
1791
1792         kref_get(&bo->list_kref);
1793
1794         if (!list_empty(&bo->ddestroy)) {
1795                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1796                 kref_put(&bo->list_kref, ttm_bo_release_list);
1797                 return ret;
1798         }
1799
1800         ttm_bo_del_from_lru(bo);
1801         spin_unlock(&glob->lru_lock);
1802
1803         /**
1804          * Move to system cached
1805          */
1806
1807         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1808             bo->ttm->caching_state != tt_cached) {
1809                 struct ttm_operation_ctx ctx = { false, false };
1810                 struct ttm_mem_reg evict_mem;
1811
1812                 evict_mem = bo->mem;
1813                 evict_mem.mm_node = NULL;
1814                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1815                 evict_mem.mem_type = TTM_PL_SYSTEM;
1816
1817                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1818                 if (unlikely(ret != 0))
1819                         goto out;
1820         }
1821
1822         /**
1823          * Make sure BO is idle.
1824          */
1825
1826         ret = ttm_bo_wait(bo, false, false);
1827         if (unlikely(ret != 0))
1828                 goto out;
1829
1830         ttm_bo_unmap_virtual(bo);
1831
1832         /**
1833          * Swap out. Buffer will be swapped in again as soon as
1834          * anyone tries to access a ttm page.
1835          */
1836
1837         if (bo->bdev->driver->swap_notify)
1838                 bo->bdev->driver->swap_notify(bo);
1839
1840         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1841 out:
1842
1843         /**
1844          *
1845          * Unreserve without putting on LRU to avoid swapping out an
1846          * already swapped buffer.
1847          */
1848         if (locked)
1849                 reservation_object_unlock(bo->resv);
1850         kref_put(&bo->list_kref, ttm_bo_release_list);
1851         return ret;
1852 }
1853 EXPORT_SYMBOL(ttm_bo_swapout);
1854
1855 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1856 {
1857         struct ttm_operation_ctx ctx = {
1858                 .interruptible = false,
1859                 .no_wait_gpu = false
1860         };
1861
1862         while (ttm_bo_swapout(bdev->glob, &ctx) == 0)
1863                 ;
1864 }
1865 EXPORT_SYMBOL(ttm_bo_swapout_all);
1866
1867 /**
1868  * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
1869  * unreserved
1870  *
1871  * @bo: Pointer to buffer
1872  */
1873 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
1874 {
1875         int ret;
1876
1877         /*
1878          * In the absense of a wait_unlocked API,
1879          * Use the bo::wu_mutex to avoid triggering livelocks due to
1880          * concurrent use of this function. Note that this use of
1881          * bo::wu_mutex can go away if we change locking order to
1882          * mmap_sem -> bo::reserve.
1883          */
1884         ret = mutex_lock_interruptible(&bo->wu_mutex);
1885         if (unlikely(ret != 0))
1886                 return -ERESTARTSYS;
1887         if (!ww_mutex_is_locked(&bo->resv->lock))
1888                 goto out_unlock;
1889         ret = reservation_object_lock_interruptible(bo->resv, NULL);
1890         if (ret == -EINTR)
1891                 ret = -ERESTARTSYS;
1892         if (unlikely(ret != 0))
1893                 goto out_unlock;
1894         reservation_object_unlock(bo->resv);
1895
1896 out_unlock:
1897         mutex_unlock(&bo->wu_mutex);
1898         return ret;
1899 }