drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_module.h>
  35 #include <drm/ttm/ttm_bo_driver.h>
  36 #include <drm/ttm/ttm_placement.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40 #include <linux/mm.h>
  41 #include <linux/file.h>
  42 #include <linux/module.h>
  43 #include <linux/atomic.h>
  44 #include <linux/reservation.h>
  45
  46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  47
  48 /**
  49  * ttm_global_mutex - protecting the global BO state
  50  */
  51 DEFINE_MUTEX(ttm_global_mutex);
  52 struct ttm_bo_global ttm_bo_glob = {
  53         .use_count = 0
  54 };
  55
  56 static struct attribute ttm_bo_count = {
  57         .name = "bo_count",
  58         .mode = S_IRUGO
  59 };
  60
  61 /* default destructor */
  62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  63 {
  64         kfree(bo);
  65 }
  66
  67 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
  68                                           uint32_t *mem_type)
  69 {
  70         int pos;
  71
  72         pos = ffs(place->flags & TTM_PL_MASK_MEM);
  73         if (unlikely(!pos))
  74                 return -EINVAL;
  75
  76         *mem_type = pos - 1;
  77         return 0;
  78 }
  79
  80 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p,
  81                                int mem_type)
  82 {
  83         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  84
  85         drm_printf(p, "    has_type: %d\n", man->has_type);
  86         drm_printf(p, "    use_type: %d\n", man->use_type);
  87         drm_printf(p, "    flags: 0x%08X\n", man->flags);
  88         drm_printf(p, "    gpu_offset: 0x%08llX\n", man->gpu_offset);
  89         drm_printf(p, "    size: %llu\n", man->size);
  90         drm_printf(p, "    available_caching: 0x%08X\n", man->available_caching);
  91         drm_printf(p, "    default_caching: 0x%08X\n", man->default_caching);
  92         if (mem_type != TTM_PL_SYSTEM)
  93                 (*man->func->debug)(man, p);
  94 }
  95
  96 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  97                                         struct ttm_placement *placement)
  98 {
  99         struct drm_printer p = drm_debug_printer(TTM_PFX);
 100         int i, ret, mem_type;
 101
 102         drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
 103                    bo, bo->mem.num_pages, bo->mem.size >> 10,
 104                    bo->mem.size >> 20);
 105         for (i = 0; i < placement->num_placement; i++) {
 106                 ret = ttm_mem_type_from_place(&placement->placement[i],
 107                                                 &mem_type);
 108                 if (ret)
 109                         return;
 110                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
 111                            i, placement->placement[i].flags, mem_type);
 112                 ttm_mem_type_debug(bo->bdev, &p, mem_type);
 113         }
 114 }
 115
 116 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 117                                   struct attribute *attr,
 118                                   char *buffer)
 119 {
 120         struct ttm_bo_global *glob =
 121                 container_of(kobj, struct ttm_bo_global, kobj);
 122
 123         return snprintf(buffer, PAGE_SIZE, "%d\n",
 124                                 atomic_read(&glob->bo_count));
 125 }
 126
 127 static struct attribute *ttm_bo_global_attrs[] = {
 128         &ttm_bo_count,
 129         NULL
 130 };
 131
 132 static const struct sysfs_ops ttm_bo_global_ops = {
 133         .show = &ttm_bo_global_show
 134 };
 135
 136 static struct kobj_type ttm_bo_glob_kobj_type  = {
 137         .release = &ttm_bo_global_kobj_release,
 138         .sysfs_ops = &ttm_bo_global_ops,
 139         .default_attrs = ttm_bo_global_attrs
 140 };
 141
 142
 143 static inline uint32_t ttm_bo_type_flags(unsigned type)
 144 {
 145         return 1 << (type);
 146 }
 147
 148 static void ttm_bo_release_list(struct kref *list_kref)
 149 {
 150         struct ttm_buffer_object *bo =
 151             container_of(list_kref, struct ttm_buffer_object, list_kref);
 152         struct ttm_bo_device *bdev = bo->bdev;
 153         size_t acc_size = bo->acc_size;
 154
 155         BUG_ON(kref_read(&bo->list_kref));
 156         BUG_ON(kref_read(&bo->kref));
 157         BUG_ON(atomic_read(&bo->cpu_writers));
 158         BUG_ON(bo->mem.mm_node != NULL);
 159         BUG_ON(!list_empty(&bo->lru));
 160         BUG_ON(!list_empty(&bo->ddestroy));
 161         ttm_tt_destroy(bo->ttm);
 162         atomic_dec(&bo->bdev->glob->bo_count);
 163         dma_fence_put(bo->moving);
 164         reservation_object_fini(&bo->ttm_resv);
 165         mutex_destroy(&bo->wu_mutex);
 166         bo->destroy(bo);
 167         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
 168 }
 169
 170 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 171 {
 172         struct ttm_bo_device *bdev = bo->bdev;
 173         struct ttm_mem_type_manager *man;
 174
 175         reservation_object_assert_held(bo->resv);
 176
 177         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 178                 BUG_ON(!list_empty(&bo->lru));
 179
 180                 man = &bdev->man[bo->mem.mem_type];
 181                 list_add_tail(&bo->lru, &man->lru[bo->priority]);
 182                 kref_get(&bo->list_kref);
 183
 184                 if (bo->ttm && !(bo->ttm->page_flags &
 185                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) {
 186                         list_add_tail(&bo->swap,
 187                                       &bdev->glob->swap_lru[bo->priority]);
 188                         kref_get(&bo->list_kref);
 189                 }
 190         }
 191 }
 192 EXPORT_SYMBOL(ttm_bo_add_to_lru);
 193
 194 static void ttm_bo_ref_bug(struct kref *list_kref)
 195 {
 196         BUG();
 197 }
 198
 199 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 200 {
 201         if (!list_empty(&bo->swap)) {
 202                 list_del_init(&bo->swap);
 203                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 204         }
 205         if (!list_empty(&bo->lru)) {
 206                 list_del_init(&bo->lru);
 207                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 208         }
 209
 210         /*
 211          * TODO: Add a driver hook to delete from
 212          * driver-specific LRU's here.
 213          */
 214 }
 215
 216 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
 217 {
 218         struct ttm_bo_global *glob = bo->bdev->glob;
 219
 220         spin_lock(&glob->lru_lock);
 221         ttm_bo_del_from_lru(bo);
 222         spin_unlock(&glob->lru_lock);
 223 }
 224 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 225
 226 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
 227                                      struct ttm_buffer_object *bo)
 228 {
 229         if (!pos->first)
 230                 pos->first = bo;
 231         pos->last = bo;
 232 }
 233
 234 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
 235                              struct ttm_lru_bulk_move *bulk)
 236 {
 237         reservation_object_assert_held(bo->resv);
 238
 239         ttm_bo_del_from_lru(bo);
 240         ttm_bo_add_to_lru(bo);
 241
 242         if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 243                 switch (bo->mem.mem_type) {
 244                 case TTM_PL_TT:
 245                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 246                         break;
 247
 248                 case TTM_PL_VRAM:
 249                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 250                         break;
 251                 }
 252                 if (bo->ttm && !(bo->ttm->page_flags &
 253                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 254                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 255         }
 256 }
 257 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 258
 259 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 260 {
 261         unsigned i;
 262
 263         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 264                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 265                 struct ttm_mem_type_manager *man;
 266
 267                 if (!pos->first)
 268                         continue;
 269
 270                 reservation_object_assert_held(pos->first->resv);
 271                 reservation_object_assert_held(pos->last->resv);
 272
 273                 man = &pos->first->bdev->man[TTM_PL_TT];
 274                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 275                                     &pos->last->lru);
 276         }
 277
 278         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 279                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 280                 struct ttm_mem_type_manager *man;
 281
 282                 if (!pos->first)
 283                         continue;
 284
 285                 reservation_object_assert_held(pos->first->resv);
 286                 reservation_object_assert_held(pos->last->resv);
 287
 288                 man = &pos->first->bdev->man[TTM_PL_VRAM];
 289                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 290                                     &pos->last->lru);
 291         }
 292
 293         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 294                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 295                 struct list_head *lru;
 296
 297                 if (!pos->first)
 298                         continue;
 299
 300                 reservation_object_assert_held(pos->first->resv);
 301                 reservation_object_assert_held(pos->last->resv);
 302
 303                 lru = &pos->first->bdev->glob->swap_lru[i];
 304                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 305         }
 306 }
 307 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 308
 309 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 310                                   struct ttm_mem_reg *mem, bool evict,
 311                                   struct ttm_operation_ctx *ctx)
 312 {
 313         struct ttm_bo_device *bdev = bo->bdev;
 314         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 315         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 316         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 317         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 318         int ret = 0;
 319
 320         if (old_is_pci || new_is_pci ||
 321             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 322                 ret = ttm_mem_io_lock(old_man, true);
 323                 if (unlikely(ret != 0))
 324                         goto out_err;
 325                 ttm_bo_unmap_virtual_locked(bo);
 326                 ttm_mem_io_unlock(old_man);
 327         }
 328
 329         /*
 330          * Create and bind a ttm if required.
 331          */
 332
 333         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 334                 if (bo->ttm == NULL) {
 335                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 336                         ret = ttm_tt_create(bo, zero);
 337                         if (ret)
 338                                 goto out_err;
 339                 }
 340
 341                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 342                 if (ret)
 343                         goto out_err;
 344
 345                 if (mem->mem_type != TTM_PL_SYSTEM) {
 346                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
 347                         if (ret)
 348                                 goto out_err;
 349                 }
 350
 351                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 352                         if (bdev->driver->move_notify)
 353                                 bdev->driver->move_notify(bo, evict, mem);
 354                         bo->mem = *mem;
 355                         mem->mm_node = NULL;
 356                         goto moved;
 357                 }
 358         }
 359
 360         if (bdev->driver->move_notify)
 361                 bdev->driver->move_notify(bo, evict, mem);
 362
 363         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 364             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 365                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 366         else if (bdev->driver->move)
 367                 ret = bdev->driver->move(bo, evict, ctx, mem);
 368         else
 369                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 370
 371         if (ret) {
 372                 if (bdev->driver->move_notify) {
 373                         swap(*mem, bo->mem);
 374                         bdev->driver->move_notify(bo, false, mem);
 375                         swap(*mem, bo->mem);
 376                 }
 377
 378                 goto out_err;
 379         }
 380
 381 moved:
 382         if (bo->evicted) {
 383                 if (bdev->driver->invalidate_caches) {
 384                         ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
 385                         if (ret)
 386                                 pr_err("Can not flush read caches\n");
 387                 }
 388                 bo->evicted = false;
 389         }
 390
 391         if (bo->mem.mm_node)
 392                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 393                     bdev->man[bo->mem.mem_type].gpu_offset;
 394         else
 395                 bo->offset = 0;
 396
 397         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 398         return 0;
 399
 400 out_err:
 401         new_man = &bdev->man[bo->mem.mem_type];
 402         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 403                 ttm_tt_destroy(bo->ttm);
 404                 bo->ttm = NULL;
 405         }
 406
 407         return ret;
 408 }
 409
 410 /**
 411  * Call bo::reserved.
 412  * Will release GPU memory type usage on destruction.
 413  * This is the place to put in driver specific hooks to release
 414  * driver private resources.
 415  * Will release the bo::reserved lock.
 416  */
 417
 418 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 419 {
 420         if (bo->bdev->driver->move_notify)
 421                 bo->bdev->driver->move_notify(bo, false, NULL);
 422
 423         ttm_tt_destroy(bo->ttm);
 424         bo->ttm = NULL;
 425         ttm_bo_mem_put(bo, &bo->mem);
 426 }
 427
 428 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 429 {
 430         int r;
 431
 432         if (bo->resv == &bo->ttm_resv)
 433                 return 0;
 434
 435         BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
 436
 437         r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
 438         if (r)
 439                 reservation_object_unlock(&bo->ttm_resv);
 440
 441         return r;
 442 }
 443
 444 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 445 {
 446         struct reservation_object_list *fobj;
 447         struct dma_fence *fence;
 448         int i;
 449
 450         fobj = reservation_object_get_list(&bo->ttm_resv);
 451         fence = reservation_object_get_excl(&bo->ttm_resv);
 452         if (fence && !fence->ops->signaled)
 453                 dma_fence_enable_sw_signaling(fence);
 454
 455         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 456                 fence = rcu_dereference_protected(fobj->shared[i],
 457                                         reservation_object_held(bo->resv));
 458
 459                 if (!fence->ops->signaled)
 460                         dma_fence_enable_sw_signaling(fence);
 461         }
 462 }
 463
 464 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 465 {
 466         struct ttm_bo_device *bdev = bo->bdev;
 467         struct ttm_bo_global *glob = bdev->glob;
 468         int ret;
 469
 470         ret = ttm_bo_individualize_resv(bo);
 471         if (ret) {
 472                 /* Last resort, if we fail to allocate memory for the
 473                  * fences block for the BO to become idle
 474                  */
 475                 reservation_object_wait_timeout_rcu(bo->resv, true, false,
 476                                                     30 * HZ);
 477                 spin_lock(&glob->lru_lock);
 478                 goto error;
 479         }
 480
 481         spin_lock(&glob->lru_lock);
 482         ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
 483         if (!ret) {
 484                 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
 485                         ttm_bo_del_from_lru(bo);
 486                         spin_unlock(&glob->lru_lock);
 487                         if (bo->resv != &bo->ttm_resv)
 488                                 reservation_object_unlock(&bo->ttm_resv);
 489
 490                         ttm_bo_cleanup_memtype_use(bo);
 491                         reservation_object_unlock(bo->resv);
 492                         return;
 493                 }
 494
 495                 ttm_bo_flush_all_fences(bo);
 496
 497                 /*
 498                  * Make NO_EVICT bos immediately available to
 499                  * shrinkers, now that they are queued for
 500                  * destruction.
 501                  */
 502                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 503                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 504                         ttm_bo_add_to_lru(bo);
 505                 }
 506
 507                 reservation_object_unlock(bo->resv);
 508         }
 509         if (bo->resv != &bo->ttm_resv)
 510                 reservation_object_unlock(&bo->ttm_resv);
 511
 512 error:
 513         kref_get(&bo->list_kref);
 514         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 515         spin_unlock(&glob->lru_lock);
 516
 517         schedule_delayed_work(&bdev->wq,
 518                               ((HZ / 100) < 1) ? 1 : HZ / 100);
 519 }
 520
 521 /**
 522  * function ttm_bo_cleanup_refs
 523  * If bo idle, remove from delayed- and lru lists, and unref.
 524  * If not idle, do nothing.
 525  *
 526  * Must be called with lru_lock and reservation held, this function
 527  * will drop the lru lock and optionally the reservation lock before returning.
 528  *
 529  * @interruptible         Any sleeps should occur interruptibly.
 530  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 531  * @unlock_resv           Unlock the reservation lock as well.
 532  */
 533
 534 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 535                                bool interruptible, bool no_wait_gpu,
 536                                bool unlock_resv)
 537 {
 538         struct ttm_bo_global *glob = bo->bdev->glob;
 539         struct reservation_object *resv;
 540         int ret;
 541
 542         if (unlikely(list_empty(&bo->ddestroy)))
 543                 resv = bo->resv;
 544         else
 545                 resv = &bo->ttm_resv;
 546
 547         if (reservation_object_test_signaled_rcu(resv, true))
 548                 ret = 0;
 549         else
 550                 ret = -EBUSY;
 551
 552         if (ret && !no_wait_gpu) {
 553                 long lret;
 554
 555                 if (unlock_resv)
 556                         reservation_object_unlock(bo->resv);
 557                 spin_unlock(&glob->lru_lock);
 558
 559                 lret = reservation_object_wait_timeout_rcu(resv, true,
 560                                                            interruptible,
 561                                                            30 * HZ);
 562
 563                 if (lret < 0)
 564                         return lret;
 565                 else if (lret == 0)
 566                         return -EBUSY;
 567
 568                 spin_lock(&glob->lru_lock);
 569                 if (unlock_resv && !reservation_object_trylock(bo->resv)) {
 570                         /*
 571                          * We raced, and lost, someone else holds the reservation now,
 572                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 573                          *
 574                          * Even if it's not the case, because we finished waiting any
 575                          * delayed destruction would succeed, so just return success
 576                          * here.
 577                          */
 578                         spin_unlock(&glob->lru_lock);
 579                         return 0;
 580                 }
 581                 ret = 0;
 582         }
 583
 584         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 585                 if (unlock_resv)
 586                         reservation_object_unlock(bo->resv);
 587                 spin_unlock(&glob->lru_lock);
 588                 return ret;
 589         }
 590
 591         ttm_bo_del_from_lru(bo);
 592         list_del_init(&bo->ddestroy);
 593         kref_put(&bo->list_kref, ttm_bo_ref_bug);
 594
 595         spin_unlock(&glob->lru_lock);
 596         ttm_bo_cleanup_memtype_use(bo);
 597
 598         if (unlock_resv)
 599                 reservation_object_unlock(bo->resv);
 600
 601         return 0;
 602 }
 603
 604 /**
 605  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 606  * encountered buffers.
 607  */
 608 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 609 {
 610         struct ttm_bo_global *glob = bdev->glob;
 611         struct list_head removed;
 612         bool empty;
 613
 614         INIT_LIST_HEAD(&removed);
 615
 616         spin_lock(&glob->lru_lock);
 617         while (!list_empty(&bdev->ddestroy)) {
 618                 struct ttm_buffer_object *bo;
 619
 620                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 621                                       ddestroy);
 622                 kref_get(&bo->list_kref);
 623                 list_move_tail(&bo->ddestroy, &removed);
 624
 625                 if (remove_all || bo->resv != &bo->ttm_resv) {
 626                         spin_unlock(&glob->lru_lock);
 627                         reservation_object_lock(bo->resv, NULL);
 628
 629                         spin_lock(&glob->lru_lock);
 630                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 631
 632                 } else if (reservation_object_trylock(bo->resv)) {
 633                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 634                 } else {
 635                         spin_unlock(&glob->lru_lock);
 636                 }
 637
 638                 kref_put(&bo->list_kref, ttm_bo_release_list);
 639                 spin_lock(&glob->lru_lock);
 640         }
 641         list_splice_tail(&removed, &bdev->ddestroy);
 642         empty = list_empty(&bdev->ddestroy);
 643         spin_unlock(&glob->lru_lock);
 644
 645         return empty;
 646 }
 647
 648 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 649 {
 650         struct ttm_bo_device *bdev =
 651             container_of(work, struct ttm_bo_device, wq.work);
 652
 653         if (!ttm_bo_delayed_delete(bdev, false))
 654                 schedule_delayed_work(&bdev->wq,
 655                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 656 }
 657
 658 static void ttm_bo_release(struct kref *kref)
 659 {
 660         struct ttm_buffer_object *bo =
 661             container_of(kref, struct ttm_buffer_object, kref);
 662         struct ttm_bo_device *bdev = bo->bdev;
 663         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 664
 665         drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
 666         ttm_mem_io_lock(man, false);
 667         ttm_mem_io_free_vm(bo);
 668         ttm_mem_io_unlock(man);
 669         ttm_bo_cleanup_refs_or_queue(bo);
 670         kref_put(&bo->list_kref, ttm_bo_release_list);
 671 }
 672
 673 void ttm_bo_put(struct ttm_buffer_object *bo)
 674 {
 675         kref_put(&bo->kref, ttm_bo_release);
 676 }
 677 EXPORT_SYMBOL(ttm_bo_put);
 678
 679 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 680 {
 681         struct ttm_buffer_object *bo = *p_bo;
 682
 683         *p_bo = NULL;
 684         ttm_bo_put(bo);
 685 }
 686 EXPORT_SYMBOL(ttm_bo_unref);
 687
 688 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 689 {
 690         return cancel_delayed_work_sync(&bdev->wq);
 691 }
 692 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 693
 694 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 695 {
 696         if (resched)
 697                 schedule_delayed_work(&bdev->wq,
 698                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 699 }
 700 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 701
 702 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 703                         struct ttm_operation_ctx *ctx)
 704 {
 705         struct ttm_bo_device *bdev = bo->bdev;
 706         struct ttm_mem_reg evict_mem;
 707         struct ttm_placement placement;
 708         int ret = 0;
 709
 710         reservation_object_assert_held(bo->resv);
 711
 712         placement.num_placement = 0;
 713         placement.num_busy_placement = 0;
 714         bdev->driver->evict_flags(bo, &placement);
 715
 716         if (!placement.num_placement && !placement.num_busy_placement) {
 717                 ret = ttm_bo_pipeline_gutting(bo);
 718                 if (ret)
 719                         return ret;
 720
 721                 return ttm_tt_create(bo, false);
 722         }
 723
 724         evict_mem = bo->mem;
 725         evict_mem.mm_node = NULL;
 726         evict_mem.bus.io_reserved_vm = false;
 727         evict_mem.bus.io_reserved_count = 0;
 728
 729         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 730         if (ret) {
 731                 if (ret != -ERESTARTSYS) {
 732                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 733                                bo);
 734                         ttm_bo_mem_space_debug(bo, &placement);
 735                 }
 736                 goto out;
 737         }
 738
 739         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 740         if (unlikely(ret)) {
 741                 if (ret != -ERESTARTSYS)
 742                         pr_err("Buffer eviction failed\n");
 743                 ttm_bo_mem_put(bo, &evict_mem);
 744                 goto out;
 745         }
 746         bo->evicted = true;
 747 out:
 748         return ret;
 749 }
 750
 751 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 752                               const struct ttm_place *place)
 753 {
 754         /* Don't evict this BO if it's outside of the
 755          * requested placement range
 756          */
 757         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 758             (place->lpfn && place->lpfn <= bo->mem.start))
 759                 return false;
 760
 761         return true;
 762 }
 763 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 764
 765 /**
 766  * Check the target bo is allowable to be evicted or swapout, including cases:
 767  *
 768  * a. if share same reservation object with ctx->resv, have assumption
 769  * reservation objects should already be locked, so not lock again and
 770  * return true directly when either the opreation allow_reserved_eviction
 771  * or the target bo already is in delayed free list;
 772  *
 773  * b. Otherwise, trylock it.
 774  */
 775 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 776                         struct ttm_operation_ctx *ctx, bool *locked)
 777 {
 778         bool ret = false;
 779
 780         *locked = false;
 781         if (bo->resv == ctx->resv) {
 782                 reservation_object_assert_held(bo->resv);
 783                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT
 784                     || !list_empty(&bo->ddestroy))
 785                         ret = true;
 786         } else {
 787                 *locked = reservation_object_trylock(bo->resv);
 788                 ret = *locked;
 789         }
 790
 791         return ret;
 792 }
 793
 794 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 795                                uint32_t mem_type,
 796                                const struct ttm_place *place,
 797                                struct ttm_operation_ctx *ctx)
 798 {
 799         struct ttm_bo_global *glob = bdev->glob;
 800         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 801         struct ttm_buffer_object *bo = NULL;
 802         bool locked = false;
 803         unsigned i;
 804         int ret;
 805
 806         spin_lock(&glob->lru_lock);
 807         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 808                 list_for_each_entry(bo, &man->lru[i], lru) {
 809                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked))
 810                                 continue;
 811
 812                         if (place && !bdev->driver->eviction_valuable(bo,
 813                                                                       place)) {
 814                                 if (locked)
 815                                         reservation_object_unlock(bo->resv);
 816                                 continue;
 817                         }
 818                         break;
 819                 }
 820
 821                 /* If the inner loop terminated early, we have our candidate */
 822                 if (&bo->lru != &man->lru[i])
 823                         break;
 824
 825                 bo = NULL;
 826         }
 827
 828         if (!bo) {
 829                 spin_unlock(&glob->lru_lock);
 830                 return -EBUSY;
 831         }
 832
 833         kref_get(&bo->list_kref);
 834
 835         if (!list_empty(&bo->ddestroy)) {
 836                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 837                                           ctx->no_wait_gpu, locked);
 838                 kref_put(&bo->list_kref, ttm_bo_release_list);
 839                 return ret;
 840         }
 841
 842         ttm_bo_del_from_lru(bo);
 843         spin_unlock(&glob->lru_lock);
 844
 845         ret = ttm_bo_evict(bo, ctx);
 846         if (locked) {
 847                 ttm_bo_unreserve(bo);
 848         } else {
 849                 spin_lock(&glob->lru_lock);
 850                 ttm_bo_add_to_lru(bo);
 851                 spin_unlock(&glob->lru_lock);
 852         }
 853
 854         kref_put(&bo->list_kref, ttm_bo_release_list);
 855         return ret;
 856 }
 857
 858 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 859 {
 860         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 861
 862         if (mem->mm_node)
 863                 (*man->func->put_node)(man, mem);
 864 }
 865 EXPORT_SYMBOL(ttm_bo_mem_put);
 866
 867 /**
 868  * Add the last move fence to the BO and reserve a new shared slot.
 869  */
 870 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 871                                  struct ttm_mem_type_manager *man,
 872                                  struct ttm_mem_reg *mem)
 873 {
 874         struct dma_fence *fence;
 875         int ret;
 876
 877         spin_lock(&man->move_lock);
 878         fence = dma_fence_get(man->move);
 879         spin_unlock(&man->move_lock);
 880
 881         if (fence) {
 882                 reservation_object_add_shared_fence(bo->resv, fence);
 883
 884                 ret = reservation_object_reserve_shared(bo->resv, 1);
 885                 if (unlikely(ret))
 886                         return ret;
 887
 888                 dma_fence_put(bo->moving);
 889                 bo->moving = fence;
 890         }
 891
 892         return 0;
 893 }
 894
 895 /**
 896  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 897  * space, or we've evicted everything and there isn't enough space.
 898  */
 899 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 900                                         uint32_t mem_type,
 901                                         const struct ttm_place *place,
 902                                         struct ttm_mem_reg *mem,
 903                                         struct ttm_operation_ctx *ctx)
 904 {
 905         struct ttm_bo_device *bdev = bo->bdev;
 906         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 907         int ret;
 908
 909         do {
 910                 ret = (*man->func->get_node)(man, bo, place, mem);
 911                 if (unlikely(ret != 0))
 912                         return ret;
 913                 if (mem->mm_node)
 914                         break;
 915                 ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);
 916                 if (unlikely(ret != 0))
 917                         return ret;
 918         } while (1);
 919         mem->mem_type = mem_type;
 920         return ttm_bo_add_move_fence(bo, man, mem);
 921 }
 922
 923 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 924                                       uint32_t cur_placement,
 925                                       uint32_t proposed_placement)
 926 {
 927         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 928         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 929
 930         /**
 931          * Keep current caching if possible.
 932          */
 933
 934         if ((cur_placement & caching) != 0)
 935                 result |= (cur_placement & caching);
 936         else if ((man->default_caching & caching) != 0)
 937                 result |= man->default_caching;
 938         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 939                 result |= TTM_PL_FLAG_CACHED;
 940         else if ((TTM_PL_FLAG_WC & caching) != 0)
 941                 result |= TTM_PL_FLAG_WC;
 942         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 943                 result |= TTM_PL_FLAG_UNCACHED;
 944
 945         return result;
 946 }
 947
 948 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 949                                  uint32_t mem_type,
 950                                  const struct ttm_place *place,
 951                                  uint32_t *masked_placement)
 952 {
 953         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 954
 955         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 956                 return false;
 957
 958         if ((place->flags & man->available_caching) == 0)
 959                 return false;
 960
 961         cur_flags |= (place->flags & man->available_caching);
 962
 963         *masked_placement = cur_flags;
 964         return true;
 965 }
 966
 967 /**
 968  * Creates space for memory region @mem according to its type.
 969  *
 970  * This function first searches for free space in compatible memory types in
 971  * the priority order defined by the driver.  If free space isn't found, then
 972  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 973  * space.
 974  */
 975 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 976                         struct ttm_placement *placement,
 977                         struct ttm_mem_reg *mem,
 978                         struct ttm_operation_ctx *ctx)
 979 {
 980         struct ttm_bo_device *bdev = bo->bdev;
 981         struct ttm_mem_type_manager *man;
 982         uint32_t mem_type = TTM_PL_SYSTEM;
 983         uint32_t cur_flags = 0;
 984         bool type_found = false;
 985         bool type_ok = false;
 986         bool has_erestartsys = false;
 987         int i, ret;
 988
 989         ret = reservation_object_reserve_shared(bo->resv, 1);
 990         if (unlikely(ret))
 991                 return ret;
 992
 993         mem->mm_node = NULL;
 994         for (i = 0; i < placement->num_placement; ++i) {
 995                 const struct ttm_place *place = &placement->placement[i];
 996
 997                 ret = ttm_mem_type_from_place(place, &mem_type);
 998                 if (ret)
 999                         return ret;
1000                 man = &bdev->man[mem_type];
1001                 if (!man->has_type || !man->use_type)
1002                         continue;
1003
1004                 type_ok = ttm_bo_mt_compatible(man, mem_type, place,
1005                                                 &cur_flags);
1006
1007                 if (!type_ok)
1008                         continue;
1009
1010                 type_found = true;
1011                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1012                                                   cur_flags);
1013                 /*
1014                  * Use the access and other non-mapping-related flag bits from
1015                  * the memory placement flags to the current flags
1016                  */
1017                 ttm_flag_masked(&cur_flags, place->flags,
1018                                 ~TTM_PL_MASK_MEMTYPE);
1019
1020                 if (mem_type == TTM_PL_SYSTEM)
1021                         break;
1022
1023                 ret = (*man->func->get_node)(man, bo, place, mem);
1024                 if (unlikely(ret))
1025                         return ret;
1026
1027                 if (mem->mm_node) {
1028                         ret = ttm_bo_add_move_fence(bo, man, mem);
1029                         if (unlikely(ret)) {
1030                                 (*man->func->put_node)(man, mem);
1031                                 return ret;
1032                         }
1033                         break;
1034                 }
1035         }
1036
1037         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
1038                 mem->mem_type = mem_type;
1039                 mem->placement = cur_flags;
1040                 return 0;
1041         }
1042
1043         for (i = 0; i < placement->num_busy_placement; ++i) {
1044                 const struct ttm_place *place = &placement->busy_placement[i];
1045
1046                 ret = ttm_mem_type_from_place(place, &mem_type);
1047                 if (ret)
1048                         return ret;
1049                 man = &bdev->man[mem_type];
1050                 if (!man->has_type || !man->use_type)
1051                         continue;
1052                 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
1053                         continue;
1054
1055                 type_found = true;
1056                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1057                                                   cur_flags);
1058                 /*
1059                  * Use the access and other non-mapping-related flag bits from
1060                  * the memory placement flags to the current flags
1061                  */
1062                 ttm_flag_masked(&cur_flags, place->flags,
1063                                 ~TTM_PL_MASK_MEMTYPE);
1064
1065                 if (mem_type == TTM_PL_SYSTEM) {
1066                         mem->mem_type = mem_type;
1067                         mem->placement = cur_flags;
1068                         mem->mm_node = NULL;
1069                         return 0;
1070                 }
1071
1072                 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx);
1073                 if (ret == 0 && mem->mm_node) {
1074                         mem->placement = cur_flags;
1075                         return 0;
1076                 }
1077                 if (ret == -ERESTARTSYS)
1078                         has_erestartsys = true;
1079         }
1080
1081         if (!type_found) {
1082                 pr_err(TTM_PFX "No compatible memory type found\n");
1083                 return -EINVAL;
1084         }
1085
1086         return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
1087 }
1088 EXPORT_SYMBOL(ttm_bo_mem_space);
1089
1090 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1091                               struct ttm_placement *placement,
1092                               struct ttm_operation_ctx *ctx)
1093 {
1094         int ret = 0;
1095         struct ttm_mem_reg mem;
1096
1097         reservation_object_assert_held(bo->resv);
1098
1099         mem.num_pages = bo->num_pages;
1100         mem.size = mem.num_pages << PAGE_SHIFT;
1101         mem.page_alignment = bo->mem.page_alignment;
1102         mem.bus.io_reserved_vm = false;
1103         mem.bus.io_reserved_count = 0;
1104         /*
1105          * Determine where to move the buffer.
1106          */
1107         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1108         if (ret)
1109                 goto out_unlock;
1110         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1111 out_unlock:
1112         if (ret && mem.mm_node)
1113                 ttm_bo_mem_put(bo, &mem);
1114         return ret;
1115 }
1116
1117 static bool ttm_bo_places_compat(const struct ttm_place *places,
1118                                  unsigned num_placement,
1119                                  struct ttm_mem_reg *mem,
1120                                  uint32_t *new_flags)
1121 {
1122         unsigned i;
1123
1124         for (i = 0; i < num_placement; i++) {
1125                 const struct ttm_place *heap = &places[i];
1126
1127                 if (mem->mm_node && (mem->start < heap->fpfn ||
1128                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1129                         continue;
1130
1131                 *new_flags = heap->flags;
1132                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1133                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1134                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1135                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1136                         return true;
1137         }
1138         return false;
1139 }
1140
1141 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1142                        struct ttm_mem_reg *mem,
1143                        uint32_t *new_flags)
1144 {
1145         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1146                                  mem, new_flags))
1147                 return true;
1148
1149         if ((placement->busy_placement != placement->placement ||
1150              placement->num_busy_placement > placement->num_placement) &&
1151             ttm_bo_places_compat(placement->busy_placement,
1152                                  placement->num_busy_placement,
1153                                  mem, new_flags))
1154                 return true;
1155
1156         return false;
1157 }
1158 EXPORT_SYMBOL(ttm_bo_mem_compat);
1159
1160 int ttm_bo_validate(struct ttm_buffer_object *bo,
1161                     struct ttm_placement *placement,
1162                     struct ttm_operation_ctx *ctx)
1163 {
1164         int ret;
1165         uint32_t new_flags;
1166
1167         reservation_object_assert_held(bo->resv);
1168         /*
1169          * Check whether we need to move buffer.
1170          */
1171         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1172                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1173                 if (ret)
1174                         return ret;
1175         } else {
1176                 /*
1177                  * Use the access and other non-mapping-related flag bits from
1178                  * the compatible memory placement flags to the active flags
1179                  */
1180                 ttm_flag_masked(&bo->mem.placement, new_flags,
1181                                 ~TTM_PL_MASK_MEMTYPE);
1182         }
1183         /*
1184          * We might need to add a TTM.
1185          */
1186         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1187                 ret = ttm_tt_create(bo, true);
1188                 if (ret)
1189                         return ret;
1190         }
1191         return 0;
1192 }
1193 EXPORT_SYMBOL(ttm_bo_validate);
1194
1195 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1196                          struct ttm_buffer_object *bo,
1197                          unsigned long size,
1198                          enum ttm_bo_type type,
1199                          struct ttm_placement *placement,
1200                          uint32_t page_alignment,
1201                          struct ttm_operation_ctx *ctx,
1202                          size_t acc_size,
1203                          struct sg_table *sg,
1204                          struct reservation_object *resv,
1205                          void (*destroy) (struct ttm_buffer_object *))
1206 {
1207         int ret = 0;
1208         unsigned long num_pages;
1209         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1210         bool locked;
1211
1212         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1213         if (ret) {
1214                 pr_err("Out of kernel memory\n");
1215                 if (destroy)
1216                         (*destroy)(bo);
1217                 else
1218                         kfree(bo);
1219                 return -ENOMEM;
1220         }
1221
1222         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1223         if (num_pages == 0) {
1224                 pr_err("Illegal buffer object size\n");
1225                 if (destroy)
1226                         (*destroy)(bo);
1227                 else
1228                         kfree(bo);
1229                 ttm_mem_global_free(mem_glob, acc_size);
1230                 return -EINVAL;
1231         }
1232         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1233
1234         kref_init(&bo->kref);
1235         kref_init(&bo->list_kref);
1236         atomic_set(&bo->cpu_writers, 0);
1237         INIT_LIST_HEAD(&bo->lru);
1238         INIT_LIST_HEAD(&bo->ddestroy);
1239         INIT_LIST_HEAD(&bo->swap);
1240         INIT_LIST_HEAD(&bo->io_reserve_lru);
1241         mutex_init(&bo->wu_mutex);
1242         bo->bdev = bdev;
1243         bo->type = type;
1244         bo->num_pages = num_pages;
1245         bo->mem.size = num_pages << PAGE_SHIFT;
1246         bo->mem.mem_type = TTM_PL_SYSTEM;
1247         bo->mem.num_pages = bo->num_pages;
1248         bo->mem.mm_node = NULL;
1249         bo->mem.page_alignment = page_alignment;
1250         bo->mem.bus.io_reserved_vm = false;
1251         bo->mem.bus.io_reserved_count = 0;
1252         bo->moving = NULL;
1253         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1254         bo->acc_size = acc_size;
1255         bo->sg = sg;
1256         if (resv) {
1257                 bo->resv = resv;
1258                 reservation_object_assert_held(bo->resv);
1259         } else {
1260                 bo->resv = &bo->ttm_resv;
1261         }
1262         reservation_object_init(&bo->ttm_resv);
1263         atomic_inc(&bo->bdev->glob->bo_count);
1264         drm_vma_node_reset(&bo->vma_node);
1265
1266         /*
1267          * For ttm_bo_type_device buffers, allocate
1268          * address space from the device.
1269          */
1270         if (bo->type == ttm_bo_type_device ||
1271             bo->type == ttm_bo_type_sg)
1272                 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
1273                                          bo->mem.num_pages);
1274
1275         /* passed reservation objects should already be locked,
1276          * since otherwise lockdep will be angered in radeon.
1277          */
1278         if (!resv) {
1279                 locked = reservation_object_trylock(bo->resv);
1280                 WARN_ON(!locked);
1281         }
1282
1283         if (likely(!ret))
1284                 ret = ttm_bo_validate(bo, placement, ctx);
1285
1286         if (unlikely(ret)) {
1287                 if (!resv)
1288                         ttm_bo_unreserve(bo);
1289
1290                 ttm_bo_put(bo);
1291                 return ret;
1292         }
1293
1294         if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
1295                 spin_lock(&bdev->glob->lru_lock);
1296                 ttm_bo_add_to_lru(bo);
1297                 spin_unlock(&bdev->glob->lru_lock);
1298         }
1299
1300         return ret;
1301 }
1302 EXPORT_SYMBOL(ttm_bo_init_reserved);
1303
1304 int ttm_bo_init(struct ttm_bo_device *bdev,
1305                 struct ttm_buffer_object *bo,
1306                 unsigned long size,
1307                 enum ttm_bo_type type,
1308                 struct ttm_placement *placement,
1309                 uint32_t page_alignment,
1310                 bool interruptible,
1311                 size_t acc_size,
1312                 struct sg_table *sg,
1313                 struct reservation_object *resv,
1314                 void (*destroy) (struct ttm_buffer_object *))
1315 {
1316         struct ttm_operation_ctx ctx = { interruptible, false };
1317         int ret;
1318
1319         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1320                                    page_alignment, &ctx, acc_size,
1321                                    sg, resv, destroy);
1322         if (ret)
1323                 return ret;
1324
1325         if (!resv)
1326                 ttm_bo_unreserve(bo);
1327
1328         return 0;
1329 }
1330 EXPORT_SYMBOL(ttm_bo_init);
1331
1332 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1333                        unsigned long bo_size,
1334                        unsigned struct_size)
1335 {
1336         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1337         size_t size = 0;
1338
1339         size += ttm_round_pot(struct_size);
1340         size += ttm_round_pot(npages * sizeof(void *));
1341         size += ttm_round_pot(sizeof(struct ttm_tt));
1342         return size;
1343 }
1344 EXPORT_SYMBOL(ttm_bo_acc_size);
1345
1346 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1347                            unsigned long bo_size,
1348                            unsigned struct_size)
1349 {
1350         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1351         size_t size = 0;
1352
1353         size += ttm_round_pot(struct_size);
1354         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1355         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1356         return size;
1357 }
1358 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1359
1360 int ttm_bo_create(struct ttm_bo_device *bdev,
1361                         unsigned long size,
1362                         enum ttm_bo_type type,
1363                         struct ttm_placement *placement,
1364                         uint32_t page_alignment,
1365                         bool interruptible,
1366                         struct ttm_buffer_object **p_bo)
1367 {
1368         struct ttm_buffer_object *bo;
1369         size_t acc_size;
1370         int ret;
1371
1372         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1373         if (unlikely(bo == NULL))
1374                 return -ENOMEM;
1375
1376         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1377         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1378                           interruptible, acc_size,
1379                           NULL, NULL, NULL);
1380         if (likely(ret == 0))
1381                 *p_bo = bo;
1382
1383         return ret;
1384 }
1385 EXPORT_SYMBOL(ttm_bo_create);
1386
1387 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1388                                    unsigned mem_type)
1389 {
1390         struct ttm_operation_ctx ctx = {
1391                 .interruptible = false,
1392                 .no_wait_gpu = false,
1393                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1394         };
1395         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1396         struct ttm_bo_global *glob = bdev->glob;
1397         struct dma_fence *fence;
1398         int ret;
1399         unsigned i;
1400
1401         /*
1402          * Can't use standard list traversal since we're unlocking.
1403          */
1404
1405         spin_lock(&glob->lru_lock);
1406         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1407                 while (!list_empty(&man->lru[i])) {
1408                         spin_unlock(&glob->lru_lock);
1409                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
1410                         if (ret)
1411                                 return ret;
1412                         spin_lock(&glob->lru_lock);
1413                 }
1414         }
1415         spin_unlock(&glob->lru_lock);
1416
1417         spin_lock(&man->move_lock);
1418         fence = dma_fence_get(man->move);
1419         spin_unlock(&man->move_lock);
1420
1421         if (fence) {
1422                 ret = dma_fence_wait(fence, false);
1423                 dma_fence_put(fence);
1424                 if (ret)
1425                         return ret;
1426         }
1427
1428         return 0;
1429 }
1430
1431 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1432 {
1433         struct ttm_mem_type_manager *man;
1434         int ret = -EINVAL;
1435
1436         if (mem_type >= TTM_NUM_MEM_TYPES) {
1437                 pr_err("Illegal memory type %d\n", mem_type);
1438                 return ret;
1439         }
1440         man = &bdev->man[mem_type];
1441
1442         if (!man->has_type) {
1443                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1444                        mem_type);
1445                 return ret;
1446         }
1447
1448         man->use_type = false;
1449         man->has_type = false;
1450
1451         ret = 0;
1452         if (mem_type > 0) {
1453                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1454                 if (ret) {
1455                         pr_err("Cleanup eviction failed\n");
1456                         return ret;
1457                 }
1458
1459                 ret = (*man->func->takedown)(man);
1460         }
1461
1462         dma_fence_put(man->move);
1463         man->move = NULL;
1464
1465         return ret;
1466 }
1467 EXPORT_SYMBOL(ttm_bo_clean_mm);
1468
1469 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1470 {
1471         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1472
1473         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1474                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1475                 return -EINVAL;
1476         }
1477
1478         if (!man->has_type) {
1479                 pr_err("Memory type %u has not been initialized\n", mem_type);
1480                 return 0;
1481         }
1482
1483         return ttm_bo_force_list_clean(bdev, mem_type);
1484 }
1485 EXPORT_SYMBOL(ttm_bo_evict_mm);
1486
1487 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1488                         unsigned long p_size)
1489 {
1490         int ret;
1491         struct ttm_mem_type_manager *man;
1492         unsigned i;
1493
1494         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1495         man = &bdev->man[type];
1496         BUG_ON(man->has_type);
1497         man->io_reserve_fastpath = true;
1498         man->use_io_reserve_lru = false;
1499         mutex_init(&man->io_reserve_mutex);
1500         spin_lock_init(&man->move_lock);
1501         INIT_LIST_HEAD(&man->io_reserve_lru);
1502
1503         ret = bdev->driver->init_mem_type(bdev, type, man);
1504         if (ret)
1505                 return ret;
1506         man->bdev = bdev;
1507
1508         if (type != TTM_PL_SYSTEM) {
1509                 ret = (*man->func->init)(man, p_size);
1510                 if (ret)
1511                         return ret;
1512         }
1513         man->has_type = true;
1514         man->use_type = true;
1515         man->size = p_size;
1516
1517         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1518                 INIT_LIST_HEAD(&man->lru[i]);
1519         man->move = NULL;
1520
1521         return 0;
1522 }
1523 EXPORT_SYMBOL(ttm_bo_init_mm);
1524
1525 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1526 {
1527         struct ttm_bo_global *glob =
1528                 container_of(kobj, struct ttm_bo_global, kobj);
1529
1530         __free_page(glob->dummy_read_page);
1531 }
1532
1533 static void ttm_bo_global_release(void)
1534 {
1535         struct ttm_bo_global *glob = &ttm_bo_glob;
1536
1537         mutex_lock(&ttm_global_mutex);
1538         if (--glob->use_count > 0)
1539                 goto out;
1540
1541         kobject_del(&glob->kobj);
1542         kobject_put(&glob->kobj);
1543         ttm_mem_global_release(&ttm_mem_glob);
1544 out:
1545         mutex_unlock(&ttm_global_mutex);
1546 }
1547
1548 static int ttm_bo_global_init(void)
1549 {
1550         struct ttm_bo_global *glob = &ttm_bo_glob;
1551         int ret = 0;
1552         unsigned i;
1553
1554         mutex_lock(&ttm_global_mutex);
1555         if (++glob->use_count > 1)
1556                 goto out;
1557
1558         ret = ttm_mem_global_init(&ttm_mem_glob);
1559         if (ret)
1560                 goto out;
1561
1562         spin_lock_init(&glob->lru_lock);
1563         glob->mem_glob = &ttm_mem_glob;
1564         glob->mem_glob->bo_glob = glob;
1565         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1566
1567         if (unlikely(glob->dummy_read_page == NULL)) {
1568                 ret = -ENOMEM;
1569                 goto out;
1570         }
1571
1572         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1573                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1574         INIT_LIST_HEAD(&glob->device_list);
1575         atomic_set(&glob->bo_count, 0);
1576
1577         ret = kobject_init_and_add(
1578                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1579         if (unlikely(ret != 0))
1580                 kobject_put(&glob->kobj);
1581 out:
1582         mutex_unlock(&ttm_global_mutex);
1583         return ret;
1584 }
1585
1586 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1587 {
1588         int ret = 0;
1589         unsigned i = TTM_NUM_MEM_TYPES;
1590         struct ttm_mem_type_manager *man;
1591         struct ttm_bo_global *glob = bdev->glob;
1592
1593         while (i--) {
1594                 man = &bdev->man[i];
1595                 if (man->has_type) {
1596                         man->use_type = false;
1597                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1598                                 ret = -EBUSY;
1599                                 pr_err("DRM memory manager type %d is not clean\n",
1600                                        i);
1601                         }
1602                         man->has_type = false;
1603                 }
1604         }
1605
1606         mutex_lock(&ttm_global_mutex);
1607         list_del(&bdev->device_list);
1608         mutex_unlock(&ttm_global_mutex);
1609
1610         cancel_delayed_work_sync(&bdev->wq);
1611
1612         if (ttm_bo_delayed_delete(bdev, true))
1613                 pr_debug("Delayed destroy list was clean\n");
1614
1615         spin_lock(&glob->lru_lock);
1616         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1617                 if (list_empty(&bdev->man[0].lru[0]))
1618                         pr_debug("Swap list %d was clean\n", i);
1619         spin_unlock(&glob->lru_lock);
1620
1621         drm_vma_offset_manager_destroy(&bdev->vma_manager);
1622
1623         if (!ret)
1624                 ttm_bo_global_release();
1625
1626         return ret;
1627 }
1628 EXPORT_SYMBOL(ttm_bo_device_release);
1629
1630 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1631                        struct ttm_bo_driver *driver,
1632                        struct address_space *mapping,
1633                        uint64_t file_page_offset,
1634                        bool need_dma32)
1635 {
1636         struct ttm_bo_global *glob = &ttm_bo_glob;
1637         int ret;
1638
1639         ret = ttm_bo_global_init();
1640         if (ret)
1641                 return ret;
1642
1643         bdev->driver = driver;
1644
1645         memset(bdev->man, 0, sizeof(bdev->man));
1646
1647         /*
1648          * Initialize the system memory buffer type.
1649          * Other types need to be driver / IOCTL initialized.
1650          */
1651         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1652         if (unlikely(ret != 0))
1653                 goto out_no_sys;
1654
1655         drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
1656                                     0x10000000);
1657         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1658         INIT_LIST_HEAD(&bdev->ddestroy);
1659         bdev->dev_mapping = mapping;
1660         bdev->glob = glob;
1661         bdev->need_dma32 = need_dma32;
1662         mutex_lock(&ttm_global_mutex);
1663         list_add_tail(&bdev->device_list, &glob->device_list);
1664         mutex_unlock(&ttm_global_mutex);
1665
1666         return 0;
1667 out_no_sys:
1668         ttm_bo_global_release();
1669         return ret;
1670 }
1671 EXPORT_SYMBOL(ttm_bo_device_init);
1672
1673 /*
1674  * buffer object vm functions.
1675  */
1676
1677 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1678 {
1679         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1680
1681         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1682                 if (mem->mem_type == TTM_PL_SYSTEM)
1683                         return false;
1684
1685                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1686                         return false;
1687
1688                 if (mem->placement & TTM_PL_FLAG_CACHED)
1689                         return false;
1690         }
1691         return true;
1692 }
1693
1694 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1695 {
1696         struct ttm_bo_device *bdev = bo->bdev;
1697
1698         drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1699         ttm_mem_io_free_vm(bo);
1700 }
1701
1702 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1703 {
1704         struct ttm_bo_device *bdev = bo->bdev;
1705         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1706
1707         ttm_mem_io_lock(man, false);
1708         ttm_bo_unmap_virtual_locked(bo);
1709         ttm_mem_io_unlock(man);
1710 }
1711
1712
1713 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1714
1715 int ttm_bo_wait(struct ttm_buffer_object *bo,
1716                 bool interruptible, bool no_wait)
1717 {
1718         long timeout = 15 * HZ;
1719
1720         if (no_wait) {
1721                 if (reservation_object_test_signaled_rcu(bo->resv, true))
1722                         return 0;
1723                 else
1724                         return -EBUSY;
1725         }
1726
1727         timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
1728                                                       interruptible, timeout);
1729         if (timeout < 0)
1730                 return timeout;
1731
1732         if (timeout == 0)
1733                 return -EBUSY;
1734
1735         reservation_object_add_excl_fence(bo->resv, NULL);
1736         return 0;
1737 }
1738 EXPORT_SYMBOL(ttm_bo_wait);
1739
1740 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1741 {
1742         int ret = 0;
1743
1744         /*
1745          * Using ttm_bo_reserve makes sure the lru lists are updated.
1746          */
1747
1748         ret = ttm_bo_reserve(bo, true, no_wait, NULL);
1749         if (unlikely(ret != 0))
1750                 return ret;
1751         ret = ttm_bo_wait(bo, true, no_wait);
1752         if (likely(ret == 0))
1753                 atomic_inc(&bo->cpu_writers);
1754         ttm_bo_unreserve(bo);
1755         return ret;
1756 }
1757 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1758
1759 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1760 {
1761         atomic_dec(&bo->cpu_writers);
1762 }
1763 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1764
1765 /**
1766  * A buffer object shrink method that tries to swap out the first
1767  * buffer object on the bo_global::swap_lru list.
1768  */
1769 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1770 {
1771         struct ttm_buffer_object *bo;
1772         int ret = -EBUSY;
1773         bool locked;
1774         unsigned i;
1775
1776         spin_lock(&glob->lru_lock);
1777         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1778                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1779                         if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) {
1780                                 ret = 0;
1781                                 break;
1782                         }
1783                 }
1784                 if (!ret)
1785                         break;
1786         }
1787
1788         if (ret) {
1789                 spin_unlock(&glob->lru_lock);
1790                 return ret;
1791         }
1792
1793         kref_get(&bo->list_kref);
1794
1795         if (!list_empty(&bo->ddestroy)) {
1796                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1797                 kref_put(&bo->list_kref, ttm_bo_release_list);
1798                 return ret;
1799         }
1800
1801         ttm_bo_del_from_lru(bo);
1802         spin_unlock(&glob->lru_lock);
1803
1804         /**
1805          * Move to system cached
1806          */
1807
1808         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1809             bo->ttm->caching_state != tt_cached) {
1810                 struct ttm_operation_ctx ctx = { false, false };
1811                 struct ttm_mem_reg evict_mem;
1812
1813                 evict_mem = bo->mem;
1814                 evict_mem.mm_node = NULL;
1815                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1816                 evict_mem.mem_type = TTM_PL_SYSTEM;
1817
1818                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1819                 if (unlikely(ret != 0))
1820                         goto out;
1821         }
1822
1823         /**
1824          * Make sure BO is idle.
1825          */
1826
1827         ret = ttm_bo_wait(bo, false, false);
1828         if (unlikely(ret != 0))
1829                 goto out;
1830
1831         ttm_bo_unmap_virtual(bo);
1832
1833         /**
1834          * Swap out. Buffer will be swapped in again as soon as
1835          * anyone tries to access a ttm page.
1836          */
1837
1838         if (bo->bdev->driver->swap_notify)
1839                 bo->bdev->driver->swap_notify(bo);
1840
1841         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1842 out:
1843
1844         /**
1845          *
1846          * Unreserve without putting on LRU to avoid swapping out an
1847          * already swapped buffer.
1848          */
1849         if (locked)
1850                 reservation_object_unlock(bo->resv);
1851         kref_put(&bo->list_kref, ttm_bo_release_list);
1852         return ret;
1853 }
1854 EXPORT_SYMBOL(ttm_bo_swapout);
1855
1856 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1857 {
1858         struct ttm_operation_ctx ctx = {
1859                 .interruptible = false,
1860                 .no_wait_gpu = false
1861         };
1862
1863         while (ttm_bo_swapout(bdev->glob, &ctx) == 0)
1864                 ;
1865 }
1866 EXPORT_SYMBOL(ttm_bo_swapout_all);
1867
1868 /**
1869  * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
1870  * unreserved
1871  *
1872  * @bo: Pointer to buffer
1873  */
1874 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
1875 {
1876         int ret;
1877
1878         /*
1879          * In the absense of a wait_unlocked API,
1880          * Use the bo::wu_mutex to avoid triggering livelocks due to
1881          * concurrent use of this function. Note that this use of
1882          * bo::wu_mutex can go away if we change locking order to
1883          * mmap_sem -> bo::reserve.
1884          */
1885         ret = mutex_lock_interruptible(&bo->wu_mutex);
1886         if (unlikely(ret != 0))
1887                 return -ERESTARTSYS;
1888         if (!ww_mutex_is_locked(&bo->resv->lock))
1889                 goto out_unlock;
1890         ret = reservation_object_lock_interruptible(bo->resv, NULL);
1891         if (ret == -EINTR)
1892                 ret = -ERESTARTSYS;
1893         if (unlikely(ret != 0))
1894                 goto out_unlock;
1895         reservation_object_unlock(bo->resv);
1896
1897 out_unlock:
1898         mutex_unlock(&bo->wu_mutex);
1899         return ret;
1900 }