drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26 #include "amdgpu.h"
  27 #include "amdgpu_vm.h"
  28 #include "amdgpu_atomfirmware.h"
  29 #include "atom.h"
  30
  31 static int amdgpu_vram_mgr_free_backup_pages(struct amdgpu_vram_mgr *mgr,
  32                                              uint32_t num_pages);
  33
  34 static inline struct amdgpu_vram_mgr *to_vram_mgr(struct ttm_resource_manager *man)
  35 {
  36         return container_of(man, struct amdgpu_vram_mgr, manager);
  37 }
  38
  39 static inline struct amdgpu_device *to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
  40 {
  41         return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
  42 }
  43
  44 /**
  45  * DOC: mem_info_vram_total
  46  *
  47  * The amdgpu driver provides a sysfs API for reporting current total VRAM
  48  * available on the device
  49  * The file mem_info_vram_total is used for this and returns the total
  50  * amount of VRAM in bytes
  51  */
  52 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
  53                 struct device_attribute *attr, char *buf)
  54 {
  55         struct drm_device *ddev = dev_get_drvdata(dev);
  56         struct amdgpu_device *adev = drm_to_adev(ddev);
  57
  58         return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.real_vram_size);
  59 }
  60
  61 /**
  62  * DOC: mem_info_vis_vram_total
  63  *
  64  * The amdgpu driver provides a sysfs API for reporting current total
  65  * visible VRAM available on the device
  66  * The file mem_info_vis_vram_total is used for this and returns the total
  67  * amount of visible VRAM in bytes
  68  */
  69 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
  70                 struct device_attribute *attr, char *buf)
  71 {
  72         struct drm_device *ddev = dev_get_drvdata(dev);
  73         struct amdgpu_device *adev = drm_to_adev(ddev);
  74
  75         return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.visible_vram_size);
  76 }
  77
  78 /**
  79  * DOC: mem_info_vram_used
  80  *
  81  * The amdgpu driver provides a sysfs API for reporting current total VRAM
  82  * available on the device
  83  * The file mem_info_vram_used is used for this and returns the total
  84  * amount of currently used VRAM in bytes
  85  */
  86 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
  87                 struct device_attribute *attr, char *buf)
  88 {
  89         struct drm_device *ddev = dev_get_drvdata(dev);
  90         struct amdgpu_device *adev = drm_to_adev(ddev);
  91         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
  92
  93         return snprintf(buf, PAGE_SIZE, "%llu\n",
  94                         amdgpu_vram_mgr_usage(man));
  95 }
  96
  97 /**
  98  * DOC: mem_info_vis_vram_used
  99  *
 100  * The amdgpu driver provides a sysfs API for reporting current total of
 101  * used visible VRAM
 102  * The file mem_info_vis_vram_used is used for this and returns the total
 103  * amount of currently used visible VRAM in bytes
 104  */
 105 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
 106                 struct device_attribute *attr, char *buf)
 107 {
 108         struct drm_device *ddev = dev_get_drvdata(dev);
 109         struct amdgpu_device *adev = drm_to_adev(ddev);
 110         struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
 111
 112         return snprintf(buf, PAGE_SIZE, "%llu\n",
 113                         amdgpu_vram_mgr_vis_usage(man));
 114 }
 115
 116 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
 117                                                  struct device_attribute *attr,
 118                                                  char *buf)
 119 {
 120         struct drm_device *ddev = dev_get_drvdata(dev);
 121         struct amdgpu_device *adev = drm_to_adev(ddev);
 122
 123         switch (adev->gmc.vram_vendor) {
 124         case SAMSUNG:
 125                 return snprintf(buf, PAGE_SIZE, "samsung\n");
 126         case INFINEON:
 127                 return snprintf(buf, PAGE_SIZE, "infineon\n");
 128         case ELPIDA:
 129                 return snprintf(buf, PAGE_SIZE, "elpida\n");
 130         case ETRON:
 131                 return snprintf(buf, PAGE_SIZE, "etron\n");
 132         case NANYA:
 133                 return snprintf(buf, PAGE_SIZE, "nanya\n");
 134         case HYNIX:
 135                 return snprintf(buf, PAGE_SIZE, "hynix\n");
 136         case MOSEL:
 137                 return snprintf(buf, PAGE_SIZE, "mosel\n");
 138         case WINBOND:
 139                 return snprintf(buf, PAGE_SIZE, "winbond\n");
 140         case ESMT:
 141                 return snprintf(buf, PAGE_SIZE, "esmt\n");
 142         case MICRON:
 143                 return snprintf(buf, PAGE_SIZE, "micron\n");
 144         default:
 145                 return snprintf(buf, PAGE_SIZE, "unknown\n");
 146         }
 147 }
 148
 149 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
 150                    amdgpu_mem_info_vram_total_show, NULL);
 151 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
 152                    amdgpu_mem_info_vis_vram_total_show,NULL);
 153 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
 154                    amdgpu_mem_info_vram_used_show, NULL);
 155 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
 156                    amdgpu_mem_info_vis_vram_used_show, NULL);
 157 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
 158                    amdgpu_mem_info_vram_vendor, NULL);
 159
 160 static const struct attribute *amdgpu_vram_mgr_attributes[] = {
 161         &dev_attr_mem_info_vram_total.attr,
 162         &dev_attr_mem_info_vis_vram_total.attr,
 163         &dev_attr_mem_info_vram_used.attr,
 164         &dev_attr_mem_info_vis_vram_used.attr,
 165         &dev_attr_mem_info_vram_vendor.attr,
 166         NULL
 167 };
 168
 169 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func;
 170
 171 /**
 172  * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
 173  *
 174  * @adev: amdgpu_device pointer
 175  *
 176  * Allocate and initialize the VRAM manager.
 177  */
 178 int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
 179 {
 180         struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
 181         struct ttm_resource_manager *man = &mgr->manager;
 182         int ret;
 183
 184         ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT);
 185
 186         man->func = &amdgpu_vram_mgr_func;
 187
 188         drm_mm_init(&mgr->mm, 0, man->size);
 189         spin_lock_init(&mgr->lock);
 190         INIT_LIST_HEAD(&mgr->reservations_pending);
 191         INIT_LIST_HEAD(&mgr->reserved_pages);
 192         INIT_LIST_HEAD(&mgr->backup_pages);
 193
 194         /* Add the two VRAM-related sysfs files */
 195         ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
 196         if (ret)
 197                 DRM_ERROR("Failed to register sysfs\n");
 198
 199         ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
 200         ttm_resource_manager_set_used(man, true);
 201         return 0;
 202 }
 203
 204 /**
 205  * amdgpu_vram_mgr_fini - free and destroy VRAM manager
 206  *
 207  * @adev: amdgpu_device pointer
 208  *
 209  * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
 210  * allocated inside it.
 211  */
 212 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
 213 {
 214         struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
 215         struct ttm_resource_manager *man = &mgr->manager;
 216         int ret;
 217         struct amdgpu_vram_reservation *rsv, *temp;
 218
 219         ttm_resource_manager_set_used(man, false);
 220
 221         ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
 222         if (ret)
 223                 return;
 224
 225         spin_lock(&mgr->lock);
 226         list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
 227                 kfree(rsv);
 228
 229         list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
 230                 drm_mm_remove_node(&rsv->mm_node);
 231                 kfree(rsv);
 232         }
 233
 234         list_for_each_entry_safe(rsv, temp, &mgr->backup_pages, node) {
 235                 drm_mm_remove_node(&rsv->mm_node);
 236                 kfree(rsv);
 237         }
 238         drm_mm_takedown(&mgr->mm);
 239         spin_unlock(&mgr->lock);
 240
 241         sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
 242
 243         ttm_resource_manager_cleanup(man);
 244         ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
 245 }
 246
 247 /**
 248  * amdgpu_vram_mgr_vis_size - Calculate visible node size
 249  *
 250  * @adev: amdgpu_device pointer
 251  * @node: MM node structure
 252  *
 253  * Calculate how many bytes of the MM node are inside visible VRAM
 254  */
 255 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
 256                                     struct drm_mm_node *node)
 257 {
 258         uint64_t start = node->start << PAGE_SHIFT;
 259         uint64_t end = (node->size + node->start) << PAGE_SHIFT;
 260
 261         if (start >= adev->gmc.visible_vram_size)
 262                 return 0;
 263
 264         return (end > adev->gmc.visible_vram_size ?
 265                 adev->gmc.visible_vram_size : end) - start;
 266 }
 267
 268 /**
 269  * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
 270  *
 271  * @bo: &amdgpu_bo buffer object (must be in VRAM)
 272  *
 273  * Returns:
 274  * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
 275  */
 276 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
 277 {
 278         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 279         struct ttm_resource *mem = &bo->tbo.mem;
 280         struct drm_mm_node *nodes = mem->mm_node;
 281         unsigned pages = mem->num_pages;
 282         u64 usage;
 283
 284         if (amdgpu_gmc_vram_full_visible(&adev->gmc))
 285                 return amdgpu_bo_size(bo);
 286
 287         if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
 288                 return 0;
 289
 290         for (usage = 0; nodes && pages; pages -= nodes->size, nodes++)
 291                 usage += amdgpu_vram_mgr_vis_size(adev, nodes);
 292
 293         return usage;
 294 }
 295
 296 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
 297 {
 298         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 299         struct amdgpu_device *adev = to_amdgpu_device(mgr);
 300         struct drm_mm *mm = &mgr->mm;
 301         struct amdgpu_vram_reservation *rsv, *temp;
 302         uint64_t vis_usage;
 303
 304         list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) {
 305                 if (drm_mm_reserve_node(mm, &rsv->mm_node))
 306                         continue;
 307
 308                 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
 309                         rsv->mm_node.start << PAGE_SHIFT, rsv->mm_node.size);
 310
 311                 vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node);
 312                 atomic64_add(vis_usage, &mgr->vis_usage);
 313                 atomic64_add(rsv->mm_node.size << PAGE_SHIFT, &mgr->usage);
 314                 list_move(&rsv->node, &mgr->reserved_pages);
 315
 316                 amdgpu_vram_mgr_free_backup_pages(mgr, rsv->mm_node.size);
 317         }
 318 }
 319
 320 /**
 321  * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
 322  *
 323  * @man: TTM memory type manager
 324  * @start: start address of the range in VRAM
 325  * @size: size of the range
 326  *
 327  * Reserve memory from start addess with the specified size in VRAM
 328  */
 329 int amdgpu_vram_mgr_reserve_range(struct ttm_resource_manager *man,
 330                                   uint64_t start, uint64_t size)
 331 {
 332         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 333         struct amdgpu_device *adev = to_amdgpu_device(mgr);
 334         struct amdgpu_vram_reservation *rsv;
 335
 336         rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
 337         if (!rsv)
 338                 return -ENOMEM;
 339
 340         INIT_LIST_HEAD(&rsv->node);
 341         rsv->mm_node.start = start >> PAGE_SHIFT;
 342         rsv->mm_node.size = size >> PAGE_SHIFT;
 343
 344         dev_dbg(adev->dev, "Pending Reservation: 0x%llx\n", start);
 345
 346         spin_lock(&mgr->lock);
 347         list_add_tail(&rsv->node, &mgr->reservations_pending);
 348         amdgpu_vram_mgr_do_reserve(man);
 349         spin_unlock(&mgr->lock);
 350
 351         return 0;
 352 }
 353
 354 static int amdgpu_vram_mgr_free_backup_pages(struct amdgpu_vram_mgr *mgr,
 355                                              uint32_t num_pages)
 356 {
 357         struct amdgpu_device *adev = to_amdgpu_device(mgr);
 358         struct amdgpu_vram_reservation *rsv;
 359         uint32_t i;
 360         uint64_t vis_usage = 0, total_usage = 0;
 361
 362         if (num_pages > mgr->num_backup_pages) {
 363                 dev_warn(adev->dev, "No enough backup pages\n");
 364                 return -EINVAL;
 365         }
 366
 367         for (i = 0; i < num_pages; i++) {
 368                 rsv = list_first_entry(&mgr->backup_pages,
 369                                        struct amdgpu_vram_reservation, node);
 370                 vis_usage += amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node);
 371                 total_usage += (rsv->mm_node.size << PAGE_SHIFT);
 372                 drm_mm_remove_node(&rsv->mm_node);
 373                 list_del(&rsv->node);
 374                 kfree(rsv);
 375                 mgr->num_backup_pages--;
 376         }
 377
 378         atomic64_sub(total_usage, &mgr->usage);
 379         atomic64_sub(vis_usage, &mgr->vis_usage);
 380
 381         return 0;
 382 }
 383
 384 int amdgpu_vram_mgr_reserve_backup_pages(struct ttm_resource_manager *man,
 385                                          uint32_t num_pages)
 386 {
 387         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 388         struct amdgpu_device *adev = to_amdgpu_device(mgr);
 389         struct amdgpu_vram_reservation *rsv;
 390         struct drm_mm *mm = &mgr->mm;
 391         uint32_t i;
 392         int ret = 0;
 393         uint64_t vis_usage, total_usage;
 394
 395         for (i = 0; i < num_pages; i++) {
 396                 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
 397                 if (!rsv) {
 398                         ret = -ENOMEM;
 399                         goto pro_end;
 400                 }
 401
 402                 INIT_LIST_HEAD(&rsv->node);
 403
 404                 ret = drm_mm_insert_node(mm, &rsv->mm_node, 1);
 405                 if (ret) {
 406                         dev_err(adev->dev, "failed to reserve backup page %d, ret 0x%x\n", i, ret);
 407                         kfree(rsv);
 408                         goto pro_end;
 409                 }
 410
 411                 vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node);
 412                 total_usage = (rsv->mm_node.size << PAGE_SHIFT);
 413
 414                 spin_lock(&mgr->lock);
 415                 atomic64_add(vis_usage, &mgr->vis_usage);
 416                 atomic64_add(total_usage, &mgr->usage);
 417                 list_add_tail(&rsv->node, &mgr->backup_pages);
 418                 mgr->num_backup_pages++;
 419                 spin_unlock(&mgr->lock);
 420         }
 421
 422 pro_end:
 423         if (ret) {
 424                 spin_lock(&mgr->lock);
 425                 amdgpu_vram_mgr_free_backup_pages(mgr, mgr->num_backup_pages);
 426                 spin_unlock(&mgr->lock);
 427         }
 428
 429         return ret;
 430 }
 431
 432 /**
 433  * amdgpu_vram_mgr_query_page_status - query the reservation status
 434  *
 435  * @man: TTM memory type manager
 436  * @start: start address of a page in VRAM
 437  *
 438  * Returns:
 439  *      -EBUSY: the page is still hold and in pending list
 440  *      0: the page has been reserved
 441  *      -ENOENT: the input page is not a reservation
 442  */
 443 int amdgpu_vram_mgr_query_page_status(struct ttm_resource_manager *man,
 444                                       uint64_t start)
 445 {
 446         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 447         struct amdgpu_vram_reservation *rsv;
 448         int ret;
 449
 450         spin_lock(&mgr->lock);
 451
 452         list_for_each_entry(rsv, &mgr->reservations_pending, node) {
 453                 if ((rsv->mm_node.start <= start) &&
 454                     (start < (rsv->mm_node.start + rsv->mm_node.size))) {
 455                         ret = -EBUSY;
 456                         goto out;
 457                 }
 458         }
 459
 460         list_for_each_entry(rsv, &mgr->reserved_pages, node) {
 461                 if ((rsv->mm_node.start <= start) &&
 462                     (start < (rsv->mm_node.start + rsv->mm_node.size))) {
 463                         ret = 0;
 464                         goto out;
 465                 }
 466         }
 467
 468         ret = -ENOENT;
 469 out:
 470         spin_unlock(&mgr->lock);
 471         return ret;
 472 }
 473
 474 /**
 475  * amdgpu_vram_mgr_virt_start - update virtual start address
 476  *
 477  * @mem: ttm_resource to update
 478  * @node: just allocated node
 479  *
 480  * Calculate a virtual BO start address to easily check if everything is CPU
 481  * accessible.
 482  */
 483 static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem,
 484                                        struct drm_mm_node *node)
 485 {
 486         unsigned long start;
 487
 488         start = node->start + node->size;
 489         if (start > mem->num_pages)
 490                 start -= mem->num_pages;
 491         else
 492                 start = 0;
 493         mem->start = max(mem->start, start);
 494 }
 495
 496 /**
 497  * amdgpu_vram_mgr_new - allocate new ranges
 498  *
 499  * @man: TTM memory type manager
 500  * @tbo: TTM BO we need this range for
 501  * @place: placement flags and restrictions
 502  * @mem: the resulting mem object
 503  *
 504  * Allocate VRAM for the given BO.
 505  */
 506 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
 507                                struct ttm_buffer_object *tbo,
 508                                const struct ttm_place *place,
 509                                struct ttm_resource *mem)
 510 {
 511         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 512         struct amdgpu_device *adev = to_amdgpu_device(mgr);
 513         struct drm_mm *mm = &mgr->mm;
 514         struct drm_mm_node *nodes;
 515         enum drm_mm_insert_mode mode;
 516         unsigned long lpfn, num_nodes, pages_per_node, pages_left;
 517         uint64_t vis_usage = 0, mem_bytes, max_bytes;
 518         unsigned i;
 519         int r;
 520
 521         lpfn = place->lpfn;
 522         if (!lpfn)
 523                 lpfn = man->size;
 524
 525         max_bytes = adev->gmc.mc_vram_size;
 526         if (tbo->type != ttm_bo_type_kernel)
 527                 max_bytes -= AMDGPU_VM_RESERVED_VRAM;
 528
 529         /* bail out quickly if there's likely not enough VRAM for this BO */
 530         mem_bytes = (u64)mem->num_pages << PAGE_SHIFT;
 531         if (atomic64_add_return(mem_bytes, &mgr->usage) > max_bytes) {
 532                 atomic64_sub(mem_bytes, &mgr->usage);
 533                 return -ENOSPC;
 534         }
 535
 536         if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
 537                 pages_per_node = ~0ul;
 538                 num_nodes = 1;
 539         } else {
 540 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 541                 pages_per_node = HPAGE_PMD_NR;
 542 #else
 543                 /* default to 2MB */
 544                 pages_per_node = (2UL << (20UL - PAGE_SHIFT));
 545 #endif
 546                 pages_per_node = max((uint32_t)pages_per_node, mem->page_alignment);
 547                 num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
 548         }
 549
 550         nodes = kvmalloc_array((uint32_t)num_nodes, sizeof(*nodes),
 551                                GFP_KERNEL | __GFP_ZERO);
 552         if (!nodes) {
 553                 atomic64_sub(mem_bytes, &mgr->usage);
 554                 return -ENOMEM;
 555         }
 556
 557         mode = DRM_MM_INSERT_BEST;
 558         if (place->flags & TTM_PL_FLAG_TOPDOWN)
 559                 mode = DRM_MM_INSERT_HIGH;
 560
 561         mem->start = 0;
 562         pages_left = mem->num_pages;
 563
 564         spin_lock(&mgr->lock);
 565         for (i = 0; pages_left >= pages_per_node; ++i) {
 566                 unsigned long pages = rounddown_pow_of_two(pages_left);
 567
 568                 /* Limit maximum size to 2GB due to SG table limitations */
 569                 pages = min(pages, (2UL << (30 - PAGE_SHIFT)));
 570
 571                 r = drm_mm_insert_node_in_range(mm, &nodes[i], pages,
 572                                                 pages_per_node, 0,
 573                                                 place->fpfn, lpfn,
 574                                                 mode);
 575                 if (unlikely(r))
 576                         break;
 577
 578                 vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
 579                 amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
 580                 pages_left -= pages;
 581         }
 582
 583         for (; pages_left; ++i) {
 584                 unsigned long pages = min(pages_left, pages_per_node);
 585                 uint32_t alignment = mem->page_alignment;
 586
 587                 if (pages == pages_per_node)
 588                         alignment = pages_per_node;
 589
 590                 r = drm_mm_insert_node_in_range(mm, &nodes[i],
 591                                                 pages, alignment, 0,
 592                                                 place->fpfn, lpfn,
 593                                                 mode);
 594                 if (unlikely(r))
 595                         goto error;
 596
 597                 vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
 598                 amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
 599                 pages_left -= pages;
 600         }
 601         spin_unlock(&mgr->lock);
 602
 603         atomic64_add(vis_usage, &mgr->vis_usage);
 604
 605         mem->mm_node = nodes;
 606
 607         return 0;
 608
 609 error:
 610         while (i--)
 611                 drm_mm_remove_node(&nodes[i]);
 612         spin_unlock(&mgr->lock);
 613         atomic64_sub(mem->num_pages << PAGE_SHIFT, &mgr->usage);
 614
 615         kvfree(nodes);
 616         return r;
 617 }
 618
 619 /**
 620  * amdgpu_vram_mgr_del - free ranges
 621  *
 622  * @man: TTM memory type manager
 623  * @mem: TTM memory object
 624  *
 625  * Free the allocated VRAM again.
 626  */
 627 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
 628                                 struct ttm_resource *mem)
 629 {
 630         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 631         struct amdgpu_device *adev = to_amdgpu_device(mgr);
 632         struct drm_mm_node *nodes = mem->mm_node;
 633         uint64_t usage = 0, vis_usage = 0;
 634         unsigned pages = mem->num_pages;
 635
 636         if (!mem->mm_node)
 637                 return;
 638
 639         spin_lock(&mgr->lock);
 640         while (pages) {
 641                 pages -= nodes->size;
 642                 drm_mm_remove_node(nodes);
 643                 usage += nodes->size << PAGE_SHIFT;
 644                 vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
 645                 ++nodes;
 646         }
 647         amdgpu_vram_mgr_do_reserve(man);
 648         spin_unlock(&mgr->lock);
 649
 650         atomic64_sub(usage, &mgr->usage);
 651         atomic64_sub(vis_usage, &mgr->vis_usage);
 652
 653         kvfree(mem->mm_node);
 654         mem->mm_node = NULL;
 655 }
 656
 657 /**
 658  * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
 659  *
 660  * @adev: amdgpu device pointer
 661  * @mem: TTM memory object
 662  * @dev: the other device
 663  * @dir: dma direction
 664  * @sgt: resulting sg table
 665  *
 666  * Allocate and fill a sg table from a VRAM allocation.
 667  */
 668 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
 669                               struct ttm_resource *mem,
 670                               struct device *dev,
 671                               enum dma_data_direction dir,
 672                               struct sg_table **sgt)
 673 {
 674         struct drm_mm_node *node;
 675         struct scatterlist *sg;
 676         int num_entries = 0;
 677         unsigned int pages;
 678         int i, r;
 679
 680         *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
 681         if (!*sgt)
 682                 return -ENOMEM;
 683
 684         for (pages = mem->num_pages, node = mem->mm_node;
 685              pages; pages -= node->size, ++node)
 686                 ++num_entries;
 687
 688         r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
 689         if (r)
 690                 goto error_free;
 691
 692         for_each_sgtable_sg((*sgt), sg, i)
 693                 sg->length = 0;
 694
 695         node = mem->mm_node;
 696         for_each_sgtable_sg((*sgt), sg, i) {
 697                 phys_addr_t phys = (node->start << PAGE_SHIFT) +
 698                         adev->gmc.aper_base;
 699                 size_t size = node->size << PAGE_SHIFT;
 700                 dma_addr_t addr;
 701
 702                 ++node;
 703                 addr = dma_map_resource(dev, phys, size, dir,
 704                                         DMA_ATTR_SKIP_CPU_SYNC);
 705                 r = dma_mapping_error(dev, addr);
 706                 if (r)
 707                         goto error_unmap;
 708
 709                 sg_set_page(sg, NULL, size, 0);
 710                 sg_dma_address(sg) = addr;
 711                 sg_dma_len(sg) = size;
 712         }
 713         return 0;
 714
 715 error_unmap:
 716         for_each_sgtable_sg((*sgt), sg, i) {
 717                 if (!sg->length)
 718                         continue;
 719
 720                 dma_unmap_resource(dev, sg->dma_address,
 721                                    sg->length, dir,
 722                                    DMA_ATTR_SKIP_CPU_SYNC);
 723         }
 724         sg_free_table(*sgt);
 725
 726 error_free:
 727         kfree(*sgt);
 728         return r;
 729 }
 730
 731 /**
 732  * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
 733  *
 734  * @dev: device pointer
 735  * @dir: data direction of resource to unmap
 736  * @sgt: sg table to free
 737  *
 738  * Free a previously allocate sg table.
 739  */
 740 void amdgpu_vram_mgr_free_sgt(struct device *dev,
 741                               enum dma_data_direction dir,
 742                               struct sg_table *sgt)
 743 {
 744         struct scatterlist *sg;
 745         int i;
 746
 747         for_each_sgtable_sg(sgt, sg, i)
 748                 dma_unmap_resource(dev, sg->dma_address,
 749                                    sg->length, dir,
 750                                    DMA_ATTR_SKIP_CPU_SYNC);
 751         sg_free_table(sgt);
 752         kfree(sgt);
 753 }
 754
 755 /**
 756  * amdgpu_vram_mgr_usage - how many bytes are used in this domain
 757  *
 758  * @man: TTM memory type manager
 759  *
 760  * Returns how many bytes are used in this domain.
 761  */
 762 uint64_t amdgpu_vram_mgr_usage(struct ttm_resource_manager *man)
 763 {
 764         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 765
 766         return atomic64_read(&mgr->usage);
 767 }
 768
 769 /**
 770  * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
 771  *
 772  * @man: TTM memory type manager
 773  *
 774  * Returns how many bytes are used in the visible part of VRAM
 775  */
 776 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_resource_manager *man)
 777 {
 778         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 779
 780         return atomic64_read(&mgr->vis_usage);
 781 }
 782
 783 /**
 784  * amdgpu_vram_mgr_debug - dump VRAM table
 785  *
 786  * @man: TTM memory type manager
 787  * @printer: DRM printer to use
 788  *
 789  * Dump the table content using printk.
 790  */
 791 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
 792                                   struct drm_printer *printer)
 793 {
 794         struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
 795
 796         spin_lock(&mgr->lock);
 797         drm_mm_print(&mgr->mm, printer);
 798         spin_unlock(&mgr->lock);
 799
 800         drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
 801                    man->size, amdgpu_vram_mgr_usage(man) >> 20,
 802                    amdgpu_vram_mgr_vis_usage(man) >> 20);
 803 }
 804
 805 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
 806         .alloc  = amdgpu_vram_mgr_new,
 807         .free   = amdgpu_vram_mgr_del,
 808         .debug  = amdgpu_vram_mgr_debug
 809 };