2 * Copyright 2016 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Christian König
25 #include <linux/dma-mapping.h>
26 #include <drm/ttm/ttm_range_manager.h>
29 #include "amdgpu_vm.h"
30 #include "amdgpu_res_cursor.h"
31 #include "amdgpu_atomfirmware.h"
34 struct amdgpu_vram_reservation {
35 struct list_head node;
36 struct drm_mm_node mm_node;
39 static inline struct amdgpu_vram_mgr *
40 to_vram_mgr(struct ttm_resource_manager *man)
42 return container_of(man, struct amdgpu_vram_mgr, manager);
45 static inline struct amdgpu_device *
46 to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
48 return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
52 * DOC: mem_info_vram_total
54 * The amdgpu driver provides a sysfs API for reporting current total VRAM
55 * available on the device
56 * The file mem_info_vram_total is used for this and returns the total
57 * amount of VRAM in bytes
59 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
60 struct device_attribute *attr, char *buf)
62 struct drm_device *ddev = dev_get_drvdata(dev);
63 struct amdgpu_device *adev = drm_to_adev(ddev);
65 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size);
69 * DOC: mem_info_vis_vram_total
71 * The amdgpu driver provides a sysfs API for reporting current total
72 * visible VRAM available on the device
73 * The file mem_info_vis_vram_total is used for this and returns the total
74 * amount of visible VRAM in bytes
76 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
77 struct device_attribute *attr, char *buf)
79 struct drm_device *ddev = dev_get_drvdata(dev);
80 struct amdgpu_device *adev = drm_to_adev(ddev);
82 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size);
86 * DOC: mem_info_vram_used
88 * The amdgpu driver provides a sysfs API for reporting current total VRAM
89 * available on the device
90 * The file mem_info_vram_used is used for this and returns the total
91 * amount of currently used VRAM in bytes
93 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
94 struct device_attribute *attr,
97 struct drm_device *ddev = dev_get_drvdata(dev);
98 struct amdgpu_device *adev = drm_to_adev(ddev);
100 return sysfs_emit(buf, "%llu\n",
101 amdgpu_vram_mgr_usage(&adev->mman.vram_mgr));
105 * DOC: mem_info_vis_vram_used
107 * The amdgpu driver provides a sysfs API for reporting current total of
109 * The file mem_info_vis_vram_used is used for this and returns the total
110 * amount of currently used visible VRAM in bytes
112 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
113 struct device_attribute *attr,
116 struct drm_device *ddev = dev_get_drvdata(dev);
117 struct amdgpu_device *adev = drm_to_adev(ddev);
119 return sysfs_emit(buf, "%llu\n",
120 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr));
124 * DOC: mem_info_vram_vendor
126 * The amdgpu driver provides a sysfs API for reporting the vendor of the
128 * The file mem_info_vram_vendor is used for this and returns the name of the
131 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
132 struct device_attribute *attr,
135 struct drm_device *ddev = dev_get_drvdata(dev);
136 struct amdgpu_device *adev = drm_to_adev(ddev);
138 switch (adev->gmc.vram_vendor) {
140 return sysfs_emit(buf, "samsung\n");
142 return sysfs_emit(buf, "infineon\n");
144 return sysfs_emit(buf, "elpida\n");
146 return sysfs_emit(buf, "etron\n");
148 return sysfs_emit(buf, "nanya\n");
150 return sysfs_emit(buf, "hynix\n");
152 return sysfs_emit(buf, "mosel\n");
154 return sysfs_emit(buf, "winbond\n");
156 return sysfs_emit(buf, "esmt\n");
158 return sysfs_emit(buf, "micron\n");
160 return sysfs_emit(buf, "unknown\n");
164 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
165 amdgpu_mem_info_vram_total_show, NULL);
166 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
167 amdgpu_mem_info_vis_vram_total_show,NULL);
168 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
169 amdgpu_mem_info_vram_used_show, NULL);
170 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
171 amdgpu_mem_info_vis_vram_used_show, NULL);
172 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
173 amdgpu_mem_info_vram_vendor, NULL);
175 static struct attribute *amdgpu_vram_mgr_attributes[] = {
176 &dev_attr_mem_info_vram_total.attr,
177 &dev_attr_mem_info_vis_vram_total.attr,
178 &dev_attr_mem_info_vram_used.attr,
179 &dev_attr_mem_info_vis_vram_used.attr,
180 &dev_attr_mem_info_vram_vendor.attr,
184 const struct attribute_group amdgpu_vram_mgr_attr_group = {
185 .attrs = amdgpu_vram_mgr_attributes
189 * amdgpu_vram_mgr_vis_size - Calculate visible node size
191 * @adev: amdgpu_device pointer
192 * @node: MM node structure
194 * Calculate how many bytes of the MM node are inside visible VRAM
196 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
197 struct drm_mm_node *node)
199 uint64_t start = node->start << PAGE_SHIFT;
200 uint64_t end = (node->size + node->start) << PAGE_SHIFT;
202 if (start >= adev->gmc.visible_vram_size)
205 return (end > adev->gmc.visible_vram_size ?
206 adev->gmc.visible_vram_size : end) - start;
210 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
212 * @bo: &amdgpu_bo buffer object (must be in VRAM)
215 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
217 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
219 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
220 struct ttm_resource *res = bo->tbo.resource;
221 unsigned pages = res->num_pages;
222 struct drm_mm_node *mm;
225 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
226 return amdgpu_bo_size(bo);
228 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
231 mm = &container_of(res, struct ttm_range_mgr_node, base)->mm_nodes[0];
232 for (usage = 0; pages; pages -= mm->size, mm++)
233 usage += amdgpu_vram_mgr_vis_size(adev, mm);
238 /* Commit the reservation of VRAM pages */
239 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
241 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
242 struct amdgpu_device *adev = to_amdgpu_device(mgr);
243 struct drm_mm *mm = &mgr->mm;
244 struct amdgpu_vram_reservation *rsv, *temp;
247 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node) {
248 if (drm_mm_reserve_node(mm, &rsv->mm_node))
251 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
252 rsv->mm_node.start, rsv->mm_node.size);
254 vis_usage = amdgpu_vram_mgr_vis_size(adev, &rsv->mm_node);
255 atomic64_add(vis_usage, &mgr->vis_usage);
256 atomic64_add(rsv->mm_node.size << PAGE_SHIFT, &mgr->usage);
257 list_move(&rsv->node, &mgr->reserved_pages);
262 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
264 * @mgr: amdgpu_vram_mgr pointer
265 * @start: start address of the range in VRAM
266 * @size: size of the range
268 * Reserve memory from start address with the specified size in VRAM
270 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
271 uint64_t start, uint64_t size)
273 struct amdgpu_vram_reservation *rsv;
275 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
279 INIT_LIST_HEAD(&rsv->node);
280 rsv->mm_node.start = start >> PAGE_SHIFT;
281 rsv->mm_node.size = size >> PAGE_SHIFT;
283 spin_lock(&mgr->lock);
284 list_add_tail(&rsv->node, &mgr->reservations_pending);
285 amdgpu_vram_mgr_do_reserve(&mgr->manager);
286 spin_unlock(&mgr->lock);
292 * amdgpu_vram_mgr_query_page_status - query the reservation status
294 * @mgr: amdgpu_vram_mgr pointer
295 * @start: start address of a page in VRAM
298 * -EBUSY: the page is still hold and in pending list
299 * 0: the page has been reserved
300 * -ENOENT: the input page is not a reservation
302 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
305 struct amdgpu_vram_reservation *rsv;
308 spin_lock(&mgr->lock);
310 list_for_each_entry(rsv, &mgr->reservations_pending, node) {
311 if ((rsv->mm_node.start <= start) &&
312 (start < (rsv->mm_node.start + rsv->mm_node.size))) {
318 list_for_each_entry(rsv, &mgr->reserved_pages, node) {
319 if ((rsv->mm_node.start <= start) &&
320 (start < (rsv->mm_node.start + rsv->mm_node.size))) {
328 spin_unlock(&mgr->lock);
333 * amdgpu_vram_mgr_virt_start - update virtual start address
335 * @mem: ttm_resource to update
336 * @node: just allocated node
338 * Calculate a virtual BO start address to easily check if everything is CPU
341 static void amdgpu_vram_mgr_virt_start(struct ttm_resource *mem,
342 struct drm_mm_node *node)
346 start = node->start + node->size;
347 if (start > mem->num_pages)
348 start -= mem->num_pages;
351 mem->start = max(mem->start, start);
355 * amdgpu_vram_mgr_new - allocate new ranges
357 * @man: TTM memory type manager
358 * @tbo: TTM BO we need this range for
359 * @place: placement flags and restrictions
360 * @res: the resulting mem object
362 * Allocate VRAM for the given BO.
364 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
365 struct ttm_buffer_object *tbo,
366 const struct ttm_place *place,
367 struct ttm_resource **res)
369 unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages;
370 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
371 struct amdgpu_device *adev = to_amdgpu_device(mgr);
372 uint64_t vis_usage = 0, mem_bytes, max_bytes;
373 struct ttm_range_mgr_node *node;
374 struct drm_mm *mm = &mgr->mm;
375 enum drm_mm_insert_mode mode;
383 max_bytes = adev->gmc.mc_vram_size;
384 if (tbo->type != ttm_bo_type_kernel)
385 max_bytes -= AMDGPU_VM_RESERVED_VRAM;
387 /* bail out quickly if there's likely not enough VRAM for this BO */
388 mem_bytes = tbo->base.size;
389 if (atomic64_add_return(mem_bytes, &mgr->usage) > max_bytes) {
394 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
395 pages_per_node = ~0ul;
398 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
399 pages_per_node = HPAGE_PMD_NR;
402 pages_per_node = 2UL << (20UL - PAGE_SHIFT);
404 pages_per_node = max_t(uint32_t, pages_per_node,
405 tbo->page_alignment);
406 num_nodes = DIV_ROUND_UP_ULL(PFN_UP(mem_bytes), pages_per_node);
409 node = kvmalloc(struct_size(node, mm_nodes, num_nodes),
410 GFP_KERNEL | __GFP_ZERO);
416 ttm_resource_init(tbo, place, &node->base);
418 mode = DRM_MM_INSERT_BEST;
419 if (place->flags & TTM_PL_FLAG_TOPDOWN)
420 mode = DRM_MM_INSERT_HIGH;
422 pages_left = node->base.num_pages;
424 /* Limit maximum size to 2GB due to SG table limitations */
425 pages = min(pages_left, 2UL << (30 - PAGE_SHIFT));
428 spin_lock(&mgr->lock);
430 uint32_t alignment = tbo->page_alignment;
432 if (pages >= pages_per_node)
433 alignment = pages_per_node;
435 r = drm_mm_insert_node_in_range(mm, &node->mm_nodes[i], pages,
436 alignment, 0, place->fpfn,
439 if (pages > pages_per_node) {
440 if (is_power_of_2(pages))
443 pages = rounddown_pow_of_two(pages);
449 vis_usage += amdgpu_vram_mgr_vis_size(adev, &node->mm_nodes[i]);
450 amdgpu_vram_mgr_virt_start(&node->base, &node->mm_nodes[i]);
454 if (pages > pages_left)
457 spin_unlock(&mgr->lock);
460 node->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
462 if (adev->gmc.xgmi.connected_to_cpu)
463 node->base.bus.caching = ttm_cached;
465 node->base.bus.caching = ttm_write_combined;
467 atomic64_add(vis_usage, &mgr->vis_usage);
473 drm_mm_remove_node(&node->mm_nodes[i]);
474 spin_unlock(&mgr->lock);
475 ttm_resource_fini(man, &node->base);
479 atomic64_sub(mem_bytes, &mgr->usage);
484 * amdgpu_vram_mgr_del - free ranges
486 * @man: TTM memory type manager
487 * @res: TTM memory object
489 * Free the allocated VRAM again.
491 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
492 struct ttm_resource *res)
494 struct ttm_range_mgr_node *node = to_ttm_range_mgr_node(res);
495 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
496 struct amdgpu_device *adev = to_amdgpu_device(mgr);
497 uint64_t usage = 0, vis_usage = 0;
500 spin_lock(&mgr->lock);
501 for (i = 0, pages = res->num_pages; pages;
502 pages -= node->mm_nodes[i].size, ++i) {
503 struct drm_mm_node *mm = &node->mm_nodes[i];
505 drm_mm_remove_node(mm);
506 usage += mm->size << PAGE_SHIFT;
507 vis_usage += amdgpu_vram_mgr_vis_size(adev, mm);
509 amdgpu_vram_mgr_do_reserve(man);
510 spin_unlock(&mgr->lock);
512 atomic64_sub(usage, &mgr->usage);
513 atomic64_sub(vis_usage, &mgr->vis_usage);
515 ttm_resource_fini(man, res);
520 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
522 * @adev: amdgpu device pointer
523 * @res: TTM memory object
524 * @offset: byte offset from the base of VRAM BO
525 * @length: number of bytes to export in sg_table
526 * @dev: the other device
527 * @dir: dma direction
528 * @sgt: resulting sg table
530 * Allocate and fill a sg table from a VRAM allocation.
532 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
533 struct ttm_resource *res,
534 u64 offset, u64 length,
536 enum dma_data_direction dir,
537 struct sg_table **sgt)
539 struct amdgpu_res_cursor cursor;
540 struct scatterlist *sg;
544 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
548 /* Determine the number of DRM_MM nodes to export */
549 amdgpu_res_first(res, offset, length, &cursor);
550 while (cursor.remaining) {
552 amdgpu_res_next(&cursor, cursor.size);
555 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
559 /* Initialize scatterlist nodes of sg_table */
560 for_each_sgtable_sg((*sgt), sg, i)
564 * Walk down DRM_MM nodes to populate scatterlist nodes
565 * @note: Use iterator api to get first the DRM_MM node
566 * and the number of bytes from it. Access the following
567 * DRM_MM node(s) if more buffer needs to exported
569 amdgpu_res_first(res, offset, length, &cursor);
570 for_each_sgtable_sg((*sgt), sg, i) {
571 phys_addr_t phys = cursor.start + adev->gmc.aper_base;
572 size_t size = cursor.size;
575 addr = dma_map_resource(dev, phys, size, dir,
576 DMA_ATTR_SKIP_CPU_SYNC);
577 r = dma_mapping_error(dev, addr);
581 sg_set_page(sg, NULL, size, 0);
582 sg_dma_address(sg) = addr;
583 sg_dma_len(sg) = size;
585 amdgpu_res_next(&cursor, cursor.size);
591 for_each_sgtable_sg((*sgt), sg, i) {
595 dma_unmap_resource(dev, sg->dma_address,
597 DMA_ATTR_SKIP_CPU_SYNC);
607 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
609 * @dev: device pointer
610 * @dir: data direction of resource to unmap
611 * @sgt: sg table to free
613 * Free a previously allocate sg table.
615 void amdgpu_vram_mgr_free_sgt(struct device *dev,
616 enum dma_data_direction dir,
617 struct sg_table *sgt)
619 struct scatterlist *sg;
622 for_each_sgtable_sg(sgt, sg, i)
623 dma_unmap_resource(dev, sg->dma_address,
625 DMA_ATTR_SKIP_CPU_SYNC);
631 * amdgpu_vram_mgr_usage - how many bytes are used in this domain
633 * @mgr: amdgpu_vram_mgr pointer
635 * Returns how many bytes are used in this domain.
637 uint64_t amdgpu_vram_mgr_usage(struct amdgpu_vram_mgr *mgr)
639 return atomic64_read(&mgr->usage);
643 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
645 * @mgr: amdgpu_vram_mgr pointer
647 * Returns how many bytes are used in the visible part of VRAM
649 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
651 return atomic64_read(&mgr->vis_usage);
655 * amdgpu_vram_mgr_debug - dump VRAM table
657 * @man: TTM memory type manager
658 * @printer: DRM printer to use
660 * Dump the table content using printk.
662 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
663 struct drm_printer *printer)
665 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
667 spin_lock(&mgr->lock);
668 drm_mm_print(&mgr->mm, printer);
669 spin_unlock(&mgr->lock);
671 drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
672 man->size, amdgpu_vram_mgr_usage(mgr) >> 20,
673 amdgpu_vram_mgr_vis_usage(mgr) >> 20);
676 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
677 .alloc = amdgpu_vram_mgr_new,
678 .free = amdgpu_vram_mgr_del,
679 .debug = amdgpu_vram_mgr_debug
683 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
685 * @adev: amdgpu_device pointer
687 * Allocate and initialize the VRAM manager.
689 int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
691 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
692 struct ttm_resource_manager *man = &mgr->manager;
694 ttm_resource_manager_init(man, &adev->mman.bdev,
695 adev->gmc.real_vram_size >> PAGE_SHIFT);
697 man->func = &amdgpu_vram_mgr_func;
699 drm_mm_init(&mgr->mm, 0, man->size);
700 spin_lock_init(&mgr->lock);
701 INIT_LIST_HEAD(&mgr->reservations_pending);
702 INIT_LIST_HEAD(&mgr->reserved_pages);
704 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
705 ttm_resource_manager_set_used(man, true);
710 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
712 * @adev: amdgpu_device pointer
714 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
715 * allocated inside it.
717 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
719 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
720 struct ttm_resource_manager *man = &mgr->manager;
722 struct amdgpu_vram_reservation *rsv, *temp;
724 ttm_resource_manager_set_used(man, false);
726 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
730 spin_lock(&mgr->lock);
731 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
734 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
735 drm_mm_remove_node(&rsv->mm_node);
738 drm_mm_takedown(&mgr->mm);
739 spin_unlock(&mgr->lock);
741 ttm_resource_manager_cleanup(man);
742 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);