1 // SPDX-License-Identifier: MIT
3 * Copyright © 2021 Intel Corporation
6 #include <linux/slab.h>
8 #include <drm/ttm/ttm_placement.h>
9 #include <drm/ttm/ttm_bo.h>
11 #include <drm/drm_buddy.h>
13 #include "i915_ttm_buddy_manager.h"
17 struct i915_ttm_buddy_manager {
18 struct ttm_resource_manager manager;
20 struct list_head reserved;
22 unsigned long visible_size;
23 unsigned long visible_avail;
24 unsigned long visible_reserved;
25 u64 default_page_size;
28 static struct i915_ttm_buddy_manager *
29 to_buddy_manager(struct ttm_resource_manager *man)
31 return container_of(man, struct i915_ttm_buddy_manager, manager);
34 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
35 struct ttm_buffer_object *bo,
36 const struct ttm_place *place,
37 struct ttm_resource **res)
39 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
40 struct i915_ttm_buddy_resource *bman_res;
41 struct drm_buddy *mm = &bman->mm;
42 unsigned long n_pages, lpfn;
51 bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL);
55 ttm_resource_init(bo, place, &bman_res->base);
56 INIT_LIST_HEAD(&bman_res->blocks);
59 if (place->flags & TTM_PL_FLAG_TOPDOWN)
60 bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
62 if (place->fpfn || lpfn != man->size)
63 bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
65 GEM_BUG_ON(!bman_res->base.size);
66 size = bman_res->base.size;
68 min_page_size = bman->default_page_size;
69 if (bo->page_alignment)
70 min_page_size = bo->page_alignment << PAGE_SHIFT;
72 GEM_BUG_ON(min_page_size < mm->chunk_size);
73 GEM_BUG_ON(!IS_ALIGNED(size, min_page_size));
75 if (place->fpfn + PFN_UP(bman_res->base.size) != place->lpfn &&
76 place->flags & TTM_PL_FLAG_CONTIGUOUS) {
79 size = roundup_pow_of_two(size);
82 pages = size >> ilog2(mm->chunk_size);
87 if (size > lpfn << PAGE_SHIFT) {
92 n_pages = size >> ilog2(mm->chunk_size);
94 mutex_lock(&bman->lock);
95 if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) {
96 mutex_unlock(&bman->lock);
101 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
102 (u64)lpfn << PAGE_SHIFT,
103 (u64)n_pages << PAGE_SHIFT,
108 goto err_free_blocks;
110 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
111 u64 original_size = (u64)bman_res->base.size;
113 drm_buddy_block_trim(mm,
118 if (lpfn <= bman->visible_size) {
119 bman_res->used_visible_size = PFN_UP(bman_res->base.size);
121 struct drm_buddy_block *block;
123 list_for_each_entry(block, &bman_res->blocks, link) {
124 unsigned long start =
125 drm_buddy_block_offset(block) >> PAGE_SHIFT;
127 if (start < bman->visible_size) {
128 unsigned long end = start +
129 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
131 bman_res->used_visible_size +=
132 min(end, bman->visible_size) - start;
137 if (bman_res->used_visible_size)
138 bman->visible_avail -= bman_res->used_visible_size;
140 mutex_unlock(&bman->lock);
142 *res = &bman_res->base;
146 drm_buddy_free_list(mm, &bman_res->blocks);
147 mutex_unlock(&bman->lock);
149 ttm_resource_fini(man, &bman_res->base);
154 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
155 struct ttm_resource *res)
157 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
158 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
160 mutex_lock(&bman->lock);
161 drm_buddy_free_list(&bman->mm, &bman_res->blocks);
162 bman->visible_avail += bman_res->used_visible_size;
163 mutex_unlock(&bman->lock);
165 ttm_resource_fini(man, res);
169 static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
170 struct ttm_resource *res,
171 const struct ttm_place *place,
174 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
175 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
176 struct drm_buddy *mm = &bman->mm;
177 struct drm_buddy_block *block;
179 if (!place->fpfn && !place->lpfn)
182 GEM_BUG_ON(!place->lpfn);
185 * If we just want something mappable then we can quickly check
186 * if the current victim resource is using any of the CPU
190 place->lpfn == i915_ttm_buddy_man_visible_size(man))
191 return bman_res->used_visible_size > 0;
193 /* Check each drm buddy block individually */
194 list_for_each_entry(block, &bman_res->blocks, link) {
196 drm_buddy_block_offset(block) >> PAGE_SHIFT;
197 unsigned long lpfn = fpfn +
198 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
200 if (place->fpfn < lpfn && place->lpfn > fpfn)
207 static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
208 struct ttm_resource *res,
209 const struct ttm_place *place,
212 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
213 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
214 struct drm_buddy *mm = &bman->mm;
215 struct drm_buddy_block *block;
217 if (!place->fpfn && !place->lpfn)
220 GEM_BUG_ON(!place->lpfn);
223 place->lpfn == i915_ttm_buddy_man_visible_size(man))
224 return bman_res->used_visible_size == PFN_UP(res->size);
226 /* Check each drm buddy block individually */
227 list_for_each_entry(block, &bman_res->blocks, link) {
229 drm_buddy_block_offset(block) >> PAGE_SHIFT;
230 unsigned long lpfn = fpfn +
231 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
233 if (fpfn < place->fpfn || lpfn > place->lpfn)
240 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
241 struct drm_printer *printer)
243 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
244 struct drm_buddy_block *block;
246 mutex_lock(&bman->lock);
247 drm_printf(printer, "default_page_size: %lluKiB\n",
248 bman->default_page_size >> 10);
249 drm_printf(printer, "visible_avail: %lluMiB\n",
250 (u64)bman->visible_avail << PAGE_SHIFT >> 20);
251 drm_printf(printer, "visible_size: %lluMiB\n",
252 (u64)bman->visible_size << PAGE_SHIFT >> 20);
253 drm_printf(printer, "visible_reserved: %lluMiB\n",
254 (u64)bman->visible_reserved << PAGE_SHIFT >> 20);
256 drm_buddy_print(&bman->mm, printer);
258 drm_printf(printer, "reserved:\n");
259 list_for_each_entry(block, &bman->reserved, link)
260 drm_buddy_block_print(&bman->mm, block, printer);
261 mutex_unlock(&bman->lock);
264 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = {
265 .alloc = i915_ttm_buddy_man_alloc,
266 .free = i915_ttm_buddy_man_free,
267 .intersects = i915_ttm_buddy_man_intersects,
268 .compatible = i915_ttm_buddy_man_compatible,
269 .debug = i915_ttm_buddy_man_debug,
273 * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager
274 * @bdev: The ttm device
275 * @type: Memory type we want to manage
276 * @use_tt: Set use_tt for the manager
277 * @size: The size in bytes to manage
278 * @visible_size: The CPU visible size in bytes to manage
279 * @default_page_size: The default minimum page size in bytes for allocations,
280 * this must be at least as large as @chunk_size, and can be overridden by
281 * setting the BO page_alignment, to be larger or smaller as needed.
282 * @chunk_size: The minimum page size in bytes for our allocations i.e
285 * Note that the starting address is assumed to be zero here, since this
286 * simplifies keeping the property where allocated blocks having natural
287 * power-of-two alignment. So long as the real starting address is some large
288 * power-of-two, or naturally start from zero, then this should be fine. Also
289 * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment
290 * if say there is some unusable range from the start of the region. We can
291 * revisit this in the future and make the interface accept an actual starting
292 * offset and let it take care of the rest.
294 * Note that if the @size is not aligned to the @chunk_size then we perform the
295 * required rounding to get the usable size. The final size in pages can be
296 * taken from &ttm_resource_manager.size.
298 * Return: 0 on success, negative error code on failure.
300 int i915_ttm_buddy_man_init(struct ttm_device *bdev,
301 unsigned int type, bool use_tt,
302 u64 size, u64 visible_size, u64 default_page_size,
305 struct ttm_resource_manager *man;
306 struct i915_ttm_buddy_manager *bman;
309 bman = kzalloc(sizeof(*bman), GFP_KERNEL);
313 err = drm_buddy_init(&bman->mm, size, chunk_size);
317 mutex_init(&bman->lock);
318 INIT_LIST_HEAD(&bman->reserved);
319 GEM_BUG_ON(default_page_size < chunk_size);
320 bman->default_page_size = default_page_size;
321 bman->visible_size = visible_size >> PAGE_SHIFT;
322 bman->visible_avail = bman->visible_size;
324 man = &bman->manager;
325 man->use_tt = use_tt;
326 man->func = &i915_ttm_buddy_manager_func;
327 ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT);
329 ttm_resource_manager_set_used(man, true);
330 ttm_set_driver_manager(bdev, type, man);
340 * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager
341 * @bdev: The ttm device
342 * @type: Memory type we want to manage
344 * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will
345 * also be freed for us here.
347 * Return: 0 on success, negative error code on failure.
349 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
351 struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
352 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
353 struct drm_buddy *mm = &bman->mm;
356 ttm_resource_manager_set_used(man, false);
358 ret = ttm_resource_manager_evict_all(bdev, man);
362 ttm_set_driver_manager(bdev, type, NULL);
364 mutex_lock(&bman->lock);
365 drm_buddy_free_list(mm, &bman->reserved);
367 bman->visible_avail += bman->visible_reserved;
368 WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
369 mutex_unlock(&bman->lock);
371 ttm_resource_manager_cleanup(man);
378 * i915_ttm_buddy_man_reserve - Reserve address range
379 * @man: The buddy allocator ttm manager
380 * @start: The offset in bytes, where the region start is assumed to be zero
381 * @size: The size in bytes
383 * Note that the starting address for the region is always assumed to be zero.
385 * Return: 0 on success, negative error code on failure.
387 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
390 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
391 struct drm_buddy *mm = &bman->mm;
392 unsigned long fpfn = start >> PAGE_SHIFT;
393 unsigned long flags = 0;
396 flags |= DRM_BUDDY_RANGE_ALLOCATION;
398 mutex_lock(&bman->lock);
399 ret = drm_buddy_alloc_blocks(mm, start,
401 size, mm->chunk_size,
405 if (fpfn < bman->visible_size) {
406 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT);
407 unsigned long visible = min(lpfn, bman->visible_size) - fpfn;
409 bman->visible_reserved += visible;
410 bman->visible_avail -= visible;
412 mutex_unlock(&bman->lock);
418 * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion
420 * @man: The buddy allocator ttm manager
422 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man)
424 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
426 return bman->visible_size;
430 * i915_ttm_buddy_man_avail - Query the avail tracking for the manager.
432 * @man: The buddy allocator ttm manager
433 * @avail: The total available memory in pages for the entire manager.
434 * @visible_avail: The total available memory in pages for the CPU visible
435 * portion. Note that this will always give the same value as @avail on
436 * configurations that don't have a small BAR.
438 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man,
439 u64 *avail, u64 *visible_avail)
441 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
443 mutex_lock(&bman->lock);
444 *avail = bman->mm.avail >> PAGE_SHIFT;
445 *visible_avail = bman->visible_avail;
446 mutex_unlock(&bman->lock);
449 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
450 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man,
453 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
455 bman->visible_size = size;