]> Git Repo - linux.git/blob - drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
Merge tag 'amd-drm-next-6.5-2023-06-09' of https://gitlab.freedesktop.org/agd5f/linux...
[linux.git] / drivers / gpu / drm / i915 / i915_ttm_buddy_manager.c
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5
6 #include <linux/slab.h>
7
8 #include <drm/ttm/ttm_placement.h>
9 #include <drm/ttm/ttm_bo.h>
10
11 #include <drm/drm_buddy.h>
12
13 #include "i915_ttm_buddy_manager.h"
14
15 #include "i915_gem.h"
16
17 struct i915_ttm_buddy_manager {
18         struct ttm_resource_manager manager;
19         struct drm_buddy mm;
20         struct list_head reserved;
21         struct mutex lock;
22         unsigned long visible_size;
23         unsigned long visible_avail;
24         unsigned long visible_reserved;
25         u64 default_page_size;
26 };
27
28 static struct i915_ttm_buddy_manager *
29 to_buddy_manager(struct ttm_resource_manager *man)
30 {
31         return container_of(man, struct i915_ttm_buddy_manager, manager);
32 }
33
34 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
35                                     struct ttm_buffer_object *bo,
36                                     const struct ttm_place *place,
37                                     struct ttm_resource **res)
38 {
39         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
40         struct i915_ttm_buddy_resource *bman_res;
41         struct drm_buddy *mm = &bman->mm;
42         unsigned long n_pages, lpfn;
43         u64 min_page_size;
44         u64 size;
45         int err;
46
47         lpfn = place->lpfn;
48         if (!lpfn)
49                 lpfn = man->size;
50
51         bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL);
52         if (!bman_res)
53                 return -ENOMEM;
54
55         ttm_resource_init(bo, place, &bman_res->base);
56         INIT_LIST_HEAD(&bman_res->blocks);
57         bman_res->mm = mm;
58
59         if (place->flags & TTM_PL_FLAG_TOPDOWN)
60                 bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
61
62         if (place->fpfn || lpfn != man->size)
63                 bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
64
65         GEM_BUG_ON(!bman_res->base.size);
66         size = bman_res->base.size;
67
68         min_page_size = bman->default_page_size;
69         if (bo->page_alignment)
70                 min_page_size = bo->page_alignment << PAGE_SHIFT;
71
72         GEM_BUG_ON(min_page_size < mm->chunk_size);
73         GEM_BUG_ON(!IS_ALIGNED(size, min_page_size));
74
75         if (place->fpfn + PFN_UP(bman_res->base.size) != place->lpfn &&
76             place->flags & TTM_PL_FLAG_CONTIGUOUS) {
77                 unsigned long pages;
78
79                 size = roundup_pow_of_two(size);
80                 min_page_size = size;
81
82                 pages = size >> ilog2(mm->chunk_size);
83                 if (pages > lpfn)
84                         lpfn = pages;
85         }
86
87         if (size > lpfn << PAGE_SHIFT) {
88                 err = -E2BIG;
89                 goto err_free_res;
90         }
91
92         n_pages = size >> ilog2(mm->chunk_size);
93
94         mutex_lock(&bman->lock);
95         if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) {
96                 mutex_unlock(&bman->lock);
97                 err = -ENOSPC;
98                 goto err_free_res;
99         }
100
101         err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
102                                      (u64)lpfn << PAGE_SHIFT,
103                                      (u64)n_pages << PAGE_SHIFT,
104                                      min_page_size,
105                                      &bman_res->blocks,
106                                      bman_res->flags);
107         if (unlikely(err))
108                 goto err_free_blocks;
109
110         if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
111                 u64 original_size = (u64)bman_res->base.size;
112
113                 drm_buddy_block_trim(mm,
114                                      original_size,
115                                      &bman_res->blocks);
116         }
117
118         if (lpfn <= bman->visible_size) {
119                 bman_res->used_visible_size = PFN_UP(bman_res->base.size);
120         } else {
121                 struct drm_buddy_block *block;
122
123                 list_for_each_entry(block, &bman_res->blocks, link) {
124                         unsigned long start =
125                                 drm_buddy_block_offset(block) >> PAGE_SHIFT;
126
127                         if (start < bman->visible_size) {
128                                 unsigned long end = start +
129                                         (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
130
131                                 bman_res->used_visible_size +=
132                                         min(end, bman->visible_size) - start;
133                         }
134                 }
135         }
136
137         if (bman_res->used_visible_size)
138                 bman->visible_avail -= bman_res->used_visible_size;
139
140         mutex_unlock(&bman->lock);
141
142         *res = &bman_res->base;
143         return 0;
144
145 err_free_blocks:
146         drm_buddy_free_list(mm, &bman_res->blocks);
147         mutex_unlock(&bman->lock);
148 err_free_res:
149         ttm_resource_fini(man, &bman_res->base);
150         kfree(bman_res);
151         return err;
152 }
153
154 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
155                                     struct ttm_resource *res)
156 {
157         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
158         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
159
160         mutex_lock(&bman->lock);
161         drm_buddy_free_list(&bman->mm, &bman_res->blocks);
162         bman->visible_avail += bman_res->used_visible_size;
163         mutex_unlock(&bman->lock);
164
165         ttm_resource_fini(man, res);
166         kfree(bman_res);
167 }
168
169 static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
170                                           struct ttm_resource *res,
171                                           const struct ttm_place *place,
172                                           size_t size)
173 {
174         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
175         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
176         struct drm_buddy *mm = &bman->mm;
177         struct drm_buddy_block *block;
178
179         if (!place->fpfn && !place->lpfn)
180                 return true;
181
182         GEM_BUG_ON(!place->lpfn);
183
184         /*
185          * If we just want something mappable then we can quickly check
186          * if the current victim resource is using any of the CPU
187          * visible portion.
188          */
189         if (!place->fpfn &&
190             place->lpfn == i915_ttm_buddy_man_visible_size(man))
191                 return bman_res->used_visible_size > 0;
192
193         /* Check each drm buddy block individually */
194         list_for_each_entry(block, &bman_res->blocks, link) {
195                 unsigned long fpfn =
196                         drm_buddy_block_offset(block) >> PAGE_SHIFT;
197                 unsigned long lpfn = fpfn +
198                         (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
199
200                 if (place->fpfn < lpfn && place->lpfn > fpfn)
201                         return true;
202         }
203
204         return false;
205 }
206
207 static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
208                                           struct ttm_resource *res,
209                                           const struct ttm_place *place,
210                                           size_t size)
211 {
212         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
213         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
214         struct drm_buddy *mm = &bman->mm;
215         struct drm_buddy_block *block;
216
217         if (!place->fpfn && !place->lpfn)
218                 return true;
219
220         GEM_BUG_ON(!place->lpfn);
221
222         if (!place->fpfn &&
223             place->lpfn == i915_ttm_buddy_man_visible_size(man))
224                 return bman_res->used_visible_size == PFN_UP(res->size);
225
226         /* Check each drm buddy block individually */
227         list_for_each_entry(block, &bman_res->blocks, link) {
228                 unsigned long fpfn =
229                         drm_buddy_block_offset(block) >> PAGE_SHIFT;
230                 unsigned long lpfn = fpfn +
231                         (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
232
233                 if (fpfn < place->fpfn || lpfn > place->lpfn)
234                         return false;
235         }
236
237         return true;
238 }
239
240 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
241                                      struct drm_printer *printer)
242 {
243         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
244         struct drm_buddy_block *block;
245
246         mutex_lock(&bman->lock);
247         drm_printf(printer, "default_page_size: %lluKiB\n",
248                    bman->default_page_size >> 10);
249         drm_printf(printer, "visible_avail: %lluMiB\n",
250                    (u64)bman->visible_avail << PAGE_SHIFT >> 20);
251         drm_printf(printer, "visible_size: %lluMiB\n",
252                    (u64)bman->visible_size << PAGE_SHIFT >> 20);
253         drm_printf(printer, "visible_reserved: %lluMiB\n",
254                    (u64)bman->visible_reserved << PAGE_SHIFT >> 20);
255
256         drm_buddy_print(&bman->mm, printer);
257
258         drm_printf(printer, "reserved:\n");
259         list_for_each_entry(block, &bman->reserved, link)
260                 drm_buddy_block_print(&bman->mm, block, printer);
261         mutex_unlock(&bman->lock);
262 }
263
264 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = {
265         .alloc = i915_ttm_buddy_man_alloc,
266         .free = i915_ttm_buddy_man_free,
267         .intersects = i915_ttm_buddy_man_intersects,
268         .compatible = i915_ttm_buddy_man_compatible,
269         .debug = i915_ttm_buddy_man_debug,
270 };
271
272 /**
273  * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager
274  * @bdev: The ttm device
275  * @type: Memory type we want to manage
276  * @use_tt: Set use_tt for the manager
277  * @size: The size in bytes to manage
278  * @visible_size: The CPU visible size in bytes to manage
279  * @default_page_size: The default minimum page size in bytes for allocations,
280  * this must be at least as large as @chunk_size, and can be overridden by
281  * setting the BO page_alignment, to be larger or smaller as needed.
282  * @chunk_size: The minimum page size in bytes for our allocations i.e
283  * order-zero
284  *
285  * Note that the starting address is assumed to be zero here, since this
286  * simplifies keeping the property where allocated blocks having natural
287  * power-of-two alignment. So long as the real starting address is some large
288  * power-of-two, or naturally start from zero, then this should be fine.  Also
289  * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment
290  * if say there is some unusable range from the start of the region. We can
291  * revisit this in the future and make the interface accept an actual starting
292  * offset and let it take care of the rest.
293  *
294  * Note that if the @size is not aligned to the @chunk_size then we perform the
295  * required rounding to get the usable size. The final size in pages can be
296  * taken from &ttm_resource_manager.size.
297  *
298  * Return: 0 on success, negative error code on failure.
299  */
300 int i915_ttm_buddy_man_init(struct ttm_device *bdev,
301                             unsigned int type, bool use_tt,
302                             u64 size, u64 visible_size, u64 default_page_size,
303                             u64 chunk_size)
304 {
305         struct ttm_resource_manager *man;
306         struct i915_ttm_buddy_manager *bman;
307         int err;
308
309         bman = kzalloc(sizeof(*bman), GFP_KERNEL);
310         if (!bman)
311                 return -ENOMEM;
312
313         err = drm_buddy_init(&bman->mm, size, chunk_size);
314         if (err)
315                 goto err_free_bman;
316
317         mutex_init(&bman->lock);
318         INIT_LIST_HEAD(&bman->reserved);
319         GEM_BUG_ON(default_page_size < chunk_size);
320         bman->default_page_size = default_page_size;
321         bman->visible_size = visible_size >> PAGE_SHIFT;
322         bman->visible_avail = bman->visible_size;
323
324         man = &bman->manager;
325         man->use_tt = use_tt;
326         man->func = &i915_ttm_buddy_manager_func;
327         ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT);
328
329         ttm_resource_manager_set_used(man, true);
330         ttm_set_driver_manager(bdev, type, man);
331
332         return 0;
333
334 err_free_bman:
335         kfree(bman);
336         return err;
337 }
338
339 /**
340  * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager
341  * @bdev: The ttm device
342  * @type: Memory type we want to manage
343  *
344  * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will
345  * also be freed for us here.
346  *
347  * Return: 0 on success, negative error code on failure.
348  */
349 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
350 {
351         struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
352         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
353         struct drm_buddy *mm = &bman->mm;
354         int ret;
355
356         ttm_resource_manager_set_used(man, false);
357
358         ret = ttm_resource_manager_evict_all(bdev, man);
359         if (ret)
360                 return ret;
361
362         ttm_set_driver_manager(bdev, type, NULL);
363
364         mutex_lock(&bman->lock);
365         drm_buddy_free_list(mm, &bman->reserved);
366         drm_buddy_fini(mm);
367         bman->visible_avail += bman->visible_reserved;
368         WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
369         mutex_unlock(&bman->lock);
370
371         ttm_resource_manager_cleanup(man);
372         kfree(bman);
373
374         return 0;
375 }
376
377 /**
378  * i915_ttm_buddy_man_reserve - Reserve address range
379  * @man: The buddy allocator ttm manager
380  * @start: The offset in bytes, where the region start is assumed to be zero
381  * @size: The size in bytes
382  *
383  * Note that the starting address for the region is always assumed to be zero.
384  *
385  * Return: 0 on success, negative error code on failure.
386  */
387 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
388                                u64 start, u64 size)
389 {
390         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
391         struct drm_buddy *mm = &bman->mm;
392         unsigned long fpfn = start >> PAGE_SHIFT;
393         unsigned long flags = 0;
394         int ret;
395
396         flags |= DRM_BUDDY_RANGE_ALLOCATION;
397
398         mutex_lock(&bman->lock);
399         ret = drm_buddy_alloc_blocks(mm, start,
400                                      start + size,
401                                      size, mm->chunk_size,
402                                      &bman->reserved,
403                                      flags);
404
405         if (fpfn < bman->visible_size) {
406                 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT);
407                 unsigned long visible = min(lpfn, bman->visible_size) - fpfn;
408
409                 bman->visible_reserved += visible;
410                 bman->visible_avail -= visible;
411         }
412         mutex_unlock(&bman->lock);
413
414         return ret;
415 }
416
417 /**
418  * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion
419  * in pages.
420  * @man: The buddy allocator ttm manager
421  */
422 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man)
423 {
424         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
425
426         return bman->visible_size;
427 }
428
429 /**
430  * i915_ttm_buddy_man_avail - Query the avail tracking for the manager.
431  *
432  * @man: The buddy allocator ttm manager
433  * @avail: The total available memory in pages for the entire manager.
434  * @visible_avail: The total available memory in pages for the CPU visible
435  * portion. Note that this will always give the same value as @avail on
436  * configurations that don't have a small BAR.
437  */
438 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man,
439                               u64 *avail, u64 *visible_avail)
440 {
441         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
442
443         mutex_lock(&bman->lock);
444         *avail = bman->mm.avail >> PAGE_SHIFT;
445         *visible_avail = bman->visible_avail;
446         mutex_unlock(&bman->lock);
447 }
448
449 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
450 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man,
451                                            u64 size)
452 {
453         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
454
455         bman->visible_size = size;
456 }
457 #endif
This page took 0.060181 seconds and 4 git commands to generate.