]> Git Repo - J-linux.git/blob - drivers/gpu/drm/loongson/lsdc_ttm.c
Merge tag 'vfs-6.13-rc7.fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs
[J-linux.git] / drivers / gpu / drm / loongson / lsdc_ttm.c
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright (C) 2023 Loongson Technology Corporation Limited
4  */
5
6 #include <drm/drm_drv.h>
7 #include <drm/drm_file.h>
8 #include <drm/drm_gem.h>
9 #include <drm/drm_managed.h>
10 #include <drm/drm_prime.h>
11
12 #include "lsdc_drv.h"
13 #include "lsdc_ttm.h"
14
15 const char *lsdc_mem_type_to_str(uint32_t mem_type)
16 {
17         switch (mem_type) {
18         case TTM_PL_VRAM:
19                 return "VRAM";
20         case TTM_PL_TT:
21                 return "GTT";
22         case TTM_PL_SYSTEM:
23                 return "SYSTEM";
24         default:
25                 break;
26         }
27
28         return "Unknown";
29 }
30
31 const char *lsdc_domain_to_str(u32 domain)
32 {
33         switch (domain) {
34         case LSDC_GEM_DOMAIN_VRAM:
35                 return "VRAM";
36         case LSDC_GEM_DOMAIN_GTT:
37                 return "GTT";
38         case LSDC_GEM_DOMAIN_SYSTEM:
39                 return "SYSTEM";
40         default:
41                 break;
42         }
43
44         return "Unknown";
45 }
46
47 static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
48 {
49         u32 c = 0;
50         u32 pflags = 0;
51         u32 i;
52
53         if (lbo->tbo.base.size <= PAGE_SIZE)
54                 pflags |= TTM_PL_FLAG_TOPDOWN;
55
56         lbo->placement.placement = lbo->placements;
57
58         if (domain & LSDC_GEM_DOMAIN_VRAM) {
59                 lbo->placements[c].mem_type = TTM_PL_VRAM;
60                 lbo->placements[c++].flags = pflags;
61         }
62
63         if (domain & LSDC_GEM_DOMAIN_GTT) {
64                 lbo->placements[c].mem_type = TTM_PL_TT;
65                 lbo->placements[c++].flags = pflags;
66         }
67
68         if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
69                 lbo->placements[c].mem_type = TTM_PL_SYSTEM;
70                 lbo->placements[c++].flags = 0;
71         }
72
73         if (!c) {
74                 lbo->placements[c].mem_type = TTM_PL_SYSTEM;
75                 lbo->placements[c++].flags = 0;
76         }
77
78         lbo->placement.num_placement = c;
79
80         for (i = 0; i < c; ++i) {
81                 lbo->placements[i].fpfn = 0;
82                 lbo->placements[i].lpfn = 0;
83         }
84 }
85
86 static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
87 {
88         ttm_tt_fini(tt);
89         kfree(tt);
90 }
91
92 static struct ttm_tt *
93 lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
94 {
95         struct ttm_tt *tt;
96         int ret;
97
98         tt = kzalloc(sizeof(*tt), GFP_KERNEL);
99         if (!tt)
100                 return NULL;
101
102         ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
103         if (ret < 0) {
104                 kfree(tt);
105                 return NULL;
106         }
107
108         return tt;
109 }
110
111 static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
112                                 struct ttm_tt *ttm,
113                                 struct ttm_operation_ctx *ctx)
114 {
115         bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
116
117         if (slave && ttm->sg) {
118                 drm_prime_sg_to_dma_addr_array(ttm->sg,
119                                                ttm->dma_address,
120                                                ttm->num_pages);
121
122                 return 0;
123         }
124
125         return ttm_pool_alloc(&bdev->pool, ttm, ctx);
126 }
127
128 static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
129                                    struct ttm_tt *ttm)
130 {
131         bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
132
133         if (slave)
134                 return;
135
136         return ttm_pool_free(&bdev->pool, ttm);
137 }
138
139 static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
140                                 struct ttm_placement *tplacement)
141 {
142         struct ttm_resource *resource = tbo->resource;
143         struct lsdc_bo *lbo = to_lsdc_bo(tbo);
144
145         switch (resource->mem_type) {
146         case TTM_PL_VRAM:
147                 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
148                 break;
149         case TTM_PL_TT:
150         default:
151                 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
152                 break;
153         }
154
155         *tplacement = lbo->placement;
156 }
157
158 static int lsdc_bo_move(struct ttm_buffer_object *tbo,
159                         bool evict,
160                         struct ttm_operation_ctx *ctx,
161                         struct ttm_resource *new_mem,
162                         struct ttm_place *hop)
163 {
164         struct drm_device *ddev = tbo->base.dev;
165         struct ttm_resource *old_mem = tbo->resource;
166         struct lsdc_bo *lbo = to_lsdc_bo(tbo);
167         int ret;
168
169         if (unlikely(tbo->pin_count > 0)) {
170                 drm_warn(ddev, "Can't move a pinned BO\n");
171                 return -EINVAL;
172         }
173
174         ret = ttm_bo_wait_ctx(tbo, ctx);
175         if (ret)
176                 return ret;
177
178         if (!old_mem) {
179                 drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
180                         lbo, lsdc_mem_type_to_str(new_mem->mem_type),
181                         lsdc_bo_size(lbo));
182                 ttm_bo_move_null(tbo, new_mem);
183                 return 0;
184         }
185
186         if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
187                 ttm_bo_move_null(tbo, new_mem);
188                 drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
189                         lbo, lsdc_bo_size(lbo));
190                 return 0;
191         }
192
193         if (old_mem->mem_type == TTM_PL_SYSTEM &&
194             new_mem->mem_type == TTM_PL_TT) {
195                 drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
196                         lbo, lsdc_bo_size(lbo));
197                 ttm_bo_move_null(tbo, new_mem);
198                 return 0;
199         }
200
201         if (old_mem->mem_type == TTM_PL_TT &&
202             new_mem->mem_type == TTM_PL_SYSTEM) {
203                 drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
204                         lbo, lsdc_bo_size(lbo));
205                 ttm_resource_free(tbo, &tbo->resource);
206                 ttm_bo_assign_mem(tbo, new_mem);
207                 return 0;
208         }
209
210         drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
211                 lbo,
212                 lsdc_mem_type_to_str(old_mem->mem_type),
213                 lsdc_mem_type_to_str(new_mem->mem_type),
214                 lsdc_bo_size(lbo));
215
216         return ttm_bo_move_memcpy(tbo, ctx, new_mem);
217 }
218
219 static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
220                                   struct ttm_resource *mem)
221 {
222         struct lsdc_device *ldev = tdev_to_ldev(bdev);
223
224         switch (mem->mem_type) {
225         case TTM_PL_SYSTEM:
226                 break;
227         case TTM_PL_TT:
228                 break;
229         case TTM_PL_VRAM:
230                 mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
231                 mem->bus.is_iomem = true;
232                 mem->bus.caching = ttm_write_combined;
233                 break;
234         default:
235                 return -EINVAL;
236         }
237
238         return 0;
239 }
240
241 static struct ttm_device_funcs lsdc_bo_driver = {
242         .ttm_tt_create = lsdc_ttm_tt_create,
243         .ttm_tt_populate = lsdc_ttm_tt_populate,
244         .ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
245         .ttm_tt_destroy = lsdc_ttm_tt_destroy,
246         .eviction_valuable = ttm_bo_eviction_valuable,
247         .evict_flags = lsdc_bo_evict_flags,
248         .move = lsdc_bo_move,
249         .io_mem_reserve = lsdc_bo_reserve_io_mem,
250 };
251
252 u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
253 {
254         struct ttm_buffer_object *tbo = &lbo->tbo;
255         struct drm_device *ddev = tbo->base.dev;
256         struct ttm_resource *resource = tbo->resource;
257
258         if (unlikely(!tbo->pin_count)) {
259                 drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
260                 return 0;
261         }
262
263         if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
264                 return 0;
265
266         return resource->start << PAGE_SHIFT;
267 }
268
269 size_t lsdc_bo_size(struct lsdc_bo *lbo)
270 {
271         struct ttm_buffer_object *tbo = &lbo->tbo;
272
273         return tbo->base.size;
274 }
275
276 int lsdc_bo_reserve(struct lsdc_bo *lbo)
277 {
278         return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
279 }
280
281 void lsdc_bo_unreserve(struct lsdc_bo *lbo)
282 {
283         return ttm_bo_unreserve(&lbo->tbo);
284 }
285
286 int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
287 {
288         struct ttm_operation_ctx ctx = { false, false };
289         struct ttm_buffer_object *tbo = &lbo->tbo;
290         struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
291         int ret;
292
293         if (tbo->pin_count)
294                 goto bo_pinned;
295
296         if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
297                 return -EINVAL;
298
299         if (domain)
300                 lsdc_bo_set_placement(lbo, domain);
301
302         ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
303         if (unlikely(ret)) {
304                 drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
305                 return ret;
306         }
307
308         if (domain == LSDC_GEM_DOMAIN_VRAM)
309                 ldev->vram_pinned_size += lsdc_bo_size(lbo);
310         else if (domain == LSDC_GEM_DOMAIN_GTT)
311                 ldev->gtt_pinned_size += lsdc_bo_size(lbo);
312
313 bo_pinned:
314         ttm_bo_pin(tbo);
315
316         if (gpu_addr)
317                 *gpu_addr = lsdc_bo_gpu_offset(lbo);
318
319         return 0;
320 }
321
322 void lsdc_bo_unpin(struct lsdc_bo *lbo)
323 {
324         struct ttm_buffer_object *tbo = &lbo->tbo;
325         struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
326
327         if (unlikely(!tbo->pin_count)) {
328                 drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
329                 return;
330         }
331
332         ttm_bo_unpin(tbo);
333
334         if (!tbo->pin_count) {
335                 if (tbo->resource->mem_type == TTM_PL_VRAM)
336                         ldev->vram_pinned_size -= lsdc_bo_size(lbo);
337                 else if (tbo->resource->mem_type == TTM_PL_TT)
338                         ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
339         }
340 }
341
342 void lsdc_bo_ref(struct lsdc_bo *lbo)
343 {
344         drm_gem_object_get(&lbo->tbo.base);
345 }
346
347 void lsdc_bo_unref(struct lsdc_bo *lbo)
348 {
349         drm_gem_object_put(&lbo->tbo.base);
350 }
351
352 int lsdc_bo_kmap(struct lsdc_bo *lbo)
353 {
354         struct ttm_buffer_object *tbo = &lbo->tbo;
355         struct drm_gem_object *gem = &tbo->base;
356         struct drm_device *ddev = gem->dev;
357         long ret;
358         int err;
359
360         ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
361                                     MAX_SCHEDULE_TIMEOUT);
362         if (ret < 0) {
363                 drm_warn(ddev, "wait fence timeout\n");
364                 return ret;
365         }
366
367         if (lbo->kptr)
368                 return 0;
369
370         err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
371         if (err) {
372                 drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
373                 return err;
374         }
375
376         lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
377
378         return 0;
379 }
380
381 void lsdc_bo_kunmap(struct lsdc_bo *lbo)
382 {
383         if (!lbo->kptr)
384                 return;
385
386         lbo->kptr = NULL;
387         ttm_bo_kunmap(&lbo->kmap);
388 }
389
390 void lsdc_bo_clear(struct lsdc_bo *lbo)
391 {
392         lsdc_bo_kmap(lbo);
393
394         if (lbo->is_iomem)
395                 memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
396         else
397                 memset(lbo->kptr, 0, lbo->size);
398
399         lsdc_bo_kunmap(lbo);
400 }
401
402 int lsdc_bo_evict_vram(struct drm_device *ddev)
403 {
404         struct lsdc_device *ldev = to_lsdc(ddev);
405         struct ttm_device *bdev = &ldev->bdev;
406         struct ttm_resource_manager *man;
407
408         man = ttm_manager_type(bdev, TTM_PL_VRAM);
409         if (unlikely(!man))
410                 return 0;
411
412         return ttm_resource_manager_evict_all(bdev, man);
413 }
414
415 static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
416 {
417         struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
418         struct lsdc_bo *lbo = to_lsdc_bo(tbo);
419
420         mutex_lock(&ldev->gem.mutex);
421         list_del_init(&lbo->list);
422         mutex_unlock(&ldev->gem.mutex);
423
424         drm_gem_object_release(&tbo->base);
425
426         kfree(lbo);
427 }
428
429 struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
430                                u32 domain,
431                                size_t size,
432                                bool kernel,
433                                struct sg_table *sg,
434                                struct dma_resv *resv)
435 {
436         struct lsdc_device *ldev = to_lsdc(ddev);
437         struct ttm_device *bdev = &ldev->bdev;
438         struct ttm_buffer_object *tbo;
439         struct lsdc_bo *lbo;
440         enum ttm_bo_type bo_type;
441         int ret;
442
443         lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
444         if (!lbo)
445                 return ERR_PTR(-ENOMEM);
446
447         INIT_LIST_HEAD(&lbo->list);
448
449         lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
450                                         LSDC_GEM_DOMAIN_GTT |
451                                         LSDC_GEM_DOMAIN_SYSTEM);
452
453         tbo = &lbo->tbo;
454
455         size = ALIGN(size, PAGE_SIZE);
456
457         ret = drm_gem_object_init(ddev, &tbo->base, size);
458         if (ret) {
459                 kfree(lbo);
460                 return ERR_PTR(ret);
461         }
462
463         tbo->bdev = bdev;
464
465         if (kernel)
466                 bo_type = ttm_bo_type_kernel;
467         else if (sg)
468                 bo_type = ttm_bo_type_sg;
469         else
470                 bo_type = ttm_bo_type_device;
471
472         lsdc_bo_set_placement(lbo, domain);
473         lbo->size = size;
474
475         ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
476                                    false, sg, resv, lsdc_bo_destroy);
477         if (ret) {
478                 kfree(lbo);
479                 return ERR_PTR(ret);
480         }
481
482         return lbo;
483 }
484
485 struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
486                                              u32 domain,
487                                              size_t size)
488 {
489         struct lsdc_bo *lbo;
490         int ret;
491
492         lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
493         if (IS_ERR(lbo))
494                 return ERR_CAST(lbo);
495
496         ret = lsdc_bo_reserve(lbo);
497         if (unlikely(ret)) {
498                 lsdc_bo_unref(lbo);
499                 return ERR_PTR(ret);
500         }
501
502         ret = lsdc_bo_pin(lbo, domain, NULL);
503         lsdc_bo_unreserve(lbo);
504         if (unlikely(ret)) {
505                 lsdc_bo_unref(lbo);
506                 return ERR_PTR(ret);
507         }
508
509         return lbo;
510 }
511
512 void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
513 {
514         int ret;
515
516         ret = lsdc_bo_reserve(lbo);
517         if (unlikely(ret))
518                 return;
519
520         lsdc_bo_unpin(lbo);
521         lsdc_bo_unreserve(lbo);
522
523         lsdc_bo_unref(lbo);
524 }
525
526 static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
527 {
528         struct lsdc_device *ldev = (struct lsdc_device *)data;
529
530         ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
531         ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
532
533         ttm_device_fini(&ldev->bdev);
534
535         drm_dbg(ddev, "ttm finished\n");
536 }
537
538 int lsdc_ttm_init(struct lsdc_device *ldev)
539 {
540         struct drm_device *ddev = &ldev->base;
541         unsigned long num_vram_pages;
542         unsigned long num_gtt_pages;
543         int ret;
544
545         ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
546                               ddev->anon_inode->i_mapping,
547                               ddev->vma_offset_manager, false, true);
548         if (ret)
549                 return ret;
550
551         num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
552
553         ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
554         if (unlikely(ret))
555                 return ret;
556
557         drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
558
559         /* 512M is far enough for us now */
560         ldev->gtt_size = 512 << 20;
561
562         num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
563
564         ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
565         if (unlikely(ret))
566                 return ret;
567
568         drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
569
570         return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
571 }
572
573 void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
574 {
575         struct ttm_device *bdev = &ldev->bdev;
576         struct drm_device *ddev = &ldev->base;
577         struct drm_minor *minor = ddev->primary;
578         struct dentry *root = minor->debugfs_root;
579         struct ttm_resource_manager *vram_man;
580         struct ttm_resource_manager *gtt_man;
581
582         vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
583         gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
584
585         ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
586         ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
587 }
This page took 0.055302 seconds and 4 git commands to generate.