]> Git Repo - J-linux.git/blob - drivers/gpu/drm/ttm/ttm_resource.c
Merge tag 'vfs-6.13-rc7.fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs
[J-linux.git] / drivers / gpu / drm / ttm / ttm_resource.c
1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24
25 #include <linux/debugfs.h>
26 #include <linux/io-mapping.h>
27 #include <linux/iosys-map.h>
28 #include <linux/scatterlist.h>
29
30 #include <drm/ttm/ttm_bo.h>
31 #include <drm/ttm/ttm_placement.h>
32 #include <drm/ttm/ttm_resource.h>
33 #include <drm/ttm/ttm_tt.h>
34
35 #include <drm/drm_util.h>
36
37 /* Detach the cursor from the bulk move list*/
38 static void
39 ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
40 {
41         lockdep_assert_held(&cursor->man->bdev->lru_lock);
42
43         cursor->bulk = NULL;
44         list_del_init(&cursor->bulk_link);
45 }
46
47 /* Move the cursor to the end of the bulk move list it's in */
48 static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk,
49                                                struct ttm_resource_cursor *cursor)
50 {
51         struct ttm_lru_bulk_move_pos *pos;
52
53         lockdep_assert_held(&cursor->man->bdev->lru_lock);
54
55         if (WARN_ON_ONCE(bulk != cursor->bulk)) {
56                 list_del_init(&cursor->bulk_link);
57                 return;
58         }
59
60         pos = &bulk->pos[cursor->mem_type][cursor->priority];
61         if (pos->last)
62                 list_move(&cursor->hitch.link, &pos->last->lru.link);
63         ttm_resource_cursor_clear_bulk(cursor);
64 }
65
66 /* Move all cursors attached to a bulk move to its end */
67 static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk)
68 {
69         struct ttm_resource_cursor *cursor, *next;
70
71         list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
72                 ttm_resource_cursor_move_bulk_tail(bulk, cursor);
73 }
74
75 /* Remove a cursor from an empty bulk move list */
76 static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk)
77 {
78         struct ttm_resource_cursor *cursor, *next;
79
80         list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
81                 ttm_resource_cursor_clear_bulk(cursor);
82 }
83
84 /**
85  * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
86  * @cursor: The struct ttm_resource_cursor to finalize.
87  *
88  * The function pulls the LRU list cursor off any lists it was previusly
89  * attached to. Needs to be called with the LRU lock held. The function
90  * can be called multiple times after eachother.
91  */
92 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
93 {
94         lockdep_assert_held(&cursor->man->bdev->lru_lock);
95         list_del_init(&cursor->hitch.link);
96         ttm_resource_cursor_clear_bulk(cursor);
97 }
98
99 /**
100  * ttm_lru_bulk_move_init - initialize a bulk move structure
101  * @bulk: the structure to init
102  *
103  * For now just memset the structure to zero.
104  */
105 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
106 {
107         memset(bulk, 0, sizeof(*bulk));
108         INIT_LIST_HEAD(&bulk->cursor_list);
109 }
110 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
111
112 /**
113  * ttm_lru_bulk_move_fini - finalize a bulk move structure
114  * @bdev: The struct ttm_device
115  * @bulk: the structure to finalize
116  *
117  * Sanity checks that bulk moves don't have any
118  * resources left and hence no cursors attached.
119  */
120 void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
121                             struct ttm_lru_bulk_move *bulk)
122 {
123         spin_lock(&bdev->lru_lock);
124         ttm_bulk_move_drop_cursors(bulk);
125         spin_unlock(&bdev->lru_lock);
126 }
127 EXPORT_SYMBOL(ttm_lru_bulk_move_fini);
128
129 /**
130  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
131  *
132  * @bulk: bulk move structure
133  *
134  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
135  * resource order never changes. Should be called with &ttm_device.lru_lock held.
136  */
137 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
138 {
139         unsigned i, j;
140
141         ttm_bulk_move_adjust_cursors(bulk);
142         for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
143                 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
144                         struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
145                         struct ttm_resource_manager *man;
146
147                         if (!pos->first)
148                                 continue;
149
150                         lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
151                         dma_resv_assert_held(pos->first->bo->base.resv);
152                         dma_resv_assert_held(pos->last->bo->base.resv);
153
154                         man = ttm_manager_type(pos->first->bo->bdev, i);
155                         list_bulk_move_tail(&man->lru[j], &pos->first->lru.link,
156                                             &pos->last->lru.link);
157                 }
158         }
159 }
160 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
161
162 /* Return the bulk move pos object for this resource */
163 static struct ttm_lru_bulk_move_pos *
164 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
165 {
166         return &bulk->pos[res->mem_type][res->bo->priority];
167 }
168
169 /* Return the previous resource on the list (skip over non-resource list items) */
170 static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)
171 {
172         struct ttm_lru_item *lru = &cur->lru;
173
174         do {
175                 lru = list_prev_entry(lru, link);
176         } while (!ttm_lru_item_is_res(lru));
177
178         return ttm_lru_item_to_res(lru);
179 }
180
181 /* Return the next resource on the list (skip over non-resource list items) */
182 static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)
183 {
184         struct ttm_lru_item *lru = &cur->lru;
185
186         do {
187                 lru = list_next_entry(lru, link);
188         } while (!ttm_lru_item_is_res(lru));
189
190         return ttm_lru_item_to_res(lru);
191 }
192
193 /* Move the resource to the tail of the bulk move range */
194 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
195                                        struct ttm_resource *res)
196 {
197         if (pos->last != res) {
198                 if (pos->first == res)
199                         pos->first = ttm_lru_next_res(res);
200                 list_move(&res->lru.link, &pos->last->lru.link);
201                 pos->last = res;
202         }
203 }
204
205 /* Add the resource to a bulk_move cursor */
206 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
207                                   struct ttm_resource *res)
208 {
209         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
210
211         if (!pos->first) {
212                 pos->first = res;
213                 pos->last = res;
214         } else {
215                 WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);
216                 ttm_lru_bulk_move_pos_tail(pos, res);
217         }
218 }
219
220 /* Remove the resource from a bulk_move range */
221 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
222                                   struct ttm_resource *res)
223 {
224         struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
225
226         if (unlikely(WARN_ON(!pos->first || !pos->last) ||
227                      (pos->first == res && pos->last == res))) {
228                 pos->first = NULL;
229                 pos->last = NULL;
230         } else if (pos->first == res) {
231                 pos->first = ttm_lru_next_res(res);
232         } else if (pos->last == res) {
233                 pos->last = ttm_lru_prev_res(res);
234         } else {
235                 list_move(&res->lru.link, &pos->last->lru.link);
236         }
237 }
238
239 static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_object *bo)
240 {
241         /*
242          * Take care when creating a new resource for a bo, that it is not considered
243          * swapped if it's not the current resource for the bo and is thus logically
244          * associated with the ttm_tt. Think a VRAM resource created to move a
245          * swapped-out bo to VRAM.
246          */
247         if (bo->resource != res || !bo->ttm)
248                 return false;
249
250         dma_resv_assert_held(bo->base.resv);
251         return ttm_tt_is_swapped(bo->ttm);
252 }
253
254 /* Add the resource to a bulk move if the BO is configured for it */
255 void ttm_resource_add_bulk_move(struct ttm_resource *res,
256                                 struct ttm_buffer_object *bo)
257 {
258         if (bo->bulk_move && !bo->pin_count && !ttm_resource_is_swapped(res, bo))
259                 ttm_lru_bulk_move_add(bo->bulk_move, res);
260 }
261
262 /* Remove the resource from a bulk move if the BO is configured for it */
263 void ttm_resource_del_bulk_move(struct ttm_resource *res,
264                                 struct ttm_buffer_object *bo)
265 {
266         if (bo->bulk_move && !bo->pin_count && !ttm_resource_is_swapped(res, bo))
267                 ttm_lru_bulk_move_del(bo->bulk_move, res);
268 }
269
270 /* Move a resource to the LRU or bulk tail */
271 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
272 {
273         struct ttm_buffer_object *bo = res->bo;
274         struct ttm_device *bdev = bo->bdev;
275
276         lockdep_assert_held(&bo->bdev->lru_lock);
277
278         if (bo->pin_count || ttm_resource_is_swapped(res, bo)) {
279                 list_move_tail(&res->lru.link, &bdev->unevictable);
280
281         } else  if (bo->bulk_move) {
282                 struct ttm_lru_bulk_move_pos *pos =
283                         ttm_lru_bulk_move_pos(bo->bulk_move, res);
284
285                 ttm_lru_bulk_move_pos_tail(pos, res);
286         } else {
287                 struct ttm_resource_manager *man;
288
289                 man = ttm_manager_type(bdev, res->mem_type);
290                 list_move_tail(&res->lru.link, &man->lru[bo->priority]);
291         }
292 }
293
294 /**
295  * ttm_resource_init - resource object constructure
296  * @bo: buffer object this resources is allocated for
297  * @place: placement of the resource
298  * @res: the resource object to inistilize
299  *
300  * Initialize a new resource object. Counterpart of ttm_resource_fini().
301  */
302 void ttm_resource_init(struct ttm_buffer_object *bo,
303                        const struct ttm_place *place,
304                        struct ttm_resource *res)
305 {
306         struct ttm_resource_manager *man;
307
308         res->start = 0;
309         res->size = bo->base.size;
310         res->mem_type = place->mem_type;
311         res->placement = place->flags;
312         res->bus.addr = NULL;
313         res->bus.offset = 0;
314         res->bus.is_iomem = false;
315         res->bus.caching = ttm_cached;
316         res->bo = bo;
317
318         man = ttm_manager_type(bo->bdev, place->mem_type);
319         spin_lock(&bo->bdev->lru_lock);
320         if (bo->pin_count || ttm_resource_is_swapped(res, bo))
321                 list_add_tail(&res->lru.link, &bo->bdev->unevictable);
322         else
323                 list_add_tail(&res->lru.link, &man->lru[bo->priority]);
324         man->usage += res->size;
325         spin_unlock(&bo->bdev->lru_lock);
326 }
327 EXPORT_SYMBOL(ttm_resource_init);
328
329 /**
330  * ttm_resource_fini - resource destructor
331  * @man: the resource manager this resource belongs to
332  * @res: the resource to clean up
333  *
334  * Should be used by resource manager backends to clean up the TTM resource
335  * objects before freeing the underlying structure. Makes sure the resource is
336  * removed from the LRU before destruction.
337  * Counterpart of ttm_resource_init().
338  */
339 void ttm_resource_fini(struct ttm_resource_manager *man,
340                        struct ttm_resource *res)
341 {
342         struct ttm_device *bdev = man->bdev;
343
344         spin_lock(&bdev->lru_lock);
345         list_del_init(&res->lru.link);
346         man->usage -= res->size;
347         spin_unlock(&bdev->lru_lock);
348 }
349 EXPORT_SYMBOL(ttm_resource_fini);
350
351 int ttm_resource_alloc(struct ttm_buffer_object *bo,
352                        const struct ttm_place *place,
353                        struct ttm_resource **res_ptr)
354 {
355         struct ttm_resource_manager *man =
356                 ttm_manager_type(bo->bdev, place->mem_type);
357         int ret;
358
359         ret = man->func->alloc(man, bo, place, res_ptr);
360         if (ret)
361                 return ret;
362
363         spin_lock(&bo->bdev->lru_lock);
364         ttm_resource_add_bulk_move(*res_ptr, bo);
365         spin_unlock(&bo->bdev->lru_lock);
366         return 0;
367 }
368 EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);
369
370 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
371 {
372         struct ttm_resource_manager *man;
373
374         if (!*res)
375                 return;
376
377         spin_lock(&bo->bdev->lru_lock);
378         ttm_resource_del_bulk_move(*res, bo);
379         spin_unlock(&bo->bdev->lru_lock);
380         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
381         man->func->free(man, *res);
382         *res = NULL;
383 }
384 EXPORT_SYMBOL(ttm_resource_free);
385
386 /**
387  * ttm_resource_intersects - test for intersection
388  *
389  * @bdev: TTM device structure
390  * @res: The resource to test
391  * @place: The placement to test
392  * @size: How many bytes the new allocation needs.
393  *
394  * Test if @res intersects with @place and @size. Used for testing if evictions
395  * are valueable or not.
396  *
397  * Returns true if the res placement intersects with @place and @size.
398  */
399 bool ttm_resource_intersects(struct ttm_device *bdev,
400                              struct ttm_resource *res,
401                              const struct ttm_place *place,
402                              size_t size)
403 {
404         struct ttm_resource_manager *man;
405
406         if (!res)
407                 return false;
408
409         man = ttm_manager_type(bdev, res->mem_type);
410         if (!place || !man->func->intersects)
411                 return true;
412
413         return man->func->intersects(man, res, place, size);
414 }
415
416 /**
417  * ttm_resource_compatible - check if resource is compatible with placement
418  *
419  * @res: the resource to check
420  * @placement: the placement to check against
421  * @evicting: true if the caller is doing evictions
422  *
423  * Returns true if the placement is compatible.
424  */
425 bool ttm_resource_compatible(struct ttm_resource *res,
426                              struct ttm_placement *placement,
427                              bool evicting)
428 {
429         struct ttm_buffer_object *bo = res->bo;
430         struct ttm_device *bdev = bo->bdev;
431         unsigned i;
432
433         if (res->placement & TTM_PL_FLAG_TEMPORARY)
434                 return false;
435
436         for (i = 0; i < placement->num_placement; i++) {
437                 const struct ttm_place *place = &placement->placement[i];
438                 struct ttm_resource_manager *man;
439
440                 if (res->mem_type != place->mem_type)
441                         continue;
442
443                 if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :
444                                     TTM_PL_FLAG_FALLBACK))
445                         continue;
446
447                 if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&
448                     !(res->placement & TTM_PL_FLAG_CONTIGUOUS))
449                         continue;
450
451                 man = ttm_manager_type(bdev, res->mem_type);
452                 if (man->func->compatible &&
453                     !man->func->compatible(man, res, place, bo->base.size))
454                         continue;
455
456                 return true;
457         }
458         return false;
459 }
460
461 void ttm_resource_set_bo(struct ttm_resource *res,
462                          struct ttm_buffer_object *bo)
463 {
464         spin_lock(&bo->bdev->lru_lock);
465         res->bo = bo;
466         spin_unlock(&bo->bdev->lru_lock);
467 }
468
469 /**
470  * ttm_resource_manager_init
471  *
472  * @man: memory manager object to init
473  * @bdev: ttm device this manager belongs to
474  * @size: size of managed resources in arbitrary units
475  *
476  * Initialise core parts of a manager object.
477  */
478 void ttm_resource_manager_init(struct ttm_resource_manager *man,
479                                struct ttm_device *bdev,
480                                uint64_t size)
481 {
482         unsigned i;
483
484         spin_lock_init(&man->move_lock);
485         man->bdev = bdev;
486         man->size = size;
487         man->usage = 0;
488
489         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
490                 INIT_LIST_HEAD(&man->lru[i]);
491         man->move = NULL;
492 }
493 EXPORT_SYMBOL(ttm_resource_manager_init);
494
495 /*
496  * ttm_resource_manager_evict_all
497  *
498  * @bdev - device to use
499  * @man - manager to use
500  *
501  * Evict all the objects out of a memory manager until it is empty.
502  * Part of memory manager cleanup sequence.
503  */
504 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
505                                    struct ttm_resource_manager *man)
506 {
507         struct ttm_operation_ctx ctx = {
508                 .interruptible = false,
509                 .no_wait_gpu = false,
510                 .force_alloc = true
511         };
512         struct dma_fence *fence;
513         int ret;
514
515         do {
516                 ret = ttm_bo_evict_first(bdev, man, &ctx);
517                 cond_resched();
518         } while (!ret);
519
520         spin_lock(&man->move_lock);
521         fence = dma_fence_get(man->move);
522         spin_unlock(&man->move_lock);
523
524         if (fence) {
525                 ret = dma_fence_wait(fence, false);
526                 dma_fence_put(fence);
527                 if (ret)
528                         return ret;
529         }
530
531         return 0;
532 }
533 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
534
535 /**
536  * ttm_resource_manager_usage
537  *
538  * @man: A memory manager object.
539  *
540  * Return how many resources are currently used.
541  */
542 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
543 {
544         uint64_t usage;
545
546         spin_lock(&man->bdev->lru_lock);
547         usage = man->usage;
548         spin_unlock(&man->bdev->lru_lock);
549         return usage;
550 }
551 EXPORT_SYMBOL(ttm_resource_manager_usage);
552
553 /**
554  * ttm_resource_manager_debug
555  *
556  * @man: manager type to dump.
557  * @p: printer to use for debug.
558  */
559 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
560                                 struct drm_printer *p)
561 {
562         drm_printf(p, "  use_type: %d\n", man->use_type);
563         drm_printf(p, "  use_tt: %d\n", man->use_tt);
564         drm_printf(p, "  size: %llu\n", man->size);
565         drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
566         if (man->func->debug)
567                 man->func->debug(man, p);
568 }
569 EXPORT_SYMBOL(ttm_resource_manager_debug);
570
571 static void
572 ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
573                                struct ttm_lru_item *next_lru)
574 {
575         struct ttm_resource *next = ttm_lru_item_to_res(next_lru);
576         struct ttm_lru_bulk_move *bulk = NULL;
577         struct ttm_buffer_object *bo = next->bo;
578
579         lockdep_assert_held(&cursor->man->bdev->lru_lock);
580         bulk = bo->bulk_move;
581
582         if (cursor->bulk != bulk) {
583                 if (bulk) {
584                         list_move_tail(&cursor->bulk_link, &bulk->cursor_list);
585                         cursor->mem_type = next->mem_type;
586                 } else {
587                         list_del_init(&cursor->bulk_link);
588                 }
589                 cursor->bulk = bulk;
590         }
591 }
592
593 /**
594  * ttm_resource_manager_first() - Start iterating over the resources
595  * of a resource manager
596  * @man: resource manager to iterate over
597  * @cursor: cursor to record the position
598  *
599  * Initializes the cursor and starts iterating. When done iterating,
600  * the caller must explicitly call ttm_resource_cursor_fini().
601  *
602  * Return: The first resource from the resource manager.
603  */
604 struct ttm_resource *
605 ttm_resource_manager_first(struct ttm_resource_manager *man,
606                            struct ttm_resource_cursor *cursor)
607 {
608         lockdep_assert_held(&man->bdev->lru_lock);
609
610         cursor->priority = 0;
611         cursor->man = man;
612         ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH);
613         INIT_LIST_HEAD(&cursor->bulk_link);
614         list_add(&cursor->hitch.link, &man->lru[cursor->priority]);
615
616         return ttm_resource_manager_next(cursor);
617 }
618
619 /**
620  * ttm_resource_manager_next() - Continue iterating over the resource manager
621  * resources
622  * @cursor: cursor to record the position
623  *
624  * Return: the next resource from the resource manager.
625  */
626 struct ttm_resource *
627 ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
628 {
629         struct ttm_resource_manager *man = cursor->man;
630         struct ttm_lru_item *lru;
631
632         lockdep_assert_held(&man->bdev->lru_lock);
633
634         for (;;) {
635                 lru = &cursor->hitch;
636                 list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
637                         if (ttm_lru_item_is_res(lru)) {
638                                 ttm_resource_cursor_check_bulk(cursor, lru);
639                                 list_move(&cursor->hitch.link, &lru->link);
640                                 return ttm_lru_item_to_res(lru);
641                         }
642                 }
643
644                 if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
645                         break;
646
647                 list_move(&cursor->hitch.link, &man->lru[cursor->priority]);
648                 ttm_resource_cursor_clear_bulk(cursor);
649         }
650
651         ttm_resource_cursor_fini(cursor);
652
653         return NULL;
654 }
655
656 /**
657  * ttm_lru_first_res_or_null() - Return the first resource on an lru list
658  * @head: The list head of the lru list.
659  *
660  * Return: Pointer to the first resource on the lru list or NULL if
661  * there is none.
662  */
663 struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head)
664 {
665         struct ttm_lru_item *lru;
666
667         list_for_each_entry(lru, head, link) {
668                 if (ttm_lru_item_is_res(lru))
669                         return ttm_lru_item_to_res(lru);
670         }
671
672         return NULL;
673 }
674
675 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
676                                           struct iosys_map *dmap,
677                                           pgoff_t i)
678 {
679         struct ttm_kmap_iter_iomap *iter_io =
680                 container_of(iter, typeof(*iter_io), base);
681         void __iomem *addr;
682
683 retry:
684         while (i >= iter_io->cache.end) {
685                 iter_io->cache.sg = iter_io->cache.sg ?
686                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
687                 iter_io->cache.i = iter_io->cache.end;
688                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
689                         PAGE_SHIFT;
690                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
691                         iter_io->start;
692         }
693
694         if (i < iter_io->cache.i) {
695                 iter_io->cache.end = 0;
696                 iter_io->cache.sg = NULL;
697                 goto retry;
698         }
699
700         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
701                                        (((resource_size_t)i - iter_io->cache.i)
702                                         << PAGE_SHIFT));
703         iosys_map_set_vaddr_iomem(dmap, addr);
704 }
705
706 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
707                                             struct iosys_map *map)
708 {
709         io_mapping_unmap_local(map->vaddr_iomem);
710 }
711
712 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
713         .map_local =  ttm_kmap_iter_iomap_map_local,
714         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
715         .maps_tt = false,
716 };
717
718 /**
719  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
720  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
721  * @iomap: The struct io_mapping representing the underlying linear io_memory.
722  * @st: sg_table into @iomap, representing the memory of the struct
723  * ttm_resource.
724  * @start: Offset that needs to be subtracted from @st to make
725  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
726  *
727  * Return: Pointer to the embedded struct ttm_kmap_iter.
728  */
729 struct ttm_kmap_iter *
730 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
731                          struct io_mapping *iomap,
732                          struct sg_table *st,
733                          resource_size_t start)
734 {
735         iter_io->base.ops = &ttm_kmap_iter_io_ops;
736         iter_io->iomap = iomap;
737         iter_io->st = st;
738         iter_io->start = start;
739         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
740
741         return &iter_io->base;
742 }
743 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
744
745 /**
746  * DOC: Linear io iterator
747  *
748  * This code should die in the not too near future. Best would be if we could
749  * make io-mapping use memremap for all io memory, and have memremap
750  * implement a kmap_local functionality. We could then strip a huge amount of
751  * code. These linear io iterators are implemented to mimic old functionality,
752  * and they don't use kmap_local semantics at all internally. Rather ioremap or
753  * friends, and at least on 32-bit they add global TLB flushes and points
754  * of failure.
755  */
756
757 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
758                                               struct iosys_map *dmap,
759                                               pgoff_t i)
760 {
761         struct ttm_kmap_iter_linear_io *iter_io =
762                 container_of(iter, typeof(*iter_io), base);
763
764         *dmap = iter_io->dmap;
765         iosys_map_incr(dmap, i * PAGE_SIZE);
766 }
767
768 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
769         .map_local =  ttm_kmap_iter_linear_io_map_local,
770         .maps_tt = false,
771 };
772
773 /**
774  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
775  * @iter_io: The iterator to initialize
776  * @bdev: The TTM device
777  * @mem: The ttm resource representing the iomap.
778  *
779  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
780  * pointing at a linear chunk of io memory.
781  *
782  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
783  * failure.
784  */
785 struct ttm_kmap_iter *
786 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
787                              struct ttm_device *bdev,
788                              struct ttm_resource *mem)
789 {
790         int ret;
791
792         ret = ttm_mem_io_reserve(bdev, mem);
793         if (ret)
794                 goto out_err;
795         if (!mem->bus.is_iomem) {
796                 ret = -EINVAL;
797                 goto out_io_free;
798         }
799
800         if (mem->bus.addr) {
801                 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
802                 iter_io->needs_unmap = false;
803         } else {
804                 iter_io->needs_unmap = true;
805                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
806                 if (mem->bus.caching == ttm_write_combined)
807                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
808                                                   ioremap_wc(mem->bus.offset,
809                                                              mem->size));
810                 else if (mem->bus.caching == ttm_cached)
811                         iosys_map_set_vaddr(&iter_io->dmap,
812                                             memremap(mem->bus.offset, mem->size,
813                                                      MEMREMAP_WB |
814                                                      MEMREMAP_WT |
815                                                      MEMREMAP_WC));
816
817                 /* If uncached requested or if mapping cached or wc failed */
818                 if (iosys_map_is_null(&iter_io->dmap))
819                         iosys_map_set_vaddr_iomem(&iter_io->dmap,
820                                                   ioremap(mem->bus.offset,
821                                                           mem->size));
822
823                 if (iosys_map_is_null(&iter_io->dmap)) {
824                         ret = -ENOMEM;
825                         goto out_io_free;
826                 }
827         }
828
829         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
830         return &iter_io->base;
831
832 out_io_free:
833         ttm_mem_io_free(bdev, mem);
834 out_err:
835         return ERR_PTR(ret);
836 }
837
838 /**
839  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
840  * @iter_io: The iterator to initialize
841  * @bdev: The TTM device
842  * @mem: The ttm resource representing the iomap.
843  *
844  * This function is for internal TTM use only. It cleans up a memcpy kmap
845  * iterator initialized by ttm_kmap_iter_linear_io_init.
846  */
847 void
848 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
849                              struct ttm_device *bdev,
850                              struct ttm_resource *mem)
851 {
852         if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
853                 if (iter_io->dmap.is_iomem)
854                         iounmap(iter_io->dmap.vaddr_iomem);
855                 else
856                         memunmap(iter_io->dmap.vaddr);
857         }
858
859         ttm_mem_io_free(bdev, mem);
860 }
861
862 #if defined(CONFIG_DEBUG_FS)
863
864 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
865 {
866         struct ttm_resource_manager *man =
867                 (struct ttm_resource_manager *)m->private;
868         struct drm_printer p = drm_seq_file_printer(m);
869         ttm_resource_manager_debug(man, &p);
870         return 0;
871 }
872 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
873
874 #endif
875
876 /**
877  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
878  * resource manager.
879  * @man: The TTM resource manager for which the debugfs stats file be creates
880  * @parent: debugfs directory in which the file will reside
881  * @name: The filename to create.
882  *
883  * This function setups up a debugfs file that can be used to look
884  * at debug statistics of the specified ttm_resource_manager.
885  */
886 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
887                                          struct dentry * parent,
888                                          const char *name)
889 {
890 #if defined(CONFIG_DEBUG_FS)
891         debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
892 #endif
893 }
894 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
This page took 0.076339 seconds and 4 git commands to generate.