drivers/gpu/drm/i915/i915_ttm_buddy_manager.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include <linux/slab.h>
   7
   8 #include <drm/ttm/ttm_placement.h>
   9 #include <drm/ttm/ttm_bo.h>
  10
  11 #include <drm/drm_buddy.h>
  12
  13 #include "i915_ttm_buddy_manager.h"
  14
  15 #include "i915_gem.h"
  16
  17 struct i915_ttm_buddy_manager {
  18         struct ttm_resource_manager manager;
  19         struct drm_buddy mm;
  20         struct list_head reserved;
  21         struct mutex lock;
  22         unsigned long visible_size;
  23         unsigned long visible_avail;
  24         unsigned long visible_reserved;
  25         u64 default_page_size;
  26 };
  27
  28 static struct i915_ttm_buddy_manager *
  29 to_buddy_manager(struct ttm_resource_manager *man)
  30 {
  31         return container_of(man, struct i915_ttm_buddy_manager, manager);
  32 }
  33
  34 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
  35                                     struct ttm_buffer_object *bo,
  36                                     const struct ttm_place *place,
  37                                     struct ttm_resource **res)
  38 {
  39         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
  40         struct i915_ttm_buddy_resource *bman_res;
  41         struct drm_buddy *mm = &bman->mm;
  42         unsigned long n_pages, lpfn;
  43         u64 min_page_size;
  44         u64 size;
  45         int err;
  46
  47         lpfn = place->lpfn;
  48         if (!lpfn)
  49                 lpfn = man->size;
  50
  51         bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL);
  52         if (!bman_res)
  53                 return -ENOMEM;
  54
  55         ttm_resource_init(bo, place, &bman_res->base);
  56         INIT_LIST_HEAD(&bman_res->blocks);
  57         bman_res->mm = mm;
  58
  59         if (place->flags & TTM_PL_FLAG_TOPDOWN)
  60                 bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
  61
  62         if (place->fpfn || lpfn != man->size)
  63                 bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
  64
  65         GEM_BUG_ON(!bman_res->base.size);
  66         size = bman_res->base.size;
  67
  68         min_page_size = bman->default_page_size;
  69         if (bo->page_alignment)
  70                 min_page_size = bo->page_alignment << PAGE_SHIFT;
  71
  72         GEM_BUG_ON(min_page_size < mm->chunk_size);
  73         GEM_BUG_ON(!IS_ALIGNED(size, min_page_size));
  74
  75         if (place->fpfn + PFN_UP(bman_res->base.size) != place->lpfn &&
  76             place->flags & TTM_PL_FLAG_CONTIGUOUS) {
  77                 unsigned long pages;
  78
  79                 size = roundup_pow_of_two(size);
  80                 min_page_size = size;
  81
  82                 pages = size >> ilog2(mm->chunk_size);
  83                 if (pages > lpfn)
  84                         lpfn = pages;
  85         }
  86
  87         if (size > lpfn << PAGE_SHIFT) {
  88                 err = -E2BIG;
  89                 goto err_free_res;
  90         }
  91
  92         n_pages = size >> ilog2(mm->chunk_size);
  93
  94         mutex_lock(&bman->lock);
  95         if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) {
  96                 mutex_unlock(&bman->lock);
  97                 err = -ENOSPC;
  98                 goto err_free_res;
  99         }
 100
 101         err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
 102                                      (u64)lpfn << PAGE_SHIFT,
 103                                      (u64)n_pages << PAGE_SHIFT,
 104                                      min_page_size,
 105                                      &bman_res->blocks,
 106                                      bman_res->flags);
 107         if (unlikely(err))
 108                 goto err_free_blocks;
 109
 110         if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
 111                 u64 original_size = (u64)bman_res->base.size;
 112
 113                 drm_buddy_block_trim(mm,
 114                                      original_size,
 115                                      &bman_res->blocks);
 116         }
 117
 118         if (lpfn <= bman->visible_size) {
 119                 bman_res->used_visible_size = PFN_UP(bman_res->base.size);
 120         } else {
 121                 struct drm_buddy_block *block;
 122
 123                 list_for_each_entry(block, &bman_res->blocks, link) {
 124                         unsigned long start =
 125                                 drm_buddy_block_offset(block) >> PAGE_SHIFT;
 126
 127                         if (start < bman->visible_size) {
 128                                 unsigned long end = start +
 129                                         (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
 130
 131                                 bman_res->used_visible_size +=
 132                                         min(end, bman->visible_size) - start;
 133                         }
 134                 }
 135         }
 136
 137         if (bman_res->used_visible_size)
 138                 bman->visible_avail -= bman_res->used_visible_size;
 139
 140         mutex_unlock(&bman->lock);
 141
 142         *res = &bman_res->base;
 143         return 0;
 144
 145 err_free_blocks:
 146         drm_buddy_free_list(mm, &bman_res->blocks);
 147         mutex_unlock(&bman->lock);
 148 err_free_res:
 149         ttm_resource_fini(man, &bman_res->base);
 150         kfree(bman_res);
 151         return err;
 152 }
 153
 154 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
 155                                     struct ttm_resource *res)
 156 {
 157         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 158         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 159
 160         mutex_lock(&bman->lock);
 161         drm_buddy_free_list(&bman->mm, &bman_res->blocks);
 162         bman->visible_avail += bman_res->used_visible_size;
 163         mutex_unlock(&bman->lock);
 164
 165         ttm_resource_fini(man, res);
 166         kfree(bman_res);
 167 }
 168
 169 static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
 170                                           struct ttm_resource *res,
 171                                           const struct ttm_place *place,
 172                                           size_t size)
 173 {
 174         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 175         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 176         struct drm_buddy *mm = &bman->mm;
 177         struct drm_buddy_block *block;
 178
 179         if (!place->fpfn && !place->lpfn)
 180                 return true;
 181
 182         GEM_BUG_ON(!place->lpfn);
 183
 184         /*
 185          * If we just want something mappable then we can quickly check
 186          * if the current victim resource is using any of the CPU
 187          * visible portion.
 188          */
 189         if (!place->fpfn &&
 190             place->lpfn == i915_ttm_buddy_man_visible_size(man))
 191                 return bman_res->used_visible_size > 0;
 192
 193         /* Check each drm buddy block individually */
 194         list_for_each_entry(block, &bman_res->blocks, link) {
 195                 unsigned long fpfn =
 196                         drm_buddy_block_offset(block) >> PAGE_SHIFT;
 197                 unsigned long lpfn = fpfn +
 198                         (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
 199
 200                 if (place->fpfn < lpfn && place->lpfn > fpfn)
 201                         return true;
 202         }
 203
 204         return false;
 205 }
 206
 207 static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
 208                                           struct ttm_resource *res,
 209                                           const struct ttm_place *place,
 210                                           size_t size)
 211 {
 212         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 213         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 214         struct drm_buddy *mm = &bman->mm;
 215         struct drm_buddy_block *block;
 216
 217         if (!place->fpfn && !place->lpfn)
 218                 return true;
 219
 220         GEM_BUG_ON(!place->lpfn);
 221
 222         if (!place->fpfn &&
 223             place->lpfn == i915_ttm_buddy_man_visible_size(man))
 224                 return bman_res->used_visible_size == PFN_UP(res->size);
 225
 226         /* Check each drm buddy block individually */
 227         list_for_each_entry(block, &bman_res->blocks, link) {
 228                 unsigned long fpfn =
 229                         drm_buddy_block_offset(block) >> PAGE_SHIFT;
 230                 unsigned long lpfn = fpfn +
 231                         (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
 232
 233                 if (fpfn < place->fpfn || lpfn > place->lpfn)
 234                         return false;
 235         }
 236
 237         return true;
 238 }
 239
 240 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
 241                                      struct drm_printer *printer)
 242 {
 243         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 244         struct drm_buddy_block *block;
 245
 246         mutex_lock(&bman->lock);
 247         drm_printf(printer, "default_page_size: %lluKiB\n",
 248                    bman->default_page_size >> 10);
 249         drm_printf(printer, "visible_avail: %lluMiB\n",
 250                    (u64)bman->visible_avail << PAGE_SHIFT >> 20);
 251         drm_printf(printer, "visible_size: %lluMiB\n",
 252                    (u64)bman->visible_size << PAGE_SHIFT >> 20);
 253         drm_printf(printer, "visible_reserved: %lluMiB\n",
 254                    (u64)bman->visible_reserved << PAGE_SHIFT >> 20);
 255
 256         drm_buddy_print(&bman->mm, printer);
 257
 258         drm_printf(printer, "reserved:\n");
 259         list_for_each_entry(block, &bman->reserved, link)
 260                 drm_buddy_block_print(&bman->mm, block, printer);
 261         mutex_unlock(&bman->lock);
 262 }
 263
 264 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = {
 265         .alloc = i915_ttm_buddy_man_alloc,
 266         .free = i915_ttm_buddy_man_free,
 267         .intersects = i915_ttm_buddy_man_intersects,
 268         .compatible = i915_ttm_buddy_man_compatible,
 269         .debug = i915_ttm_buddy_man_debug,
 270 };
 271
 272 /**
 273  * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager
 274  * @bdev: The ttm device
 275  * @type: Memory type we want to manage
 276  * @use_tt: Set use_tt for the manager
 277  * @size: The size in bytes to manage
 278  * @visible_size: The CPU visible size in bytes to manage
 279  * @default_page_size: The default minimum page size in bytes for allocations,
 280  * this must be at least as large as @chunk_size, and can be overridden by
 281  * setting the BO page_alignment, to be larger or smaller as needed.
 282  * @chunk_size: The minimum page size in bytes for our allocations i.e
 283  * order-zero
 284  *
 285  * Note that the starting address is assumed to be zero here, since this
 286  * simplifies keeping the property where allocated blocks having natural
 287  * power-of-two alignment. So long as the real starting address is some large
 288  * power-of-two, or naturally start from zero, then this should be fine.  Also
 289  * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment
 290  * if say there is some unusable range from the start of the region. We can
 291  * revisit this in the future and make the interface accept an actual starting
 292  * offset and let it take care of the rest.
 293  *
 294  * Note that if the @size is not aligned to the @chunk_size then we perform the
 295  * required rounding to get the usable size. The final size in pages can be
 296  * taken from &ttm_resource_manager.size.
 297  *
 298  * Return: 0 on success, negative error code on failure.
 299  */
 300 int i915_ttm_buddy_man_init(struct ttm_device *bdev,
 301                             unsigned int type, bool use_tt,
 302                             u64 size, u64 visible_size, u64 default_page_size,
 303                             u64 chunk_size)
 304 {
 305         struct ttm_resource_manager *man;
 306         struct i915_ttm_buddy_manager *bman;
 307         int err;
 308
 309         bman = kzalloc(sizeof(*bman), GFP_KERNEL);
 310         if (!bman)
 311                 return -ENOMEM;
 312
 313         err = drm_buddy_init(&bman->mm, size, chunk_size);
 314         if (err)
 315                 goto err_free_bman;
 316
 317         mutex_init(&bman->lock);
 318         INIT_LIST_HEAD(&bman->reserved);
 319         GEM_BUG_ON(default_page_size < chunk_size);
 320         bman->default_page_size = default_page_size;
 321         bman->visible_size = visible_size >> PAGE_SHIFT;
 322         bman->visible_avail = bman->visible_size;
 323
 324         man = &bman->manager;
 325         man->use_tt = use_tt;
 326         man->func = &i915_ttm_buddy_manager_func;
 327         ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT);
 328
 329         ttm_resource_manager_set_used(man, true);
 330         ttm_set_driver_manager(bdev, type, man);
 331
 332         return 0;
 333
 334 err_free_bman:
 335         kfree(bman);
 336         return err;
 337 }
 338
 339 /**
 340  * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager
 341  * @bdev: The ttm device
 342  * @type: Memory type we want to manage
 343  *
 344  * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will
 345  * also be freed for us here.
 346  *
 347  * Return: 0 on success, negative error code on failure.
 348  */
 349 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
 350 {
 351         struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
 352         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 353         struct drm_buddy *mm = &bman->mm;
 354         int ret;
 355
 356         ttm_resource_manager_set_used(man, false);
 357
 358         ret = ttm_resource_manager_evict_all(bdev, man);
 359         if (ret)
 360                 return ret;
 361
 362         ttm_set_driver_manager(bdev, type, NULL);
 363
 364         mutex_lock(&bman->lock);
 365         drm_buddy_free_list(mm, &bman->reserved);
 366         drm_buddy_fini(mm);
 367         bman->visible_avail += bman->visible_reserved;
 368         WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
 369         mutex_unlock(&bman->lock);
 370
 371         ttm_resource_manager_cleanup(man);
 372         kfree(bman);
 373
 374         return 0;
 375 }
 376
 377 /**
 378  * i915_ttm_buddy_man_reserve - Reserve address range
 379  * @man: The buddy allocator ttm manager
 380  * @start: The offset in bytes, where the region start is assumed to be zero
 381  * @size: The size in bytes
 382  *
 383  * Note that the starting address for the region is always assumed to be zero.
 384  *
 385  * Return: 0 on success, negative error code on failure.
 386  */
 387 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
 388                                u64 start, u64 size)
 389 {
 390         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 391         struct drm_buddy *mm = &bman->mm;
 392         unsigned long fpfn = start >> PAGE_SHIFT;
 393         unsigned long flags = 0;
 394         int ret;
 395
 396         flags |= DRM_BUDDY_RANGE_ALLOCATION;
 397
 398         mutex_lock(&bman->lock);
 399         ret = drm_buddy_alloc_blocks(mm, start,
 400                                      start + size,
 401                                      size, mm->chunk_size,
 402                                      &bman->reserved,
 403                                      flags);
 404
 405         if (fpfn < bman->visible_size) {
 406                 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT);
 407                 unsigned long visible = min(lpfn, bman->visible_size) - fpfn;
 408
 409                 bman->visible_reserved += visible;
 410                 bman->visible_avail -= visible;
 411         }
 412         mutex_unlock(&bman->lock);
 413
 414         return ret;
 415 }
 416
 417 /**
 418  * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion
 419  * in pages.
 420  * @man: The buddy allocator ttm manager
 421  */
 422 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man)
 423 {
 424         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 425
 426         return bman->visible_size;
 427 }
 428
 429 /**
 430  * i915_ttm_buddy_man_avail - Query the avail tracking for the manager.
 431  *
 432  * @man: The buddy allocator ttm manager
 433  * @avail: The total available memory in pages for the entire manager.
 434  * @visible_avail: The total available memory in pages for the CPU visible
 435  * portion. Note that this will always give the same value as @avail on
 436  * configurations that don't have a small BAR.
 437  */
 438 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man,
 439                               u64 *avail, u64 *visible_avail)
 440 {
 441         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 442
 443         mutex_lock(&bman->lock);
 444         *avail = bman->mm.avail >> PAGE_SHIFT;
 445         *visible_avail = bman->visible_avail;
 446         mutex_unlock(&bman->lock);
 447 }
 448
 449 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 450 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man,
 451                                            u64 size)
 452 {
 453         struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
 454
 455         bman->visible_size = size;
 456 }
 457 #endif