drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020-2021 Intel Corporation
   4  */
   5
   6 #include "gt/intel_migrate.h"
   7 #include "gt/intel_gpu_commands.h"
   8 #include "gem/i915_gem_ttm_move.h"
   9
  10 #include "i915_deps.h"
  11
  12 #include "selftests/igt_reset.h"
  13 #include "selftests/igt_spinner.h"
  14
  15 static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,
  16                                  bool fill)
  17 {
  18         struct drm_i915_private *i915 = to_i915(obj->base.dev);
  19         unsigned int i, count = obj->base.size / sizeof(u32);
  20         enum i915_map_type map_type =
  21                 i915_coherent_map_type(i915, obj, false);
  22         u32 *cur;
  23         int err = 0;
  24
  25         assert_object_held(obj);
  26         cur = i915_gem_object_pin_map(obj, map_type);
  27         if (IS_ERR(cur))
  28                 return PTR_ERR(cur);
  29
  30         if (fill)
  31                 for (i = 0; i < count; ++i)
  32                         *cur++ = i;
  33         else
  34                 for (i = 0; i < count; ++i)
  35                         if (*cur++ != i) {
  36                                 pr_err("Object content mismatch at location %d of %d\n", i, count);
  37                                 err = -EINVAL;
  38                                 break;
  39                         }
  40
  41         i915_gem_object_unpin_map(obj);
  42
  43         return err;
  44 }
  45
  46 static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,
  47                               enum intel_region_id dst)
  48 {
  49         struct drm_i915_private *i915 = gt->i915;
  50         struct intel_memory_region *src_mr = i915->mm.regions[src];
  51         struct intel_memory_region *dst_mr = i915->mm.regions[dst];
  52         struct drm_i915_gem_object *obj;
  53         struct i915_gem_ww_ctx ww;
  54         int err = 0;
  55
  56         GEM_BUG_ON(!src_mr);
  57         GEM_BUG_ON(!dst_mr);
  58
  59         /* Switch object backing-store on create */
  60         obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0);
  61         if (IS_ERR(obj))
  62                 return PTR_ERR(obj);
  63
  64         for_i915_gem_ww(&ww, err, true) {
  65                 err = i915_gem_object_lock(obj, &ww);
  66                 if (err)
  67                         continue;
  68
  69                 err = igt_fill_check_buffer(obj, true);
  70                 if (err)
  71                         continue;
  72
  73                 err = i915_gem_object_migrate(obj, &ww, dst);
  74                 if (err)
  75                         continue;
  76
  77                 err = i915_gem_object_pin_pages(obj);
  78                 if (err)
  79                         continue;
  80
  81                 if (i915_gem_object_can_migrate(obj, src))
  82                         err = -EINVAL;
  83
  84                 i915_gem_object_unpin_pages(obj);
  85                 err = i915_gem_object_wait_migration(obj, true);
  86                 if (err)
  87                         continue;
  88
  89                 err = igt_fill_check_buffer(obj, false);
  90         }
  91         i915_gem_object_put(obj);
  92
  93         return err;
  94 }
  95
  96 static int igt_smem_create_migrate(void *arg)
  97 {
  98         return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM);
  99 }
 100
 101 static int igt_lmem_create_migrate(void *arg)
 102 {
 103         return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0);
 104 }
 105
 106 static int igt_same_create_migrate(void *arg)
 107 {
 108         return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0);
 109 }
 110
 111 static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,
 112                                   struct drm_i915_gem_object *obj,
 113                                   struct i915_vma *vma,
 114                                   bool silent_migrate)
 115 {
 116         int err;
 117
 118         err = i915_gem_object_lock(obj, ww);
 119         if (err)
 120                 return err;
 121
 122         if (vma) {
 123                 err = i915_vma_pin_ww(vma, ww, obj->base.size, 0,
 124                                       0UL | PIN_OFFSET_FIXED |
 125                                       PIN_USER);
 126                 if (err) {
 127                         if (err != -EINTR && err != ERESTARTSYS &&
 128                             err != -EDEADLK)
 129                                 pr_err("Failed to pin vma.\n");
 130                         return err;
 131                 }
 132
 133                 i915_vma_unpin(vma);
 134         }
 135
 136         /*
 137          * Migration will implicitly unbind (asynchronously) any bound
 138          * vmas.
 139          */
 140         if (i915_gem_object_is_lmem(obj)) {
 141                 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM);
 142                 if (err) {
 143                         if (!silent_migrate)
 144                                 pr_err("Object failed migration to smem\n");
 145                         if (err)
 146                                 return err;
 147                 }
 148
 149                 if (i915_gem_object_is_lmem(obj)) {
 150                         pr_err("object still backed by lmem\n");
 151                         err = -EINVAL;
 152                 }
 153
 154                 if (!i915_gem_object_has_struct_page(obj)) {
 155                         pr_err("object not backed by struct page\n");
 156                         err = -EINVAL;
 157                 }
 158
 159         } else {
 160                 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0);
 161                 if (err) {
 162                         if (!silent_migrate)
 163                                 pr_err("Object failed migration to lmem\n");
 164                         if (err)
 165                                 return err;
 166                 }
 167
 168                 if (i915_gem_object_has_struct_page(obj)) {
 169                         pr_err("object still backed by struct page\n");
 170                         err = -EINVAL;
 171                 }
 172
 173                 if (!i915_gem_object_is_lmem(obj)) {
 174                         pr_err("object not backed by lmem\n");
 175                         err = -EINVAL;
 176                 }
 177         }
 178
 179         return err;
 180 }
 181
 182 static int __igt_lmem_pages_migrate(struct intel_gt *gt,
 183                                     struct i915_address_space *vm,
 184                                     struct i915_deps *deps,
 185                                     struct igt_spinner *spin,
 186                                     struct dma_fence *spin_fence,
 187                                     bool borked_migrate)
 188 {
 189         struct drm_i915_private *i915 = gt->i915;
 190         struct drm_i915_gem_object *obj;
 191         struct i915_vma *vma = NULL;
 192         struct i915_gem_ww_ctx ww;
 193         struct i915_request *rq;
 194         int err;
 195         int i;
 196
 197         /* From LMEM to shmem and back again */
 198
 199         obj = i915_gem_object_create_lmem(i915, SZ_2M, 0);
 200         if (IS_ERR(obj))
 201                 return PTR_ERR(obj);
 202
 203         if (vm) {
 204                 vma = i915_vma_instance(obj, vm, NULL);
 205                 if (IS_ERR(vma)) {
 206                         err = PTR_ERR(vma);
 207                         goto out_put;
 208                 }
 209         }
 210
 211         /* Initial GPU fill, sync, CPU initialization. */
 212         for_i915_gem_ww(&ww, err, true) {
 213                 err = i915_gem_object_lock(obj, &ww);
 214                 if (err)
 215                         continue;
 216
 217                 err = ____i915_gem_object_get_pages(obj);
 218                 if (err)
 219                         continue;
 220
 221                 err = intel_migrate_clear(&gt->migrate, &ww, deps,
 222                                           obj->mm.pages->sgl, obj->pat_index,
 223                                           i915_gem_object_is_lmem(obj),
 224                                           0xdeadbeaf, &rq);
 225                 if (rq) {
 226                         err = dma_resv_reserve_fences(obj->base.resv, 1);
 227                         if (!err)
 228                                 dma_resv_add_fence(obj->base.resv, &rq->fence,
 229                                                    DMA_RESV_USAGE_KERNEL);
 230                         i915_request_put(rq);
 231                 }
 232                 if (err)
 233                         continue;
 234
 235                 if (!vma) {
 236                         err = igt_fill_check_buffer(obj, true);
 237                         if (err)
 238                                 continue;
 239                 }
 240         }
 241         if (err)
 242                 goto out_put;
 243
 244         /*
 245          * Migrate to and from smem without explicitly syncing.
 246          * Finalize with data in smem for fast readout.
 247          */
 248         for (i = 1; i <= 5; ++i) {
 249                 for_i915_gem_ww(&ww, err, true)
 250                         err = lmem_pages_migrate_one(&ww, obj, vma,
 251                                                      borked_migrate);
 252                 if (err)
 253                         goto out_put;
 254         }
 255
 256         err = i915_gem_object_lock_interruptible(obj, NULL);
 257         if (err)
 258                 goto out_put;
 259
 260         if (spin) {
 261                 if (dma_fence_is_signaled(spin_fence)) {
 262                         pr_err("Spinner was terminated by hangcheck.\n");
 263                         err = -EBUSY;
 264                         goto out_unlock;
 265                 }
 266                 igt_spinner_end(spin);
 267         }
 268
 269         /* Finally sync migration and check content. */
 270         err = i915_gem_object_wait_migration(obj, true);
 271         if (err)
 272                 goto out_unlock;
 273
 274         if (vma) {
 275                 err = i915_vma_wait_for_bind(vma);
 276                 if (err)
 277                         goto out_unlock;
 278         } else {
 279                 err = igt_fill_check_buffer(obj, false);
 280         }
 281
 282 out_unlock:
 283         i915_gem_object_unlock(obj);
 284 out_put:
 285         i915_gem_object_put(obj);
 286
 287         return err;
 288 }
 289
 290 static int igt_lmem_pages_failsafe_migrate(void *arg)
 291 {
 292         int fail_gpu, fail_alloc, ban_memcpy, ret;
 293         struct intel_gt *gt = arg;
 294
 295         for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
 296                 for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {
 297                         for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
 298                                 pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n",
 299                                         fail_gpu, fail_alloc, ban_memcpy);
 300                                 i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
 301                                 i915_ttm_migrate_set_failure_modes(fail_gpu,
 302                                                                    fail_alloc);
 303                                 ret = __igt_lmem_pages_migrate(gt, NULL, NULL,
 304                                                                NULL, NULL,
 305                                                                ban_memcpy &&
 306                                                                fail_gpu);
 307
 308                                 if (ban_memcpy && fail_gpu) {
 309                                         struct intel_gt *__gt;
 310                                         unsigned int id;
 311
 312                                         if (ret != -EIO) {
 313                                                 pr_err("expected -EIO, got (%d)\n", ret);
 314                                                 ret = -EINVAL;
 315                                         } else {
 316                                                 ret = 0;
 317                                         }
 318
 319                                         for_each_gt(__gt, gt->i915, id) {
 320                                                 intel_wakeref_t wakeref;
 321                                                 bool wedged;
 322
 323                                                 mutex_lock(&__gt->reset.mutex);
 324                                                 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
 325                                                 mutex_unlock(&__gt->reset.mutex);
 326
 327                                                 if (fail_gpu && !fail_alloc) {
 328                                                         if (!wedged) {
 329                                                                 pr_err("gt(%u) not wedged\n", id);
 330                                                                 ret = -EINVAL;
 331                                                                 continue;
 332                                                         }
 333                                                 } else if (wedged) {
 334                                                         pr_err("gt(%u) incorrectly wedged\n", id);
 335                                                         ret = -EINVAL;
 336                                                 } else {
 337                                                         continue;
 338                                                 }
 339
 340                                                 wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
 341                                                 igt_global_reset_lock(__gt);
 342                                                 intel_gt_reset(__gt, ALL_ENGINES, NULL);
 343                                                 igt_global_reset_unlock(__gt);
 344                                                 intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
 345                                         }
 346                                         if (ret)
 347                                                 goto out_err;
 348                                 }
 349                         }
 350                 }
 351         }
 352
 353 out_err:
 354         i915_ttm_migrate_set_failure_modes(false, false);
 355         i915_ttm_migrate_set_ban_memcpy(false);
 356         return ret;
 357 }
 358
 359 /*
 360  * This subtest tests that unbinding at migration is indeed performed
 361  * async. We launch a spinner and a number of migrations depending on
 362  * that spinner to have terminated. Before each migration we bind a
 363  * vma, which should then be async unbound by the migration operation.
 364  * If we are able to schedule migrations without blocking while the
 365  * spinner is still running, those unbinds are indeed async and non-
 366  * blocking.
 367  *
 368  * Note that each async bind operation is awaiting the previous migration
 369  * due to the moving fence resulting from the migration.
 370  */
 371 static int igt_async_migrate(struct intel_gt *gt)
 372 {
 373         struct intel_engine_cs *engine;
 374         enum intel_engine_id id;
 375         struct i915_ppgtt *ppgtt;
 376         struct igt_spinner spin;
 377         int err;
 378
 379         ppgtt = i915_ppgtt_create(gt, 0);
 380         if (IS_ERR(ppgtt))
 381                 return PTR_ERR(ppgtt);
 382
 383         if (igt_spinner_init(&spin, gt)) {
 384                 err = -ENOMEM;
 385                 goto out_spin;
 386         }
 387
 388         for_each_engine(engine, gt, id) {
 389                 struct ttm_operation_ctx ctx = {
 390                         .interruptible = true
 391                 };
 392                 struct dma_fence *spin_fence;
 393                 struct intel_context *ce;
 394                 struct i915_request *rq;
 395                 struct i915_deps deps;
 396
 397                 ce = intel_context_create(engine);
 398                 if (IS_ERR(ce)) {
 399                         err = PTR_ERR(ce);
 400                         goto out_ce;
 401                 }
 402
 403                 /*
 404                  * Use MI_NOOP, making the spinner non-preemptible. If there
 405                  * is a code path where we fail async operation due to the
 406                  * running spinner, we will block and fail to end the
 407                  * spinner resulting in a deadlock. But with a non-
 408                  * preemptible spinner, hangcheck will terminate the spinner
 409                  * for us, and we will later detect that and fail the test.
 410                  */
 411                 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
 412                 intel_context_put(ce);
 413                 if (IS_ERR(rq)) {
 414                         err = PTR_ERR(rq);
 415                         goto out_ce;
 416                 }
 417
 418                 i915_deps_init(&deps, GFP_KERNEL);
 419                 err = i915_deps_add_dependency(&deps, &rq->fence, &ctx);
 420                 spin_fence = dma_fence_get(&rq->fence);
 421                 i915_request_add(rq);
 422                 if (err)
 423                         goto out_ce;
 424
 425                 err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin,
 426                                                spin_fence, false);
 427                 i915_deps_fini(&deps);
 428                 dma_fence_put(spin_fence);
 429                 if (err)
 430                         goto out_ce;
 431         }
 432
 433 out_ce:
 434         igt_spinner_fini(&spin);
 435 out_spin:
 436         i915_vm_put(&ppgtt->vm);
 437
 438         return err;
 439 }
 440
 441 /*
 442  * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while
 443  * arming the migration error check and block async migration. This
 444  * will cause us to deadlock and hangcheck will terminate the spinner
 445  * causing the test to fail.
 446  */
 447 #define ASYNC_FAIL_ALLOC 1
 448 static int igt_lmem_async_migrate(void *arg)
 449 {
 450         int fail_gpu, fail_alloc, ban_memcpy, ret;
 451         struct intel_gt *gt = arg;
 452
 453         for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
 454                 for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {
 455                         for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
 456                                 pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n",
 457                                         fail_gpu, fail_alloc, ban_memcpy);
 458                                 i915_ttm_migrate_set_ban_memcpy(ban_memcpy);
 459                                 i915_ttm_migrate_set_failure_modes(fail_gpu,
 460                                                                    fail_alloc);
 461                                 ret = igt_async_migrate(gt);
 462
 463                                 if (fail_gpu && ban_memcpy) {
 464                                         struct intel_gt *__gt;
 465                                         unsigned int id;
 466
 467                                         if (ret != -EIO) {
 468                                                 pr_err("expected -EIO, got (%d)\n", ret);
 469                                                 ret = -EINVAL;
 470                                         } else {
 471                                                 ret = 0;
 472                                         }
 473
 474                                         for_each_gt(__gt, gt->i915, id) {
 475                                                 intel_wakeref_t wakeref;
 476                                                 bool wedged;
 477
 478                                                 mutex_lock(&__gt->reset.mutex);
 479                                                 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
 480                                                 mutex_unlock(&__gt->reset.mutex);
 481
 482                                                 if (fail_gpu && !fail_alloc) {
 483                                                         if (!wedged) {
 484                                                                 pr_err("gt(%u) not wedged\n", id);
 485                                                                 ret = -EINVAL;
 486                                                                 continue;
 487                                                         }
 488                                                 } else if (wedged) {
 489                                                         pr_err("gt(%u) incorrectly wedged\n", id);
 490                                                         ret = -EINVAL;
 491                                                 } else {
 492                                                         continue;
 493                                                 }
 494
 495                                                 wakeref = intel_runtime_pm_get(__gt->uncore->rpm);
 496                                                 igt_global_reset_lock(__gt);
 497                                                 intel_gt_reset(__gt, ALL_ENGINES, NULL);
 498                                                 igt_global_reset_unlock(__gt);
 499                                                 intel_runtime_pm_put(__gt->uncore->rpm, wakeref);
 500                                         }
 501                                 }
 502                                 if (ret)
 503                                         goto out_err;
 504                         }
 505                 }
 506         }
 507
 508 out_err:
 509         i915_ttm_migrate_set_failure_modes(false, false);
 510         i915_ttm_migrate_set_ban_memcpy(false);
 511         return ret;
 512 }
 513
 514 int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)
 515 {
 516         static const struct i915_subtest tests[] = {
 517                 SUBTEST(igt_smem_create_migrate),
 518                 SUBTEST(igt_lmem_create_migrate),
 519                 SUBTEST(igt_same_create_migrate),
 520                 SUBTEST(igt_lmem_pages_failsafe_migrate),
 521                 SUBTEST(igt_lmem_async_migrate),
 522         };
 523
 524         if (!HAS_LMEM(i915))
 525                 return 0;
 526
 527         return intel_gt_live_subtests(tests, to_gt(i915));
 528 }