drivers/gpu/drm/i915/gt/selftest_workarounds.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_internal.h"
   7 #include "gem/i915_gem_pm.h"
   8 #include "gt/intel_engine_user.h"
   9 #include "gt/intel_gt.h"
  10 #include "i915_selftest.h"
  11 #include "intel_reset.h"
  12
  13 #include "selftests/igt_flush_test.h"
  14 #include "selftests/igt_reset.h"
  15 #include "selftests/igt_spinner.h"
  16 #include "selftests/intel_scheduler_helpers.h"
  17 #include "selftests/mock_drm.h"
  18
  19 #include "gem/selftests/igt_gem_utils.h"
  20 #include "gem/selftests/mock_context.h"
  21
  22 static const struct wo_register {
  23         enum intel_platform platform;
  24         u32 reg;
  25 } wo_registers[] = {
  26         { INTEL_GEMINILAKE, 0x731c }
  27 };
  28
  29 struct wa_lists {
  30         struct i915_wa_list gt_wa_list;
  31         struct {
  32                 struct i915_wa_list wa_list;
  33                 struct i915_wa_list ctx_wa_list;
  34         } engine[I915_NUM_ENGINES];
  35 };
  36
  37 static int request_add_sync(struct i915_request *rq, int err)
  38 {
  39         i915_request_get(rq);
  40         i915_request_add(rq);
  41         if (i915_request_wait(rq, 0, HZ / 5) < 0)
  42                 err = -EIO;
  43         i915_request_put(rq);
  44
  45         return err;
  46 }
  47
  48 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
  49 {
  50         int err = 0;
  51
  52         i915_request_get(rq);
  53         i915_request_add(rq);
  54         if (spin && !igt_wait_for_spinner(spin, rq))
  55                 err = -ETIMEDOUT;
  56         i915_request_put(rq);
  57
  58         return err;
  59 }
  60
  61 static void
  62 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
  63 {
  64         struct intel_engine_cs *engine;
  65         enum intel_engine_id id;
  66
  67         memset(lists, 0, sizeof(*lists));
  68
  69         wa_init_start(&lists->gt_wa_list, gt, "GT_REF", "global");
  70         gt_init_workarounds(gt, &lists->gt_wa_list);
  71         wa_init_finish(&lists->gt_wa_list);
  72
  73         for_each_engine(engine, gt, id) {
  74                 struct i915_wa_list *wal = &lists->engine[id].wa_list;
  75
  76                 wa_init_start(wal, gt, "REF", engine->name);
  77                 engine_init_workarounds(engine, wal);
  78                 wa_init_finish(wal);
  79
  80                 __intel_engine_init_ctx_wa(engine,
  81                                            &lists->engine[id].ctx_wa_list,
  82                                            "CTX_REF");
  83         }
  84 }
  85
  86 static void
  87 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
  88 {
  89         struct intel_engine_cs *engine;
  90         enum intel_engine_id id;
  91
  92         for_each_engine(engine, gt, id)
  93                 intel_wa_list_free(&lists->engine[id].wa_list);
  94
  95         intel_wa_list_free(&lists->gt_wa_list);
  96 }
  97
  98 static struct drm_i915_gem_object *
  99 read_nonprivs(struct intel_context *ce)
 100 {
 101         struct intel_engine_cs *engine = ce->engine;
 102         const u32 base = engine->mmio_base;
 103         struct drm_i915_gem_object *result;
 104         struct i915_request *rq;
 105         struct i915_vma *vma;
 106         u32 srm, *cs;
 107         int err;
 108         int i;
 109
 110         result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 111         if (IS_ERR(result))
 112                 return result;
 113
 114         i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
 115
 116         cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
 117         if (IS_ERR(cs)) {
 118                 err = PTR_ERR(cs);
 119                 goto err_obj;
 120         }
 121         memset(cs, 0xc5, PAGE_SIZE);
 122         i915_gem_object_flush_map(result);
 123         i915_gem_object_unpin_map(result);
 124
 125         vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
 126         if (IS_ERR(vma)) {
 127                 err = PTR_ERR(vma);
 128                 goto err_obj;
 129         }
 130
 131         err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 132         if (err)
 133                 goto err_obj;
 134
 135         rq = intel_context_create_request(ce);
 136         if (IS_ERR(rq)) {
 137                 err = PTR_ERR(rq);
 138                 goto err_pin;
 139         }
 140
 141         err = igt_vma_move_to_active_unlocked(vma, rq, EXEC_OBJECT_WRITE);
 142         if (err)
 143                 goto err_req;
 144
 145         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 146         if (GRAPHICS_VER(engine->i915) >= 8)
 147                 srm++;
 148
 149         cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
 150         if (IS_ERR(cs)) {
 151                 err = PTR_ERR(cs);
 152                 goto err_req;
 153         }
 154
 155         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 156                 *cs++ = srm;
 157                 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
 158                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
 159                 *cs++ = 0;
 160         }
 161         intel_ring_advance(rq, cs);
 162
 163         i915_request_add(rq);
 164         i915_vma_unpin(vma);
 165
 166         return result;
 167
 168 err_req:
 169         i915_request_add(rq);
 170 err_pin:
 171         i915_vma_unpin(vma);
 172 err_obj:
 173         i915_gem_object_put(result);
 174         return ERR_PTR(err);
 175 }
 176
 177 static u32
 178 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
 179 {
 180         i915_reg_t reg = i < engine->whitelist.count ?
 181                          engine->whitelist.list[i].reg :
 182                          RING_NOPID(engine->mmio_base);
 183
 184         return i915_mmio_reg_offset(reg);
 185 }
 186
 187 static void
 188 print_results(const struct intel_engine_cs *engine, const u32 *results)
 189 {
 190         unsigned int i;
 191
 192         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 193                 u32 expected = get_whitelist_reg(engine, i);
 194                 u32 actual = results[i];
 195
 196                 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
 197                         i, expected, actual);
 198         }
 199 }
 200
 201 static int check_whitelist(struct intel_context *ce)
 202 {
 203         struct intel_engine_cs *engine = ce->engine;
 204         struct drm_i915_gem_object *results;
 205         struct intel_wedge_me wedge;
 206         u32 *vaddr;
 207         int err;
 208         int i;
 209
 210         results = read_nonprivs(ce);
 211         if (IS_ERR(results))
 212                 return PTR_ERR(results);
 213
 214         err = 0;
 215         i915_gem_object_lock(results, NULL);
 216         intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
 217                 err = i915_gem_object_set_to_cpu_domain(results, false);
 218
 219         if (intel_gt_is_wedged(engine->gt))
 220                 err = -EIO;
 221         if (err)
 222                 goto out_put;
 223
 224         vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
 225         if (IS_ERR(vaddr)) {
 226                 err = PTR_ERR(vaddr);
 227                 goto out_put;
 228         }
 229
 230         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 231                 u32 expected = get_whitelist_reg(engine, i);
 232                 u32 actual = vaddr[i];
 233
 234                 if (expected != actual) {
 235                         print_results(engine, vaddr);
 236                         pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
 237                                i, expected, actual);
 238
 239                         err = -EINVAL;
 240                         break;
 241                 }
 242         }
 243
 244         i915_gem_object_unpin_map(results);
 245 out_put:
 246         i915_gem_object_unlock(results);
 247         i915_gem_object_put(results);
 248         return err;
 249 }
 250
 251 static int do_device_reset(struct intel_engine_cs *engine)
 252 {
 253         intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
 254         return 0;
 255 }
 256
 257 static int do_engine_reset(struct intel_engine_cs *engine)
 258 {
 259         return intel_engine_reset(engine, "live_workarounds");
 260 }
 261
 262 static int do_guc_reset(struct intel_engine_cs *engine)
 263 {
 264         /* Currently a no-op as the reset is handled by GuC */
 265         return 0;
 266 }
 267
 268 static int
 269 switch_to_scratch_context(struct intel_engine_cs *engine,
 270                           struct igt_spinner *spin,
 271                           struct i915_request **rq)
 272 {
 273         struct intel_context *ce;
 274         int err = 0;
 275
 276         ce = intel_context_create(engine);
 277         if (IS_ERR(ce))
 278                 return PTR_ERR(ce);
 279
 280         *rq = igt_spinner_create_request(spin, ce, MI_NOOP);
 281         intel_context_put(ce);
 282
 283         if (IS_ERR(*rq)) {
 284                 spin = NULL;
 285                 err = PTR_ERR(*rq);
 286                 goto err;
 287         }
 288
 289         err = request_add_spin(*rq, spin);
 290 err:
 291         if (err && spin)
 292                 igt_spinner_end(spin);
 293
 294         return err;
 295 }
 296
 297 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 298                                         int (*reset)(struct intel_engine_cs *),
 299                                         const char *name)
 300 {
 301         struct intel_context *ce, *tmp;
 302         struct igt_spinner spin;
 303         struct i915_request *rq;
 304         intel_wakeref_t wakeref;
 305         int err;
 306
 307         pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
 308                 engine->whitelist.count, engine->name, name);
 309
 310         ce = intel_context_create(engine);
 311         if (IS_ERR(ce))
 312                 return PTR_ERR(ce);
 313
 314         err = igt_spinner_init(&spin, engine->gt);
 315         if (err)
 316                 goto out_ctx;
 317
 318         err = check_whitelist(ce);
 319         if (err) {
 320                 pr_err("Invalid whitelist *before* %s reset!\n", name);
 321                 goto out_spin;
 322         }
 323
 324         err = switch_to_scratch_context(engine, &spin, &rq);
 325         if (err)
 326                 goto out_spin;
 327
 328         /* Ensure the spinner hasn't aborted */
 329         if (i915_request_completed(rq)) {
 330                 pr_err("%s spinner failed to start\n", name);
 331                 err = -ETIMEDOUT;
 332                 goto out_spin;
 333         }
 334
 335         with_intel_runtime_pm(engine->uncore->rpm, wakeref)
 336                 err = reset(engine);
 337
 338         /* Ensure the reset happens and kills the engine */
 339         if (err == 0)
 340                 err = intel_selftest_wait_for_rq(rq);
 341
 342         igt_spinner_end(&spin);
 343
 344         if (err) {
 345                 pr_err("%s reset failed\n", name);
 346                 goto out_spin;
 347         }
 348
 349         err = check_whitelist(ce);
 350         if (err) {
 351                 pr_err("Whitelist not preserved in context across %s reset!\n",
 352                        name);
 353                 goto out_spin;
 354         }
 355
 356         tmp = intel_context_create(engine);
 357         if (IS_ERR(tmp)) {
 358                 err = PTR_ERR(tmp);
 359                 goto out_spin;
 360         }
 361         intel_context_put(ce);
 362         ce = tmp;
 363
 364         err = check_whitelist(ce);
 365         if (err) {
 366                 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
 367                        name);
 368                 goto out_spin;
 369         }
 370
 371 out_spin:
 372         igt_spinner_fini(&spin);
 373 out_ctx:
 374         intel_context_put(ce);
 375         return err;
 376 }
 377
 378 static struct i915_vma *create_batch(struct i915_address_space *vm)
 379 {
 380         struct drm_i915_gem_object *obj;
 381         struct i915_vma *vma;
 382         int err;
 383
 384         obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
 385         if (IS_ERR(obj))
 386                 return ERR_CAST(obj);
 387
 388         vma = i915_vma_instance(obj, vm, NULL);
 389         if (IS_ERR(vma)) {
 390                 err = PTR_ERR(vma);
 391                 goto err_obj;
 392         }
 393
 394         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 395         if (err)
 396                 goto err_obj;
 397
 398         return vma;
 399
 400 err_obj:
 401         i915_gem_object_put(obj);
 402         return ERR_PTR(err);
 403 }
 404
 405 static u32 reg_write(u32 old, u32 new, u32 rsvd)
 406 {
 407         if (rsvd == 0x0000ffff) {
 408                 old &= ~(new >> 16);
 409                 old |= new & (new >> 16);
 410         } else {
 411                 old &= ~rsvd;
 412                 old |= new & rsvd;
 413         }
 414
 415         return old;
 416 }
 417
 418 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
 419 {
 420         enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
 421         int i;
 422
 423         if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 424              RING_FORCE_TO_NONPRIV_ACCESS_WR)
 425                 return true;
 426
 427         for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
 428                 if (wo_registers[i].platform == platform &&
 429                     wo_registers[i].reg == reg)
 430                         return true;
 431         }
 432
 433         return false;
 434 }
 435
 436 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
 437 {
 438         reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
 439         switch (reg) {
 440         case 0x358:
 441         case 0x35c:
 442         case 0x3a8:
 443                 return true;
 444
 445         default:
 446                 return false;
 447         }
 448 }
 449
 450 static bool ro_register(u32 reg)
 451 {
 452         if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 453              RING_FORCE_TO_NONPRIV_ACCESS_RD)
 454                 return true;
 455
 456         return false;
 457 }
 458
 459 static int whitelist_writable_count(struct intel_engine_cs *engine)
 460 {
 461         int count = engine->whitelist.count;
 462         int i;
 463
 464         for (i = 0; i < engine->whitelist.count; i++) {
 465                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 466
 467                 if (ro_register(reg))
 468                         count--;
 469         }
 470
 471         return count;
 472 }
 473
 474 static int check_dirty_whitelist(struct intel_context *ce)
 475 {
 476         const u32 values[] = {
 477                 0x00000000,
 478                 0x01010101,
 479                 0x10100101,
 480                 0x03030303,
 481                 0x30300303,
 482                 0x05050505,
 483                 0x50500505,
 484                 0x0f0f0f0f,
 485                 0xf00ff00f,
 486                 0x10101010,
 487                 0xf0f01010,
 488                 0x30303030,
 489                 0xa0a03030,
 490                 0x50505050,
 491                 0xc0c05050,
 492                 0xf0f0f0f0,
 493                 0x11111111,
 494                 0x33333333,
 495                 0x55555555,
 496                 0x0000ffff,
 497                 0x00ff00ff,
 498                 0xff0000ff,
 499                 0xffff00ff,
 500                 0xffffffff,
 501         };
 502         struct intel_engine_cs *engine = ce->engine;
 503         struct i915_vma *scratch;
 504         struct i915_vma *batch;
 505         int err = 0, i, v, sz;
 506         u32 *cs, *results;
 507
 508         sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
 509         scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
 510         if (IS_ERR(scratch))
 511                 return PTR_ERR(scratch);
 512
 513         batch = create_batch(ce->vm);
 514         if (IS_ERR(batch)) {
 515                 err = PTR_ERR(batch);
 516                 goto out_scratch;
 517         }
 518
 519         for (i = 0; i < engine->whitelist.count; i++) {
 520                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 521                 struct i915_gem_ww_ctx ww;
 522                 u64 addr = i915_vma_offset(scratch);
 523                 struct i915_request *rq;
 524                 u32 srm, lrm, rsvd;
 525                 u32 expect;
 526                 int idx;
 527                 bool ro_reg;
 528
 529                 if (wo_register(engine, reg))
 530                         continue;
 531
 532                 if (timestamp(engine, reg))
 533                         continue; /* timestamps are expected to autoincrement */
 534
 535                 ro_reg = ro_register(reg);
 536
 537                 i915_gem_ww_ctx_init(&ww, false);
 538 retry:
 539                 cs = NULL;
 540                 err = i915_gem_object_lock(scratch->obj, &ww);
 541                 if (!err)
 542                         err = i915_gem_object_lock(batch->obj, &ww);
 543                 if (!err)
 544                         err = intel_context_pin_ww(ce, &ww);
 545                 if (err)
 546                         goto out;
 547
 548                 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 549                 if (IS_ERR(cs)) {
 550                         err = PTR_ERR(cs);
 551                         goto out_ctx;
 552                 }
 553
 554                 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 555                 if (IS_ERR(results)) {
 556                         err = PTR_ERR(results);
 557                         goto out_unmap_batch;
 558                 }
 559
 560                 /* Clear non priv flags */
 561                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 562
 563                 srm = MI_STORE_REGISTER_MEM;
 564                 lrm = MI_LOAD_REGISTER_MEM;
 565                 if (GRAPHICS_VER(engine->i915) >= 8)
 566                         lrm++, srm++;
 567
 568                 pr_debug("%s: Writing garbage to %x\n",
 569                          engine->name, reg);
 570
 571                 /* SRM original */
 572                 *cs++ = srm;
 573                 *cs++ = reg;
 574                 *cs++ = lower_32_bits(addr);
 575                 *cs++ = upper_32_bits(addr);
 576
 577                 idx = 1;
 578                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 579                         /* LRI garbage */
 580                         *cs++ = MI_LOAD_REGISTER_IMM(1);
 581                         *cs++ = reg;
 582                         *cs++ = values[v];
 583
 584                         /* SRM result */
 585                         *cs++ = srm;
 586                         *cs++ = reg;
 587                         *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 588                         *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 589                         idx++;
 590                 }
 591                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 592                         /* LRI garbage */
 593                         *cs++ = MI_LOAD_REGISTER_IMM(1);
 594                         *cs++ = reg;
 595                         *cs++ = ~values[v];
 596
 597                         /* SRM result */
 598                         *cs++ = srm;
 599                         *cs++ = reg;
 600                         *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 601                         *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 602                         idx++;
 603                 }
 604                 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
 605
 606                 /* LRM original -- don't leave garbage in the context! */
 607                 *cs++ = lrm;
 608                 *cs++ = reg;
 609                 *cs++ = lower_32_bits(addr);
 610                 *cs++ = upper_32_bits(addr);
 611
 612                 *cs++ = MI_BATCH_BUFFER_END;
 613
 614                 i915_gem_object_flush_map(batch->obj);
 615                 i915_gem_object_unpin_map(batch->obj);
 616                 intel_gt_chipset_flush(engine->gt);
 617                 cs = NULL;
 618
 619                 rq = i915_request_create(ce);
 620                 if (IS_ERR(rq)) {
 621                         err = PTR_ERR(rq);
 622                         goto out_unmap_scratch;
 623                 }
 624
 625                 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 626                         err = engine->emit_init_breadcrumb(rq);
 627                         if (err)
 628                                 goto err_request;
 629                 }
 630
 631                 err = i915_vma_move_to_active(batch, rq, 0);
 632                 if (err)
 633                         goto err_request;
 634
 635                 err = i915_vma_move_to_active(scratch, rq,
 636                                               EXEC_OBJECT_WRITE);
 637                 if (err)
 638                         goto err_request;
 639
 640                 err = engine->emit_bb_start(rq,
 641                                             i915_vma_offset(batch), PAGE_SIZE,
 642                                             0);
 643                 if (err)
 644                         goto err_request;
 645
 646 err_request:
 647                 err = request_add_sync(rq, err);
 648                 if (err) {
 649                         pr_err("%s: Futzing %x timedout; cancelling test\n",
 650                                engine->name, reg);
 651                         intel_gt_set_wedged(engine->gt);
 652                         goto out_unmap_scratch;
 653                 }
 654
 655                 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
 656                 if (!ro_reg) {
 657                         /* detect write masking */
 658                         rsvd = results[ARRAY_SIZE(values)];
 659                         if (!rsvd) {
 660                                 pr_err("%s: Unable to write to whitelisted register %x\n",
 661                                        engine->name, reg);
 662                                 err = -EINVAL;
 663                                 goto out_unmap_scratch;
 664                         }
 665                 } else {
 666                         rsvd = 0;
 667                 }
 668
 669                 expect = results[0];
 670                 idx = 1;
 671                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 672                         if (ro_reg)
 673                                 expect = results[0];
 674                         else
 675                                 expect = reg_write(expect, values[v], rsvd);
 676
 677                         if (results[idx] != expect)
 678                                 err++;
 679                         idx++;
 680                 }
 681                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 682                         if (ro_reg)
 683                                 expect = results[0];
 684                         else
 685                                 expect = reg_write(expect, ~values[v], rsvd);
 686
 687                         if (results[idx] != expect)
 688                                 err++;
 689                         idx++;
 690                 }
 691                 if (err) {
 692                         pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
 693                                engine->name, err, reg);
 694
 695                         if (ro_reg)
 696                                 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
 697                                         engine->name, reg, results[0]);
 698                         else
 699                                 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
 700                                         engine->name, reg, results[0], rsvd);
 701
 702                         expect = results[0];
 703                         idx = 1;
 704                         for (v = 0; v < ARRAY_SIZE(values); v++) {
 705                                 u32 w = values[v];
 706
 707                                 if (ro_reg)
 708                                         expect = results[0];
 709                                 else
 710                                         expect = reg_write(expect, w, rsvd);
 711                                 pr_info("Wrote %08x, read %08x, expect %08x\n",
 712                                         w, results[idx], expect);
 713                                 idx++;
 714                         }
 715                         for (v = 0; v < ARRAY_SIZE(values); v++) {
 716                                 u32 w = ~values[v];
 717
 718                                 if (ro_reg)
 719                                         expect = results[0];
 720                                 else
 721                                         expect = reg_write(expect, w, rsvd);
 722                                 pr_info("Wrote %08x, read %08x, expect %08x\n",
 723                                         w, results[idx], expect);
 724                                 idx++;
 725                         }
 726
 727                         err = -EINVAL;
 728                 }
 729 out_unmap_scratch:
 730                 i915_gem_object_unpin_map(scratch->obj);
 731 out_unmap_batch:
 732                 if (cs)
 733                         i915_gem_object_unpin_map(batch->obj);
 734 out_ctx:
 735                 intel_context_unpin(ce);
 736 out:
 737                 if (err == -EDEADLK) {
 738                         err = i915_gem_ww_ctx_backoff(&ww);
 739                         if (!err)
 740                                 goto retry;
 741                 }
 742                 i915_gem_ww_ctx_fini(&ww);
 743                 if (err)
 744                         break;
 745         }
 746
 747         if (igt_flush_test(engine->i915))
 748                 err = -EIO;
 749
 750         i915_vma_unpin_and_release(&batch, 0);
 751 out_scratch:
 752         i915_vma_unpin_and_release(&scratch, 0);
 753         return err;
 754 }
 755
 756 static int live_dirty_whitelist(void *arg)
 757 {
 758         struct intel_gt *gt = arg;
 759         struct intel_engine_cs *engine;
 760         enum intel_engine_id id;
 761
 762         /* Can the user write to the whitelisted registers? */
 763
 764         if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
 765                 return 0;
 766
 767         for_each_engine(engine, gt, id) {
 768                 struct intel_context *ce;
 769                 int err;
 770
 771                 if (engine->whitelist.count == 0)
 772                         continue;
 773
 774                 ce = intel_context_create(engine);
 775                 if (IS_ERR(ce))
 776                         return PTR_ERR(ce);
 777
 778                 err = check_dirty_whitelist(ce);
 779                 intel_context_put(ce);
 780                 if (err)
 781                         return err;
 782         }
 783
 784         return 0;
 785 }
 786
 787 static int live_reset_whitelist(void *arg)
 788 {
 789         struct intel_gt *gt = arg;
 790         struct intel_engine_cs *engine;
 791         enum intel_engine_id id;
 792         int err = 0;
 793
 794         /* If we reset the gpu, we should not lose the RING_NONPRIV */
 795         igt_global_reset_lock(gt);
 796
 797         for_each_engine(engine, gt, id) {
 798                 if (engine->whitelist.count == 0)
 799                         continue;
 800
 801                 if (intel_has_reset_engine(gt)) {
 802                         if (intel_engine_uses_guc(engine)) {
 803                                 struct intel_selftest_saved_policy saved;
 804                                 int err2;
 805
 806                                 err = intel_selftest_modify_policy(engine, &saved,
 807                                                                    SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
 808                                 if (err)
 809                                         goto out;
 810
 811                                 err = check_whitelist_across_reset(engine,
 812                                                                    do_guc_reset,
 813                                                                    "guc");
 814
 815                                 err2 = intel_selftest_restore_policy(engine, &saved);
 816                                 if (err == 0)
 817                                         err = err2;
 818                         } else {
 819                                 err = check_whitelist_across_reset(engine,
 820                                                                    do_engine_reset,
 821                                                                    "engine");
 822                         }
 823
 824                         if (err)
 825                                 goto out;
 826                 }
 827
 828                 if (intel_has_gpu_reset(gt)) {
 829                         err = check_whitelist_across_reset(engine,
 830                                                            do_device_reset,
 831                                                            "device");
 832                         if (err)
 833                                 goto out;
 834                 }
 835         }
 836
 837 out:
 838         igt_global_reset_unlock(gt);
 839         return err;
 840 }
 841
 842 static int read_whitelisted_registers(struct intel_context *ce,
 843                                       struct i915_vma *results)
 844 {
 845         struct intel_engine_cs *engine = ce->engine;
 846         struct i915_request *rq;
 847         int i, err = 0;
 848         u32 srm, *cs;
 849
 850         rq = intel_context_create_request(ce);
 851         if (IS_ERR(rq))
 852                 return PTR_ERR(rq);
 853
 854         err = igt_vma_move_to_active_unlocked(results, rq, EXEC_OBJECT_WRITE);
 855         if (err)
 856                 goto err_req;
 857
 858         srm = MI_STORE_REGISTER_MEM;
 859         if (GRAPHICS_VER(engine->i915) >= 8)
 860                 srm++;
 861
 862         cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
 863         if (IS_ERR(cs)) {
 864                 err = PTR_ERR(cs);
 865                 goto err_req;
 866         }
 867
 868         for (i = 0; i < engine->whitelist.count; i++) {
 869                 u64 offset = i915_vma_offset(results) + sizeof(u32) * i;
 870                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 871
 872                 /* Clear non priv flags */
 873                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 874
 875                 *cs++ = srm;
 876                 *cs++ = reg;
 877                 *cs++ = lower_32_bits(offset);
 878                 *cs++ = upper_32_bits(offset);
 879         }
 880         intel_ring_advance(rq, cs);
 881
 882 err_req:
 883         return request_add_sync(rq, err);
 884 }
 885
 886 static int scrub_whitelisted_registers(struct intel_context *ce)
 887 {
 888         struct intel_engine_cs *engine = ce->engine;
 889         struct i915_request *rq;
 890         struct i915_vma *batch;
 891         int i, err = 0;
 892         u32 *cs;
 893
 894         batch = create_batch(ce->vm);
 895         if (IS_ERR(batch))
 896                 return PTR_ERR(batch);
 897
 898         cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
 899         if (IS_ERR(cs)) {
 900                 err = PTR_ERR(cs);
 901                 goto err_batch;
 902         }
 903
 904         *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
 905         for (i = 0; i < engine->whitelist.count; i++) {
 906                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 907
 908                 if (ro_register(reg))
 909                         continue;
 910
 911                 /* Clear non priv flags */
 912                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 913
 914                 *cs++ = reg;
 915                 *cs++ = 0xffffffff;
 916         }
 917         *cs++ = MI_BATCH_BUFFER_END;
 918
 919         i915_gem_object_flush_map(batch->obj);
 920         intel_gt_chipset_flush(engine->gt);
 921
 922         rq = intel_context_create_request(ce);
 923         if (IS_ERR(rq)) {
 924                 err = PTR_ERR(rq);
 925                 goto err_unpin;
 926         }
 927
 928         if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 929                 err = engine->emit_init_breadcrumb(rq);
 930                 if (err)
 931                         goto err_request;
 932         }
 933
 934         err = igt_vma_move_to_active_unlocked(batch, rq, 0);
 935         if (err)
 936                 goto err_request;
 937
 938         /* Perform the writes from an unprivileged "user" batch */
 939         err = engine->emit_bb_start(rq, i915_vma_offset(batch), 0, 0);
 940
 941 err_request:
 942         err = request_add_sync(rq, err);
 943
 944 err_unpin:
 945         i915_gem_object_unpin_map(batch->obj);
 946 err_batch:
 947         i915_vma_unpin_and_release(&batch, 0);
 948         return err;
 949 }
 950
 951 struct regmask {
 952         i915_reg_t reg;
 953         u8 graphics_ver;
 954 };
 955
 956 static bool find_reg(struct drm_i915_private *i915,
 957                      i915_reg_t reg,
 958                      const struct regmask *tbl,
 959                      unsigned long count)
 960 {
 961         u32 offset = i915_mmio_reg_offset(reg);
 962
 963         while (count--) {
 964                 if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
 965                     i915_mmio_reg_offset(tbl->reg) == offset)
 966                         return true;
 967                 tbl++;
 968         }
 969
 970         return false;
 971 }
 972
 973 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
 974 {
 975         /* Alas, we must pardon some whitelists. Mistakes already made */
 976         static const struct regmask pardon[] = {
 977                 { GEN9_CTX_PREEMPT_REG, 9 },
 978                 { _MMIO(0xb118), 9 }, /* GEN8_L3SQCREG4 */
 979         };
 980
 981         return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
 982 }
 983
 984 static bool result_eq(struct intel_engine_cs *engine,
 985                       u32 a, u32 b, i915_reg_t reg)
 986 {
 987         if (a != b && !pardon_reg(engine->i915, reg)) {
 988                 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
 989                        i915_mmio_reg_offset(reg), a, b);
 990                 return false;
 991         }
 992
 993         return true;
 994 }
 995
 996 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
 997 {
 998         /* Some registers do not seem to behave and our writes unreadable */
 999         static const struct regmask wo[] = {
1000                 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1001         };
1002
1003         return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1004 }
1005
1006 static bool result_neq(struct intel_engine_cs *engine,
1007                        u32 a, u32 b, i915_reg_t reg)
1008 {
1009         if (a == b && !writeonly_reg(engine->i915, reg)) {
1010                 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1011                        i915_mmio_reg_offset(reg), a);
1012                 return false;
1013         }
1014
1015         return true;
1016 }
1017
1018 static int
1019 check_whitelisted_registers(struct intel_engine_cs *engine,
1020                             struct i915_vma *A,
1021                             struct i915_vma *B,
1022                             bool (*fn)(struct intel_engine_cs *engine,
1023                                        u32 a, u32 b,
1024                                        i915_reg_t reg))
1025 {
1026         u32 *a, *b;
1027         int i, err;
1028
1029         a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1030         if (IS_ERR(a))
1031                 return PTR_ERR(a);
1032
1033         b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1034         if (IS_ERR(b)) {
1035                 err = PTR_ERR(b);
1036                 goto err_a;
1037         }
1038
1039         err = 0;
1040         for (i = 0; i < engine->whitelist.count; i++) {
1041                 const struct i915_wa *wa = &engine->whitelist.list[i];
1042
1043                 if (i915_mmio_reg_offset(wa->reg) &
1044                     RING_FORCE_TO_NONPRIV_ACCESS_RD)
1045                         continue;
1046
1047                 if (!fn(engine, a[i], b[i], wa->reg))
1048                         err = -EINVAL;
1049         }
1050
1051         i915_gem_object_unpin_map(B->obj);
1052 err_a:
1053         i915_gem_object_unpin_map(A->obj);
1054         return err;
1055 }
1056
1057 static int live_isolated_whitelist(void *arg)
1058 {
1059         struct intel_gt *gt = arg;
1060         struct {
1061                 struct i915_vma *scratch[2];
1062         } client[2] = {};
1063         struct intel_engine_cs *engine;
1064         enum intel_engine_id id;
1065         int i, err = 0;
1066
1067         /*
1068          * Check that a write into a whitelist register works, but
1069          * invisible to a second context.
1070          */
1071
1072         if (!intel_engines_has_context_isolation(gt->i915))
1073                 return 0;
1074
1075         for (i = 0; i < ARRAY_SIZE(client); i++) {
1076                 client[i].scratch[0] =
1077                         __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1078                 if (IS_ERR(client[i].scratch[0])) {
1079                         err = PTR_ERR(client[i].scratch[0]);
1080                         goto err;
1081                 }
1082
1083                 client[i].scratch[1] =
1084                         __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1085                 if (IS_ERR(client[i].scratch[1])) {
1086                         err = PTR_ERR(client[i].scratch[1]);
1087                         i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1088                         goto err;
1089                 }
1090         }
1091
1092         for_each_engine(engine, gt, id) {
1093                 struct intel_context *ce[2];
1094
1095                 if (!engine->kernel_context->vm)
1096                         continue;
1097
1098                 if (!whitelist_writable_count(engine))
1099                         continue;
1100
1101                 ce[0] = intel_context_create(engine);
1102                 if (IS_ERR(ce[0])) {
1103                         err = PTR_ERR(ce[0]);
1104                         break;
1105                 }
1106                 ce[1] = intel_context_create(engine);
1107                 if (IS_ERR(ce[1])) {
1108                         err = PTR_ERR(ce[1]);
1109                         intel_context_put(ce[0]);
1110                         break;
1111                 }
1112
1113                 /* Read default values */
1114                 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1115                 if (err)
1116                         goto err_ce;
1117
1118                 /* Try to overwrite registers (should only affect ctx0) */
1119                 err = scrub_whitelisted_registers(ce[0]);
1120                 if (err)
1121                         goto err_ce;
1122
1123                 /* Read values from ctx1, we expect these to be defaults */
1124                 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1125                 if (err)
1126                         goto err_ce;
1127
1128                 /* Verify that both reads return the same default values */
1129                 err = check_whitelisted_registers(engine,
1130                                                   client[0].scratch[0],
1131                                                   client[1].scratch[0],
1132                                                   result_eq);
1133                 if (err)
1134                         goto err_ce;
1135
1136                 /* Read back the updated values in ctx0 */
1137                 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1138                 if (err)
1139                         goto err_ce;
1140
1141                 /* User should be granted privilege to overwhite regs */
1142                 err = check_whitelisted_registers(engine,
1143                                                   client[0].scratch[0],
1144                                                   client[0].scratch[1],
1145                                                   result_neq);
1146 err_ce:
1147                 intel_context_put(ce[1]);
1148                 intel_context_put(ce[0]);
1149                 if (err)
1150                         break;
1151         }
1152
1153 err:
1154         for (i = 0; i < ARRAY_SIZE(client); i++) {
1155                 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1156                 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1157         }
1158
1159         if (igt_flush_test(gt->i915))
1160                 err = -EIO;
1161
1162         return err;
1163 }
1164
1165 static bool
1166 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1167                 const char *str)
1168 {
1169         struct intel_engine_cs *engine;
1170         enum intel_engine_id id;
1171         bool ok = true;
1172
1173         ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1174
1175         for_each_engine(engine, gt, id) {
1176                 struct intel_context *ce;
1177
1178                 ce = intel_context_create(engine);
1179                 if (IS_ERR(ce))
1180                         return false;
1181
1182                 ok &= engine_wa_list_verify(ce,
1183                                             &lists->engine[id].wa_list,
1184                                             str) == 0;
1185
1186                 ok &= engine_wa_list_verify(ce,
1187                                             &lists->engine[id].ctx_wa_list,
1188                                             str) == 0;
1189
1190                 intel_context_put(ce);
1191         }
1192
1193         return ok;
1194 }
1195
1196 static int
1197 live_gpu_reset_workarounds(void *arg)
1198 {
1199         struct intel_gt *gt = arg;
1200         intel_wakeref_t wakeref;
1201         struct wa_lists *lists;
1202         bool ok;
1203
1204         if (!intel_has_gpu_reset(gt))
1205                 return 0;
1206
1207         lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1208         if (!lists)
1209                 return -ENOMEM;
1210
1211         pr_info("Verifying after GPU reset...\n");
1212
1213         igt_global_reset_lock(gt);
1214         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1215
1216         reference_lists_init(gt, lists);
1217
1218         ok = verify_wa_lists(gt, lists, "before reset");
1219         if (!ok)
1220                 goto out;
1221
1222         intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1223
1224         ok = verify_wa_lists(gt, lists, "after reset");
1225
1226 out:
1227         reference_lists_fini(gt, lists);
1228         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1229         igt_global_reset_unlock(gt);
1230         kfree(lists);
1231
1232         return ok ? 0 : -ESRCH;
1233 }
1234
1235 static int
1236 live_engine_reset_workarounds(void *arg)
1237 {
1238         struct intel_gt *gt = arg;
1239         struct intel_engine_cs *engine;
1240         enum intel_engine_id id;
1241         struct intel_context *ce;
1242         struct igt_spinner spin;
1243         struct i915_request *rq;
1244         intel_wakeref_t wakeref;
1245         struct wa_lists *lists;
1246         int ret = 0;
1247
1248         if (!intel_has_reset_engine(gt))
1249                 return 0;
1250
1251         lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1252         if (!lists)
1253                 return -ENOMEM;
1254
1255         igt_global_reset_lock(gt);
1256         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1257
1258         reference_lists_init(gt, lists);
1259
1260         for_each_engine(engine, gt, id) {
1261                 struct intel_selftest_saved_policy saved;
1262                 bool using_guc = intel_engine_uses_guc(engine);
1263                 bool ok;
1264                 int ret2;
1265
1266                 pr_info("Verifying after %s reset...\n", engine->name);
1267                 ret = intel_selftest_modify_policy(engine, &saved,
1268                                                    SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1269                 if (ret)
1270                         break;
1271
1272                 ce = intel_context_create(engine);
1273                 if (IS_ERR(ce)) {
1274                         ret = PTR_ERR(ce);
1275                         goto restore;
1276                 }
1277
1278                 if (!using_guc) {
1279                         ok = verify_wa_lists(gt, lists, "before reset");
1280                         if (!ok) {
1281                                 ret = -ESRCH;
1282                                 goto err;
1283                         }
1284
1285                         ret = intel_engine_reset(engine, "live_workarounds:idle");
1286                         if (ret) {
1287                                 pr_err("%s: Reset failed while idle\n", engine->name);
1288                                 goto err;
1289                         }
1290
1291                         ok = verify_wa_lists(gt, lists, "after idle reset");
1292                         if (!ok) {
1293                                 ret = -ESRCH;
1294                                 goto err;
1295                         }
1296                 }
1297
1298                 ret = igt_spinner_init(&spin, engine->gt);
1299                 if (ret)
1300                         goto err;
1301
1302                 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1303                 if (IS_ERR(rq)) {
1304                         ret = PTR_ERR(rq);
1305                         igt_spinner_fini(&spin);
1306                         goto err;
1307                 }
1308
1309                 ret = request_add_spin(rq, &spin);
1310                 if (ret) {
1311                         pr_err("%s: Spinner failed to start\n", engine->name);
1312                         igt_spinner_fini(&spin);
1313                         goto err;
1314                 }
1315
1316                 /* Ensure the spinner hasn't aborted */
1317                 if (i915_request_completed(rq)) {
1318                         ret = -ETIMEDOUT;
1319                         goto skip;
1320                 }
1321
1322                 if (!using_guc) {
1323                         ret = intel_engine_reset(engine, "live_workarounds:active");
1324                         if (ret) {
1325                                 pr_err("%s: Reset failed on an active spinner\n",
1326                                        engine->name);
1327                                 igt_spinner_fini(&spin);
1328                                 goto err;
1329                         }
1330                 }
1331
1332                 /* Ensure the reset happens and kills the engine */
1333                 if (ret == 0)
1334                         ret = intel_selftest_wait_for_rq(rq);
1335
1336 skip:
1337                 igt_spinner_end(&spin);
1338                 igt_spinner_fini(&spin);
1339
1340                 ok = verify_wa_lists(gt, lists, "after busy reset");
1341                 if (!ok)
1342                         ret = -ESRCH;
1343
1344 err:
1345                 intel_context_put(ce);
1346
1347 restore:
1348                 ret2 = intel_selftest_restore_policy(engine, &saved);
1349                 if (ret == 0)
1350                         ret = ret2;
1351                 if (ret)
1352                         break;
1353         }
1354
1355         reference_lists_fini(gt, lists);
1356         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1357         igt_global_reset_unlock(gt);
1358         kfree(lists);
1359
1360         igt_flush_test(gt->i915);
1361
1362         return ret;
1363 }
1364
1365 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1366 {
1367         static const struct i915_subtest tests[] = {
1368                 SUBTEST(live_dirty_whitelist),
1369                 SUBTEST(live_reset_whitelist),
1370                 SUBTEST(live_isolated_whitelist),
1371                 SUBTEST(live_gpu_reset_workarounds),
1372                 SUBTEST(live_engine_reset_workarounds),
1373         };
1374
1375         if (intel_gt_is_wedged(to_gt(i915)))
1376                 return 0;
1377
1378         return intel_gt_live_subtests(tests, to_gt(i915));
1379 }