drivers/gpu/drm/i915/gt/selftest_workarounds.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2018 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_pm.h"
   7 #include "gt/intel_engine_user.h"
   8 #include "gt/intel_gt.h"
   9 #include "i915_selftest.h"
  10 #include "intel_reset.h"
  11
  12 #include "selftests/igt_flush_test.h"
  13 #include "selftests/igt_reset.h"
  14 #include "selftests/igt_spinner.h"
  15 #include "selftests/mock_drm.h"
  16
  17 #include "gem/selftests/igt_gem_utils.h"
  18 #include "gem/selftests/mock_context.h"
  19
  20 static const struct wo_register {
  21         enum intel_platform platform;
  22         u32 reg;
  23 } wo_registers[] = {
  24         { INTEL_GEMINILAKE, 0x731c }
  25 };
  26
  27 struct wa_lists {
  28         struct i915_wa_list gt_wa_list;
  29         struct {
  30                 struct i915_wa_list wa_list;
  31                 struct i915_wa_list ctx_wa_list;
  32         } engine[I915_NUM_ENGINES];
  33 };
  34
  35 static int request_add_sync(struct i915_request *rq, int err)
  36 {
  37         i915_request_get(rq);
  38         i915_request_add(rq);
  39         if (i915_request_wait(rq, 0, HZ / 5) < 0)
  40                 err = -EIO;
  41         i915_request_put(rq);
  42
  43         return err;
  44 }
  45
  46 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
  47 {
  48         int err = 0;
  49
  50         i915_request_get(rq);
  51         i915_request_add(rq);
  52         if (spin && !igt_wait_for_spinner(spin, rq))
  53                 err = -ETIMEDOUT;
  54         i915_request_put(rq);
  55
  56         return err;
  57 }
  58
  59 static void
  60 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
  61 {
  62         struct intel_engine_cs *engine;
  63         enum intel_engine_id id;
  64
  65         memset(lists, 0, sizeof(*lists));
  66
  67         wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
  68         gt_init_workarounds(gt->i915, &lists->gt_wa_list);
  69         wa_init_finish(&lists->gt_wa_list);
  70
  71         for_each_engine(engine, gt, id) {
  72                 struct i915_wa_list *wal = &lists->engine[id].wa_list;
  73
  74                 wa_init_start(wal, "REF", engine->name);
  75                 engine_init_workarounds(engine, wal);
  76                 wa_init_finish(wal);
  77
  78                 __intel_engine_init_ctx_wa(engine,
  79                                            &lists->engine[id].ctx_wa_list,
  80                                            "CTX_REF");
  81         }
  82 }
  83
  84 static void
  85 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
  86 {
  87         struct intel_engine_cs *engine;
  88         enum intel_engine_id id;
  89
  90         for_each_engine(engine, gt, id)
  91                 intel_wa_list_free(&lists->engine[id].wa_list);
  92
  93         intel_wa_list_free(&lists->gt_wa_list);
  94 }
  95
  96 static struct drm_i915_gem_object *
  97 read_nonprivs(struct intel_context *ce)
  98 {
  99         struct intel_engine_cs *engine = ce->engine;
 100         const u32 base = engine->mmio_base;
 101         struct drm_i915_gem_object *result;
 102         struct i915_request *rq;
 103         struct i915_vma *vma;
 104         u32 srm, *cs;
 105         int err;
 106         int i;
 107
 108         result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 109         if (IS_ERR(result))
 110                 return result;
 111
 112         i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
 113
 114         cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
 115         if (IS_ERR(cs)) {
 116                 err = PTR_ERR(cs);
 117                 goto err_obj;
 118         }
 119         memset(cs, 0xc5, PAGE_SIZE);
 120         i915_gem_object_flush_map(result);
 121         i915_gem_object_unpin_map(result);
 122
 123         vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
 124         if (IS_ERR(vma)) {
 125                 err = PTR_ERR(vma);
 126                 goto err_obj;
 127         }
 128
 129         err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 130         if (err)
 131                 goto err_obj;
 132
 133         rq = intel_context_create_request(ce);
 134         if (IS_ERR(rq)) {
 135                 err = PTR_ERR(rq);
 136                 goto err_pin;
 137         }
 138
 139         i915_vma_lock(vma);
 140         err = i915_request_await_object(rq, vma->obj, true);
 141         if (err == 0)
 142                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 143         i915_vma_unlock(vma);
 144         if (err)
 145                 goto err_req;
 146
 147         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 148         if (GRAPHICS_VER(engine->i915) >= 8)
 149                 srm++;
 150
 151         cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
 152         if (IS_ERR(cs)) {
 153                 err = PTR_ERR(cs);
 154                 goto err_req;
 155         }
 156
 157         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 158                 *cs++ = srm;
 159                 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
 160                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
 161                 *cs++ = 0;
 162         }
 163         intel_ring_advance(rq, cs);
 164
 165         i915_request_add(rq);
 166         i915_vma_unpin(vma);
 167
 168         return result;
 169
 170 err_req:
 171         i915_request_add(rq);
 172 err_pin:
 173         i915_vma_unpin(vma);
 174 err_obj:
 175         i915_gem_object_put(result);
 176         return ERR_PTR(err);
 177 }
 178
 179 static u32
 180 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
 181 {
 182         i915_reg_t reg = i < engine->whitelist.count ?
 183                          engine->whitelist.list[i].reg :
 184                          RING_NOPID(engine->mmio_base);
 185
 186         return i915_mmio_reg_offset(reg);
 187 }
 188
 189 static void
 190 print_results(const struct intel_engine_cs *engine, const u32 *results)
 191 {
 192         unsigned int i;
 193
 194         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 195                 u32 expected = get_whitelist_reg(engine, i);
 196                 u32 actual = results[i];
 197
 198                 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
 199                         i, expected, actual);
 200         }
 201 }
 202
 203 static int check_whitelist(struct intel_context *ce)
 204 {
 205         struct intel_engine_cs *engine = ce->engine;
 206         struct drm_i915_gem_object *results;
 207         struct intel_wedge_me wedge;
 208         u32 *vaddr;
 209         int err;
 210         int i;
 211
 212         results = read_nonprivs(ce);
 213         if (IS_ERR(results))
 214                 return PTR_ERR(results);
 215
 216         err = 0;
 217         i915_gem_object_lock(results, NULL);
 218         intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
 219                 err = i915_gem_object_set_to_cpu_domain(results, false);
 220
 221         if (intel_gt_is_wedged(engine->gt))
 222                 err = -EIO;
 223         if (err)
 224                 goto out_put;
 225
 226         vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
 227         if (IS_ERR(vaddr)) {
 228                 err = PTR_ERR(vaddr);
 229                 goto out_put;
 230         }
 231
 232         for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 233                 u32 expected = get_whitelist_reg(engine, i);
 234                 u32 actual = vaddr[i];
 235
 236                 if (expected != actual) {
 237                         print_results(engine, vaddr);
 238                         pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
 239                                i, expected, actual);
 240
 241                         err = -EINVAL;
 242                         break;
 243                 }
 244         }
 245
 246         i915_gem_object_unpin_map(results);
 247 out_put:
 248         i915_gem_object_unlock(results);
 249         i915_gem_object_put(results);
 250         return err;
 251 }
 252
 253 static int do_device_reset(struct intel_engine_cs *engine)
 254 {
 255         intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
 256         return 0;
 257 }
 258
 259 static int do_engine_reset(struct intel_engine_cs *engine)
 260 {
 261         return intel_engine_reset(engine, "live_workarounds");
 262 }
 263
 264 static int
 265 switch_to_scratch_context(struct intel_engine_cs *engine,
 266                           struct igt_spinner *spin)
 267 {
 268         struct intel_context *ce;
 269         struct i915_request *rq;
 270         int err = 0;
 271
 272         ce = intel_context_create(engine);
 273         if (IS_ERR(ce))
 274                 return PTR_ERR(ce);
 275
 276         rq = igt_spinner_create_request(spin, ce, MI_NOOP);
 277         intel_context_put(ce);
 278
 279         if (IS_ERR(rq)) {
 280                 spin = NULL;
 281                 err = PTR_ERR(rq);
 282                 goto err;
 283         }
 284
 285         err = request_add_spin(rq, spin);
 286 err:
 287         if (err && spin)
 288                 igt_spinner_end(spin);
 289
 290         return err;
 291 }
 292
 293 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 294                                         int (*reset)(struct intel_engine_cs *),
 295                                         const char *name)
 296 {
 297         struct intel_context *ce, *tmp;
 298         struct igt_spinner spin;
 299         intel_wakeref_t wakeref;
 300         int err;
 301
 302         pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
 303                 engine->whitelist.count, engine->name, name);
 304
 305         ce = intel_context_create(engine);
 306         if (IS_ERR(ce))
 307                 return PTR_ERR(ce);
 308
 309         err = igt_spinner_init(&spin, engine->gt);
 310         if (err)
 311                 goto out_ctx;
 312
 313         err = check_whitelist(ce);
 314         if (err) {
 315                 pr_err("Invalid whitelist *before* %s reset!\n", name);
 316                 goto out_spin;
 317         }
 318
 319         err = switch_to_scratch_context(engine, &spin);
 320         if (err)
 321                 goto out_spin;
 322
 323         with_intel_runtime_pm(engine->uncore->rpm, wakeref)
 324                 err = reset(engine);
 325
 326         igt_spinner_end(&spin);
 327
 328         if (err) {
 329                 pr_err("%s reset failed\n", name);
 330                 goto out_spin;
 331         }
 332
 333         err = check_whitelist(ce);
 334         if (err) {
 335                 pr_err("Whitelist not preserved in context across %s reset!\n",
 336                        name);
 337                 goto out_spin;
 338         }
 339
 340         tmp = intel_context_create(engine);
 341         if (IS_ERR(tmp)) {
 342                 err = PTR_ERR(tmp);
 343                 goto out_spin;
 344         }
 345         intel_context_put(ce);
 346         ce = tmp;
 347
 348         err = check_whitelist(ce);
 349         if (err) {
 350                 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
 351                        name);
 352                 goto out_spin;
 353         }
 354
 355 out_spin:
 356         igt_spinner_fini(&spin);
 357 out_ctx:
 358         intel_context_put(ce);
 359         return err;
 360 }
 361
 362 static struct i915_vma *create_batch(struct i915_address_space *vm)
 363 {
 364         struct drm_i915_gem_object *obj;
 365         struct i915_vma *vma;
 366         int err;
 367
 368         obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
 369         if (IS_ERR(obj))
 370                 return ERR_CAST(obj);
 371
 372         vma = i915_vma_instance(obj, vm, NULL);
 373         if (IS_ERR(vma)) {
 374                 err = PTR_ERR(vma);
 375                 goto err_obj;
 376         }
 377
 378         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 379         if (err)
 380                 goto err_obj;
 381
 382         return vma;
 383
 384 err_obj:
 385         i915_gem_object_put(obj);
 386         return ERR_PTR(err);
 387 }
 388
 389 static u32 reg_write(u32 old, u32 new, u32 rsvd)
 390 {
 391         if (rsvd == 0x0000ffff) {
 392                 old &= ~(new >> 16);
 393                 old |= new & (new >> 16);
 394         } else {
 395                 old &= ~rsvd;
 396                 old |= new & rsvd;
 397         }
 398
 399         return old;
 400 }
 401
 402 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
 403 {
 404         enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
 405         int i;
 406
 407         if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 408              RING_FORCE_TO_NONPRIV_ACCESS_WR)
 409                 return true;
 410
 411         for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
 412                 if (wo_registers[i].platform == platform &&
 413                     wo_registers[i].reg == reg)
 414                         return true;
 415         }
 416
 417         return false;
 418 }
 419
 420 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
 421 {
 422         reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
 423         switch (reg) {
 424         case 0x358:
 425         case 0x35c:
 426         case 0x3a8:
 427                 return true;
 428
 429         default:
 430                 return false;
 431         }
 432 }
 433
 434 static bool ro_register(u32 reg)
 435 {
 436         if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 437              RING_FORCE_TO_NONPRIV_ACCESS_RD)
 438                 return true;
 439
 440         return false;
 441 }
 442
 443 static int whitelist_writable_count(struct intel_engine_cs *engine)
 444 {
 445         int count = engine->whitelist.count;
 446         int i;
 447
 448         for (i = 0; i < engine->whitelist.count; i++) {
 449                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 450
 451                 if (ro_register(reg))
 452                         count--;
 453         }
 454
 455         return count;
 456 }
 457
 458 static int check_dirty_whitelist(struct intel_context *ce)
 459 {
 460         const u32 values[] = {
 461                 0x00000000,
 462                 0x01010101,
 463                 0x10100101,
 464                 0x03030303,
 465                 0x30300303,
 466                 0x05050505,
 467                 0x50500505,
 468                 0x0f0f0f0f,
 469                 0xf00ff00f,
 470                 0x10101010,
 471                 0xf0f01010,
 472                 0x30303030,
 473                 0xa0a03030,
 474                 0x50505050,
 475                 0xc0c05050,
 476                 0xf0f0f0f0,
 477                 0x11111111,
 478                 0x33333333,
 479                 0x55555555,
 480                 0x0000ffff,
 481                 0x00ff00ff,
 482                 0xff0000ff,
 483                 0xffff00ff,
 484                 0xffffffff,
 485         };
 486         struct intel_engine_cs *engine = ce->engine;
 487         struct i915_vma *scratch;
 488         struct i915_vma *batch;
 489         int err = 0, i, v, sz;
 490         u32 *cs, *results;
 491
 492         sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
 493         scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
 494         if (IS_ERR(scratch))
 495                 return PTR_ERR(scratch);
 496
 497         batch = create_batch(ce->vm);
 498         if (IS_ERR(batch)) {
 499                 err = PTR_ERR(batch);
 500                 goto out_scratch;
 501         }
 502
 503         for (i = 0; i < engine->whitelist.count; i++) {
 504                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 505                 struct i915_gem_ww_ctx ww;
 506                 u64 addr = scratch->node.start;
 507                 struct i915_request *rq;
 508                 u32 srm, lrm, rsvd;
 509                 u32 expect;
 510                 int idx;
 511                 bool ro_reg;
 512
 513                 if (wo_register(engine, reg))
 514                         continue;
 515
 516                 if (timestamp(engine, reg))
 517                         continue; /* timestamps are expected to autoincrement */
 518
 519                 ro_reg = ro_register(reg);
 520
 521                 i915_gem_ww_ctx_init(&ww, false);
 522 retry:
 523                 cs = NULL;
 524                 err = i915_gem_object_lock(scratch->obj, &ww);
 525                 if (!err)
 526                         err = i915_gem_object_lock(batch->obj, &ww);
 527                 if (!err)
 528                         err = intel_context_pin_ww(ce, &ww);
 529                 if (err)
 530                         goto out;
 531
 532                 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 533                 if (IS_ERR(cs)) {
 534                         err = PTR_ERR(cs);
 535                         goto out_ctx;
 536                 }
 537
 538                 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 539                 if (IS_ERR(results)) {
 540                         err = PTR_ERR(results);
 541                         goto out_unmap_batch;
 542                 }
 543
 544                 /* Clear non priv flags */
 545                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 546
 547                 srm = MI_STORE_REGISTER_MEM;
 548                 lrm = MI_LOAD_REGISTER_MEM;
 549                 if (GRAPHICS_VER(engine->i915) >= 8)
 550                         lrm++, srm++;
 551
 552                 pr_debug("%s: Writing garbage to %x\n",
 553                          engine->name, reg);
 554
 555                 /* SRM original */
 556                 *cs++ = srm;
 557                 *cs++ = reg;
 558                 *cs++ = lower_32_bits(addr);
 559                 *cs++ = upper_32_bits(addr);
 560
 561                 idx = 1;
 562                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 563                         /* LRI garbage */
 564                         *cs++ = MI_LOAD_REGISTER_IMM(1);
 565                         *cs++ = reg;
 566                         *cs++ = values[v];
 567
 568                         /* SRM result */
 569                         *cs++ = srm;
 570                         *cs++ = reg;
 571                         *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 572                         *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 573                         idx++;
 574                 }
 575                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 576                         /* LRI garbage */
 577                         *cs++ = MI_LOAD_REGISTER_IMM(1);
 578                         *cs++ = reg;
 579                         *cs++ = ~values[v];
 580
 581                         /* SRM result */
 582                         *cs++ = srm;
 583                         *cs++ = reg;
 584                         *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 585                         *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 586                         idx++;
 587                 }
 588                 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
 589
 590                 /* LRM original -- don't leave garbage in the context! */
 591                 *cs++ = lrm;
 592                 *cs++ = reg;
 593                 *cs++ = lower_32_bits(addr);
 594                 *cs++ = upper_32_bits(addr);
 595
 596                 *cs++ = MI_BATCH_BUFFER_END;
 597
 598                 i915_gem_object_flush_map(batch->obj);
 599                 i915_gem_object_unpin_map(batch->obj);
 600                 intel_gt_chipset_flush(engine->gt);
 601                 cs = NULL;
 602
 603                 rq = i915_request_create(ce);
 604                 if (IS_ERR(rq)) {
 605                         err = PTR_ERR(rq);
 606                         goto out_unmap_scratch;
 607                 }
 608
 609                 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 610                         err = engine->emit_init_breadcrumb(rq);
 611                         if (err)
 612                                 goto err_request;
 613                 }
 614
 615                 err = i915_request_await_object(rq, batch->obj, false);
 616                 if (err == 0)
 617                         err = i915_vma_move_to_active(batch, rq, 0);
 618                 if (err)
 619                         goto err_request;
 620
 621                 err = i915_request_await_object(rq, scratch->obj, true);
 622                 if (err == 0)
 623                         err = i915_vma_move_to_active(scratch, rq,
 624                                                       EXEC_OBJECT_WRITE);
 625                 if (err)
 626                         goto err_request;
 627
 628                 err = engine->emit_bb_start(rq,
 629                                             batch->node.start, PAGE_SIZE,
 630                                             0);
 631                 if (err)
 632                         goto err_request;
 633
 634 err_request:
 635                 err = request_add_sync(rq, err);
 636                 if (err) {
 637                         pr_err("%s: Futzing %x timedout; cancelling test\n",
 638                                engine->name, reg);
 639                         intel_gt_set_wedged(engine->gt);
 640                         goto out_unmap_scratch;
 641                 }
 642
 643                 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
 644                 if (!ro_reg) {
 645                         /* detect write masking */
 646                         rsvd = results[ARRAY_SIZE(values)];
 647                         if (!rsvd) {
 648                                 pr_err("%s: Unable to write to whitelisted register %x\n",
 649                                        engine->name, reg);
 650                                 err = -EINVAL;
 651                                 goto out_unmap_scratch;
 652                         }
 653                 } else {
 654                         rsvd = 0;
 655                 }
 656
 657                 expect = results[0];
 658                 idx = 1;
 659                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 660                         if (ro_reg)
 661                                 expect = results[0];
 662                         else
 663                                 expect = reg_write(expect, values[v], rsvd);
 664
 665                         if (results[idx] != expect)
 666                                 err++;
 667                         idx++;
 668                 }
 669                 for (v = 0; v < ARRAY_SIZE(values); v++) {
 670                         if (ro_reg)
 671                                 expect = results[0];
 672                         else
 673                                 expect = reg_write(expect, ~values[v], rsvd);
 674
 675                         if (results[idx] != expect)
 676                                 err++;
 677                         idx++;
 678                 }
 679                 if (err) {
 680                         pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
 681                                engine->name, err, reg);
 682
 683                         if (ro_reg)
 684                                 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
 685                                         engine->name, reg, results[0]);
 686                         else
 687                                 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
 688                                         engine->name, reg, results[0], rsvd);
 689
 690                         expect = results[0];
 691                         idx = 1;
 692                         for (v = 0; v < ARRAY_SIZE(values); v++) {
 693                                 u32 w = values[v];
 694
 695                                 if (ro_reg)
 696                                         expect = results[0];
 697                                 else
 698                                         expect = reg_write(expect, w, rsvd);
 699                                 pr_info("Wrote %08x, read %08x, expect %08x\n",
 700                                         w, results[idx], expect);
 701                                 idx++;
 702                         }
 703                         for (v = 0; v < ARRAY_SIZE(values); v++) {
 704                                 u32 w = ~values[v];
 705
 706                                 if (ro_reg)
 707                                         expect = results[0];
 708                                 else
 709                                         expect = reg_write(expect, w, rsvd);
 710                                 pr_info("Wrote %08x, read %08x, expect %08x\n",
 711                                         w, results[idx], expect);
 712                                 idx++;
 713                         }
 714
 715                         err = -EINVAL;
 716                 }
 717 out_unmap_scratch:
 718                 i915_gem_object_unpin_map(scratch->obj);
 719 out_unmap_batch:
 720                 if (cs)
 721                         i915_gem_object_unpin_map(batch->obj);
 722 out_ctx:
 723                 intel_context_unpin(ce);
 724 out:
 725                 if (err == -EDEADLK) {
 726                         err = i915_gem_ww_ctx_backoff(&ww);
 727                         if (!err)
 728                                 goto retry;
 729                 }
 730                 i915_gem_ww_ctx_fini(&ww);
 731                 if (err)
 732                         break;
 733         }
 734
 735         if (igt_flush_test(engine->i915))
 736                 err = -EIO;
 737
 738         i915_vma_unpin_and_release(&batch, 0);
 739 out_scratch:
 740         i915_vma_unpin_and_release(&scratch, 0);
 741         return err;
 742 }
 743
 744 static int live_dirty_whitelist(void *arg)
 745 {
 746         struct intel_gt *gt = arg;
 747         struct intel_engine_cs *engine;
 748         enum intel_engine_id id;
 749
 750         /* Can the user write to the whitelisted registers? */
 751
 752         if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
 753                 return 0;
 754
 755         for_each_engine(engine, gt, id) {
 756                 struct intel_context *ce;
 757                 int err;
 758
 759                 if (engine->whitelist.count == 0)
 760                         continue;
 761
 762                 ce = intel_context_create(engine);
 763                 if (IS_ERR(ce))
 764                         return PTR_ERR(ce);
 765
 766                 err = check_dirty_whitelist(ce);
 767                 intel_context_put(ce);
 768                 if (err)
 769                         return err;
 770         }
 771
 772         return 0;
 773 }
 774
 775 static int live_reset_whitelist(void *arg)
 776 {
 777         struct intel_gt *gt = arg;
 778         struct intel_engine_cs *engine;
 779         enum intel_engine_id id;
 780         int err = 0;
 781
 782         /* If we reset the gpu, we should not lose the RING_NONPRIV */
 783         igt_global_reset_lock(gt);
 784
 785         for_each_engine(engine, gt, id) {
 786                 if (engine->whitelist.count == 0)
 787                         continue;
 788
 789                 if (intel_has_reset_engine(gt)) {
 790                         err = check_whitelist_across_reset(engine,
 791                                                            do_engine_reset,
 792                                                            "engine");
 793                         if (err)
 794                                 goto out;
 795                 }
 796
 797                 if (intel_has_gpu_reset(gt)) {
 798                         err = check_whitelist_across_reset(engine,
 799                                                            do_device_reset,
 800                                                            "device");
 801                         if (err)
 802                                 goto out;
 803                 }
 804         }
 805
 806 out:
 807         igt_global_reset_unlock(gt);
 808         return err;
 809 }
 810
 811 static int read_whitelisted_registers(struct intel_context *ce,
 812                                       struct i915_vma *results)
 813 {
 814         struct intel_engine_cs *engine = ce->engine;
 815         struct i915_request *rq;
 816         int i, err = 0;
 817         u32 srm, *cs;
 818
 819         rq = intel_context_create_request(ce);
 820         if (IS_ERR(rq))
 821                 return PTR_ERR(rq);
 822
 823         i915_vma_lock(results);
 824         err = i915_request_await_object(rq, results->obj, true);
 825         if (err == 0)
 826                 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
 827         i915_vma_unlock(results);
 828         if (err)
 829                 goto err_req;
 830
 831         srm = MI_STORE_REGISTER_MEM;
 832         if (GRAPHICS_VER(engine->i915) >= 8)
 833                 srm++;
 834
 835         cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
 836         if (IS_ERR(cs)) {
 837                 err = PTR_ERR(cs);
 838                 goto err_req;
 839         }
 840
 841         for (i = 0; i < engine->whitelist.count; i++) {
 842                 u64 offset = results->node.start + sizeof(u32) * i;
 843                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 844
 845                 /* Clear non priv flags */
 846                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 847
 848                 *cs++ = srm;
 849                 *cs++ = reg;
 850                 *cs++ = lower_32_bits(offset);
 851                 *cs++ = upper_32_bits(offset);
 852         }
 853         intel_ring_advance(rq, cs);
 854
 855 err_req:
 856         return request_add_sync(rq, err);
 857 }
 858
 859 static int scrub_whitelisted_registers(struct intel_context *ce)
 860 {
 861         struct intel_engine_cs *engine = ce->engine;
 862         struct i915_request *rq;
 863         struct i915_vma *batch;
 864         int i, err = 0;
 865         u32 *cs;
 866
 867         batch = create_batch(ce->vm);
 868         if (IS_ERR(batch))
 869                 return PTR_ERR(batch);
 870
 871         cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
 872         if (IS_ERR(cs)) {
 873                 err = PTR_ERR(cs);
 874                 goto err_batch;
 875         }
 876
 877         *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
 878         for (i = 0; i < engine->whitelist.count; i++) {
 879                 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 880
 881                 if (ro_register(reg))
 882                         continue;
 883
 884                 /* Clear non priv flags */
 885                 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 886
 887                 *cs++ = reg;
 888                 *cs++ = 0xffffffff;
 889         }
 890         *cs++ = MI_BATCH_BUFFER_END;
 891
 892         i915_gem_object_flush_map(batch->obj);
 893         intel_gt_chipset_flush(engine->gt);
 894
 895         rq = intel_context_create_request(ce);
 896         if (IS_ERR(rq)) {
 897                 err = PTR_ERR(rq);
 898                 goto err_unpin;
 899         }
 900
 901         if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 902                 err = engine->emit_init_breadcrumb(rq);
 903                 if (err)
 904                         goto err_request;
 905         }
 906
 907         i915_vma_lock(batch);
 908         err = i915_request_await_object(rq, batch->obj, false);
 909         if (err == 0)
 910                 err = i915_vma_move_to_active(batch, rq, 0);
 911         i915_vma_unlock(batch);
 912         if (err)
 913                 goto err_request;
 914
 915         /* Perform the writes from an unprivileged "user" batch */
 916         err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
 917
 918 err_request:
 919         err = request_add_sync(rq, err);
 920
 921 err_unpin:
 922         i915_gem_object_unpin_map(batch->obj);
 923 err_batch:
 924         i915_vma_unpin_and_release(&batch, 0);
 925         return err;
 926 }
 927
 928 struct regmask {
 929         i915_reg_t reg;
 930         u8 graphics_ver;
 931 };
 932
 933 static bool find_reg(struct drm_i915_private *i915,
 934                      i915_reg_t reg,
 935                      const struct regmask *tbl,
 936                      unsigned long count)
 937 {
 938         u32 offset = i915_mmio_reg_offset(reg);
 939
 940         while (count--) {
 941                 if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
 942                     i915_mmio_reg_offset(tbl->reg) == offset)
 943                         return true;
 944                 tbl++;
 945         }
 946
 947         return false;
 948 }
 949
 950 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
 951 {
 952         /* Alas, we must pardon some whitelists. Mistakes already made */
 953         static const struct regmask pardon[] = {
 954                 { GEN9_CTX_PREEMPT_REG, 9 },
 955                 { GEN8_L3SQCREG4, 9 },
 956         };
 957
 958         return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
 959 }
 960
 961 static bool result_eq(struct intel_engine_cs *engine,
 962                       u32 a, u32 b, i915_reg_t reg)
 963 {
 964         if (a != b && !pardon_reg(engine->i915, reg)) {
 965                 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
 966                        i915_mmio_reg_offset(reg), a, b);
 967                 return false;
 968         }
 969
 970         return true;
 971 }
 972
 973 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
 974 {
 975         /* Some registers do not seem to behave and our writes unreadable */
 976         static const struct regmask wo[] = {
 977                 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
 978         };
 979
 980         return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
 981 }
 982
 983 static bool result_neq(struct intel_engine_cs *engine,
 984                        u32 a, u32 b, i915_reg_t reg)
 985 {
 986         if (a == b && !writeonly_reg(engine->i915, reg)) {
 987                 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
 988                        i915_mmio_reg_offset(reg), a);
 989                 return false;
 990         }
 991
 992         return true;
 993 }
 994
 995 static int
 996 check_whitelisted_registers(struct intel_engine_cs *engine,
 997                             struct i915_vma *A,
 998                             struct i915_vma *B,
 999                             bool (*fn)(struct intel_engine_cs *engine,
1000                                        u32 a, u32 b,
1001                                        i915_reg_t reg))
1002 {
1003         u32 *a, *b;
1004         int i, err;
1005
1006         a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1007         if (IS_ERR(a))
1008                 return PTR_ERR(a);
1009
1010         b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1011         if (IS_ERR(b)) {
1012                 err = PTR_ERR(b);
1013                 goto err_a;
1014         }
1015
1016         err = 0;
1017         for (i = 0; i < engine->whitelist.count; i++) {
1018                 const struct i915_wa *wa = &engine->whitelist.list[i];
1019
1020                 if (i915_mmio_reg_offset(wa->reg) &
1021                     RING_FORCE_TO_NONPRIV_ACCESS_RD)
1022                         continue;
1023
1024                 if (!fn(engine, a[i], b[i], wa->reg))
1025                         err = -EINVAL;
1026         }
1027
1028         i915_gem_object_unpin_map(B->obj);
1029 err_a:
1030         i915_gem_object_unpin_map(A->obj);
1031         return err;
1032 }
1033
1034 static int live_isolated_whitelist(void *arg)
1035 {
1036         struct intel_gt *gt = arg;
1037         struct {
1038                 struct i915_vma *scratch[2];
1039         } client[2] = {};
1040         struct intel_engine_cs *engine;
1041         enum intel_engine_id id;
1042         int i, err = 0;
1043
1044         /*
1045          * Check that a write into a whitelist register works, but
1046          * invisible to a second context.
1047          */
1048
1049         if (!intel_engines_has_context_isolation(gt->i915))
1050                 return 0;
1051
1052         for (i = 0; i < ARRAY_SIZE(client); i++) {
1053                 client[i].scratch[0] =
1054                         __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1055                 if (IS_ERR(client[i].scratch[0])) {
1056                         err = PTR_ERR(client[i].scratch[0]);
1057                         goto err;
1058                 }
1059
1060                 client[i].scratch[1] =
1061                         __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1062                 if (IS_ERR(client[i].scratch[1])) {
1063                         err = PTR_ERR(client[i].scratch[1]);
1064                         i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1065                         goto err;
1066                 }
1067         }
1068
1069         for_each_engine(engine, gt, id) {
1070                 struct intel_context *ce[2];
1071
1072                 if (!engine->kernel_context->vm)
1073                         continue;
1074
1075                 if (!whitelist_writable_count(engine))
1076                         continue;
1077
1078                 ce[0] = intel_context_create(engine);
1079                 if (IS_ERR(ce[0])) {
1080                         err = PTR_ERR(ce[0]);
1081                         break;
1082                 }
1083                 ce[1] = intel_context_create(engine);
1084                 if (IS_ERR(ce[1])) {
1085                         err = PTR_ERR(ce[1]);
1086                         intel_context_put(ce[0]);
1087                         break;
1088                 }
1089
1090                 /* Read default values */
1091                 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1092                 if (err)
1093                         goto err_ce;
1094
1095                 /* Try to overwrite registers (should only affect ctx0) */
1096                 err = scrub_whitelisted_registers(ce[0]);
1097                 if (err)
1098                         goto err_ce;
1099
1100                 /* Read values from ctx1, we expect these to be defaults */
1101                 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1102                 if (err)
1103                         goto err_ce;
1104
1105                 /* Verify that both reads return the same default values */
1106                 err = check_whitelisted_registers(engine,
1107                                                   client[0].scratch[0],
1108                                                   client[1].scratch[0],
1109                                                   result_eq);
1110                 if (err)
1111                         goto err_ce;
1112
1113                 /* Read back the updated values in ctx0 */
1114                 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1115                 if (err)
1116                         goto err_ce;
1117
1118                 /* User should be granted privilege to overwhite regs */
1119                 err = check_whitelisted_registers(engine,
1120                                                   client[0].scratch[0],
1121                                                   client[0].scratch[1],
1122                                                   result_neq);
1123 err_ce:
1124                 intel_context_put(ce[1]);
1125                 intel_context_put(ce[0]);
1126                 if (err)
1127                         break;
1128         }
1129
1130 err:
1131         for (i = 0; i < ARRAY_SIZE(client); i++) {
1132                 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1133                 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1134         }
1135
1136         if (igt_flush_test(gt->i915))
1137                 err = -EIO;
1138
1139         return err;
1140 }
1141
1142 static bool
1143 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1144                 const char *str)
1145 {
1146         struct intel_engine_cs *engine;
1147         enum intel_engine_id id;
1148         bool ok = true;
1149
1150         ok &= wa_list_verify(gt->uncore, &lists->gt_wa_list, str);
1151
1152         for_each_engine(engine, gt, id) {
1153                 struct intel_context *ce;
1154
1155                 ce = intel_context_create(engine);
1156                 if (IS_ERR(ce))
1157                         return false;
1158
1159                 ok &= engine_wa_list_verify(ce,
1160                                             &lists->engine[id].wa_list,
1161                                             str) == 0;
1162
1163                 ok &= engine_wa_list_verify(ce,
1164                                             &lists->engine[id].ctx_wa_list,
1165                                             str) == 0;
1166
1167                 intel_context_put(ce);
1168         }
1169
1170         return ok;
1171 }
1172
1173 static int
1174 live_gpu_reset_workarounds(void *arg)
1175 {
1176         struct intel_gt *gt = arg;
1177         intel_wakeref_t wakeref;
1178         struct wa_lists lists;
1179         bool ok;
1180
1181         if (!intel_has_gpu_reset(gt))
1182                 return 0;
1183
1184         pr_info("Verifying after GPU reset...\n");
1185
1186         igt_global_reset_lock(gt);
1187         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1188
1189         reference_lists_init(gt, &lists);
1190
1191         ok = verify_wa_lists(gt, &lists, "before reset");
1192         if (!ok)
1193                 goto out;
1194
1195         intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1196
1197         ok = verify_wa_lists(gt, &lists, "after reset");
1198
1199 out:
1200         reference_lists_fini(gt, &lists);
1201         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1202         igt_global_reset_unlock(gt);
1203
1204         return ok ? 0 : -ESRCH;
1205 }
1206
1207 static int
1208 live_engine_reset_workarounds(void *arg)
1209 {
1210         struct intel_gt *gt = arg;
1211         struct intel_engine_cs *engine;
1212         enum intel_engine_id id;
1213         struct intel_context *ce;
1214         struct igt_spinner spin;
1215         struct i915_request *rq;
1216         intel_wakeref_t wakeref;
1217         struct wa_lists lists;
1218         int ret = 0;
1219
1220         if (!intel_has_reset_engine(gt))
1221                 return 0;
1222
1223         igt_global_reset_lock(gt);
1224         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1225
1226         reference_lists_init(gt, &lists);
1227
1228         for_each_engine(engine, gt, id) {
1229                 bool ok;
1230
1231                 pr_info("Verifying after %s reset...\n", engine->name);
1232                 ce = intel_context_create(engine);
1233                 if (IS_ERR(ce)) {
1234                         ret = PTR_ERR(ce);
1235                         break;
1236                 }
1237
1238                 ok = verify_wa_lists(gt, &lists, "before reset");
1239                 if (!ok) {
1240                         ret = -ESRCH;
1241                         goto err;
1242                 }
1243
1244                 ret = intel_engine_reset(engine, "live_workarounds:idle");
1245                 if (ret) {
1246                         pr_err("%s: Reset failed while idle\n", engine->name);
1247                         goto err;
1248                 }
1249
1250                 ok = verify_wa_lists(gt, &lists, "after idle reset");
1251                 if (!ok) {
1252                         ret = -ESRCH;
1253                         goto err;
1254                 }
1255
1256                 ret = igt_spinner_init(&spin, engine->gt);
1257                 if (ret)
1258                         goto err;
1259
1260                 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1261                 if (IS_ERR(rq)) {
1262                         ret = PTR_ERR(rq);
1263                         igt_spinner_fini(&spin);
1264                         goto err;
1265                 }
1266
1267                 ret = request_add_spin(rq, &spin);
1268                 if (ret) {
1269                         pr_err("%s: Spinner failed to start\n", engine->name);
1270                         igt_spinner_fini(&spin);
1271                         goto err;
1272                 }
1273
1274                 ret = intel_engine_reset(engine, "live_workarounds:active");
1275                 if (ret) {
1276                         pr_err("%s: Reset failed on an active spinner\n",
1277                                engine->name);
1278                         igt_spinner_fini(&spin);
1279                         goto err;
1280                 }
1281
1282                 igt_spinner_end(&spin);
1283                 igt_spinner_fini(&spin);
1284
1285                 ok = verify_wa_lists(gt, &lists, "after busy reset");
1286                 if (!ok) {
1287                         ret = -ESRCH;
1288                         goto err;
1289                 }
1290
1291 err:
1292                 intel_context_put(ce);
1293                 if (ret)
1294                         break;
1295         }
1296
1297         reference_lists_fini(gt, &lists);
1298         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1299         igt_global_reset_unlock(gt);
1300
1301         igt_flush_test(gt->i915);
1302
1303         return ret;
1304 }
1305
1306 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1307 {
1308         static const struct i915_subtest tests[] = {
1309                 SUBTEST(live_dirty_whitelist),
1310                 SUBTEST(live_reset_whitelist),
1311                 SUBTEST(live_isolated_whitelist),
1312                 SUBTEST(live_gpu_reset_workarounds),
1313                 SUBTEST(live_engine_reset_workarounds),
1314         };
1315
1316         if (intel_gt_is_wedged(&i915->gt))
1317                 return 0;
1318
1319         return intel_gt_live_subtests(tests, &i915->gt);
1320 }