drivers/gpu/drm/i915/gt/intel_gt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "debugfs_gt.h"
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_gt.h"
  10 #include "intel_gt_buffer_pool.h"
  11 #include "intel_gt_clock_utils.h"
  12 #include "intel_gt_pm.h"
  13 #include "intel_gt_requests.h"
  14 #include "intel_mocs.h"
  15 #include "intel_rc6.h"
  16 #include "intel_renderstate.h"
  17 #include "intel_rps.h"
  18 #include "intel_uncore.h"
  19 #include "intel_pm.h"
  20 #include "shmem_utils.h"
  21
  22 void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
  23 {
  24         gt->i915 = i915;
  25         gt->uncore = &i915->uncore;
  26
  27         spin_lock_init(&gt->irq_lock);
  28
  29         INIT_LIST_HEAD(&gt->closed_vma);
  30         spin_lock_init(&gt->closed_lock);
  31
  32         intel_gt_init_buffer_pool(gt);
  33         intel_gt_init_reset(gt);
  34         intel_gt_init_requests(gt);
  35         intel_gt_init_timelines(gt);
  36         intel_gt_pm_init_early(gt);
  37
  38         intel_rps_init_early(&gt->rps);
  39         intel_uc_init_early(&gt->uc);
  40 }
  41
  42 void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt)
  43 {
  44         gt->ggtt = ggtt;
  45 }
  46
  47 int intel_gt_init_mmio(struct intel_gt *gt)
  48 {
  49         intel_uc_init_mmio(&gt->uc);
  50         intel_sseu_info_init(gt);
  51
  52         return intel_engines_init_mmio(gt);
  53 }
  54
  55 static void init_unused_ring(struct intel_gt *gt, u32 base)
  56 {
  57         struct intel_uncore *uncore = gt->uncore;
  58
  59         intel_uncore_write(uncore, RING_CTL(base), 0);
  60         intel_uncore_write(uncore, RING_HEAD(base), 0);
  61         intel_uncore_write(uncore, RING_TAIL(base), 0);
  62         intel_uncore_write(uncore, RING_START(base), 0);
  63 }
  64
  65 static void init_unused_rings(struct intel_gt *gt)
  66 {
  67         struct drm_i915_private *i915 = gt->i915;
  68
  69         if (IS_I830(i915)) {
  70                 init_unused_ring(gt, PRB1_BASE);
  71                 init_unused_ring(gt, SRB0_BASE);
  72                 init_unused_ring(gt, SRB1_BASE);
  73                 init_unused_ring(gt, SRB2_BASE);
  74                 init_unused_ring(gt, SRB3_BASE);
  75         } else if (IS_GEN(i915, 2)) {
  76                 init_unused_ring(gt, SRB0_BASE);
  77                 init_unused_ring(gt, SRB1_BASE);
  78         } else if (IS_GEN(i915, 3)) {
  79                 init_unused_ring(gt, PRB1_BASE);
  80                 init_unused_ring(gt, PRB2_BASE);
  81         }
  82 }
  83
  84 int intel_gt_init_hw(struct intel_gt *gt)
  85 {
  86         struct drm_i915_private *i915 = gt->i915;
  87         struct intel_uncore *uncore = gt->uncore;
  88         int ret;
  89
  90         gt->last_init_time = ktime_get();
  91
  92         /* Double layer security blanket, see i915_gem_init() */
  93         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
  94
  95         if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9)
  96                 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
  97
  98         if (IS_HASWELL(i915))
  99                 intel_uncore_write(uncore,
 100                                    MI_PREDICATE_RESULT_2,
 101                                    IS_HSW_GT3(i915) ?
 102                                    LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 103
 104         /* Apply the GT workarounds... */
 105         intel_gt_apply_workarounds(gt);
 106         /* ...and determine whether they are sticking. */
 107         intel_gt_verify_workarounds(gt, "init");
 108
 109         intel_gt_init_swizzling(gt);
 110
 111         /*
 112          * At least 830 can leave some of the unused rings
 113          * "active" (ie. head != tail) after resume which
 114          * will prevent c3 entry. Makes sure all unused rings
 115          * are totally idle.
 116          */
 117         init_unused_rings(gt);
 118
 119         ret = i915_ppgtt_init_hw(gt);
 120         if (ret) {
 121                 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
 122                 goto out;
 123         }
 124
 125         /* We can't enable contexts until all firmware is loaded */
 126         ret = intel_uc_init_hw(&gt->uc);
 127         if (ret) {
 128                 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
 129                 goto out;
 130         }
 131
 132         intel_mocs_init(gt);
 133
 134 out:
 135         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 136         return ret;
 137 }
 138
 139 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
 140 {
 141         intel_uncore_rmw(uncore, reg, 0, set);
 142 }
 143
 144 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
 145 {
 146         intel_uncore_rmw(uncore, reg, clr, 0);
 147 }
 148
 149 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
 150 {
 151         intel_uncore_rmw(uncore, reg, 0, 0);
 152 }
 153
 154 static void gen8_clear_engine_error_register(struct intel_engine_cs *engine)
 155 {
 156         GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
 157         GEN6_RING_FAULT_REG_POSTING_READ(engine);
 158 }
 159
 160 void
 161 intel_gt_clear_error_registers(struct intel_gt *gt,
 162                                intel_engine_mask_t engine_mask)
 163 {
 164         struct drm_i915_private *i915 = gt->i915;
 165         struct intel_uncore *uncore = gt->uncore;
 166         u32 eir;
 167
 168         if (!IS_GEN(i915, 2))
 169                 clear_register(uncore, PGTBL_ER);
 170
 171         if (INTEL_GEN(i915) < 4)
 172                 clear_register(uncore, IPEIR(RENDER_RING_BASE));
 173         else
 174                 clear_register(uncore, IPEIR_I965);
 175
 176         clear_register(uncore, EIR);
 177         eir = intel_uncore_read(uncore, EIR);
 178         if (eir) {
 179                 /*
 180                  * some errors might have become stuck,
 181                  * mask them.
 182                  */
 183                 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
 184                 rmw_set(uncore, EMR, eir);
 185                 intel_uncore_write(uncore, GEN2_IIR,
 186                                    I915_MASTER_ERROR_INTERRUPT);
 187         }
 188
 189         if (INTEL_GEN(i915) >= 12) {
 190                 rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
 191                 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
 192         } else if (INTEL_GEN(i915) >= 8) {
 193                 rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
 194                 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
 195         } else if (INTEL_GEN(i915) >= 6) {
 196                 struct intel_engine_cs *engine;
 197                 enum intel_engine_id id;
 198
 199                 for_each_engine_masked(engine, gt, engine_mask, id)
 200                         gen8_clear_engine_error_register(engine);
 201         }
 202 }
 203
 204 static void gen6_check_faults(struct intel_gt *gt)
 205 {
 206         struct intel_engine_cs *engine;
 207         enum intel_engine_id id;
 208         u32 fault;
 209
 210         for_each_engine(engine, gt, id) {
 211                 fault = GEN6_RING_FAULT_REG_READ(engine);
 212                 if (fault & RING_FAULT_VALID) {
 213                         drm_dbg(&engine->i915->drm, "Unexpected fault\n"
 214                                 "\tAddr: 0x%08lx\n"
 215                                 "\tAddress space: %s\n"
 216                                 "\tSource ID: %d\n"
 217                                 "\tType: %d\n",
 218                                 fault & PAGE_MASK,
 219                                 fault & RING_FAULT_GTTSEL_MASK ?
 220                                 "GGTT" : "PPGTT",
 221                                 RING_FAULT_SRCID(fault),
 222                                 RING_FAULT_FAULT_TYPE(fault));
 223                 }
 224         }
 225 }
 226
 227 static void gen8_check_faults(struct intel_gt *gt)
 228 {
 229         struct intel_uncore *uncore = gt->uncore;
 230         i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
 231         u32 fault;
 232
 233         if (INTEL_GEN(gt->i915) >= 12) {
 234                 fault_reg = GEN12_RING_FAULT_REG;
 235                 fault_data0_reg = GEN12_FAULT_TLB_DATA0;
 236                 fault_data1_reg = GEN12_FAULT_TLB_DATA1;
 237         } else {
 238                 fault_reg = GEN8_RING_FAULT_REG;
 239                 fault_data0_reg = GEN8_FAULT_TLB_DATA0;
 240                 fault_data1_reg = GEN8_FAULT_TLB_DATA1;
 241         }
 242
 243         fault = intel_uncore_read(uncore, fault_reg);
 244         if (fault & RING_FAULT_VALID) {
 245                 u32 fault_data0, fault_data1;
 246                 u64 fault_addr;
 247
 248                 fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
 249                 fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
 250
 251                 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
 252                              ((u64)fault_data0 << 12);
 253
 254                 drm_dbg(&uncore->i915->drm, "Unexpected fault\n"
 255                         "\tAddr: 0x%08x_%08x\n"
 256                         "\tAddress space: %s\n"
 257                         "\tEngine ID: %d\n"
 258                         "\tSource ID: %d\n"
 259                         "\tType: %d\n",
 260                         upper_32_bits(fault_addr), lower_32_bits(fault_addr),
 261                         fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
 262                         GEN8_RING_FAULT_ENGINE_ID(fault),
 263                         RING_FAULT_SRCID(fault),
 264                         RING_FAULT_FAULT_TYPE(fault));
 265         }
 266 }
 267
 268 void intel_gt_check_and_clear_faults(struct intel_gt *gt)
 269 {
 270         struct drm_i915_private *i915 = gt->i915;
 271
 272         /* From GEN8 onwards we only have one 'All Engine Fault Register' */
 273         if (INTEL_GEN(i915) >= 8)
 274                 gen8_check_faults(gt);
 275         else if (INTEL_GEN(i915) >= 6)
 276                 gen6_check_faults(gt);
 277         else
 278                 return;
 279
 280         intel_gt_clear_error_registers(gt, ALL_ENGINES);
 281 }
 282
 283 void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
 284 {
 285         struct intel_uncore *uncore = gt->uncore;
 286         intel_wakeref_t wakeref;
 287
 288         /*
 289          * No actual flushing is required for the GTT write domain for reads
 290          * from the GTT domain. Writes to it "immediately" go to main memory
 291          * as far as we know, so there's no chipset flush. It also doesn't
 292          * land in the GPU render cache.
 293          *
 294          * However, we do have to enforce the order so that all writes through
 295          * the GTT land before any writes to the device, such as updates to
 296          * the GATT itself.
 297          *
 298          * We also have to wait a bit for the writes to land from the GTT.
 299          * An uncached read (i.e. mmio) seems to be ideal for the round-trip
 300          * timing. This issue has only been observed when switching quickly
 301          * between GTT writes and CPU reads from inside the kernel on recent hw,
 302          * and it appears to only affect discrete GTT blocks (i.e. on LLC
 303          * system agents we cannot reproduce this behaviour, until Cannonlake
 304          * that was!).
 305          */
 306
 307         wmb();
 308
 309         if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
 310                 return;
 311
 312         intel_gt_chipset_flush(gt);
 313
 314         with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
 315                 unsigned long flags;
 316
 317                 spin_lock_irqsave(&uncore->lock, flags);
 318                 intel_uncore_posting_read_fw(uncore,
 319                                              RING_HEAD(RENDER_RING_BASE));
 320                 spin_unlock_irqrestore(&uncore->lock, flags);
 321         }
 322 }
 323
 324 void intel_gt_chipset_flush(struct intel_gt *gt)
 325 {
 326         wmb();
 327         if (INTEL_GEN(gt->i915) < 6)
 328                 intel_gtt_chipset_flush();
 329 }
 330
 331 void intel_gt_driver_register(struct intel_gt *gt)
 332 {
 333         intel_rps_driver_register(&gt->rps);
 334
 335         debugfs_gt_register(gt);
 336 }
 337
 338 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
 339 {
 340         struct drm_i915_private *i915 = gt->i915;
 341         struct drm_i915_gem_object *obj;
 342         struct i915_vma *vma;
 343         int ret;
 344
 345         obj = i915_gem_object_create_stolen(i915, size);
 346         if (IS_ERR(obj))
 347                 obj = i915_gem_object_create_internal(i915, size);
 348         if (IS_ERR(obj)) {
 349                 DRM_ERROR("Failed to allocate scratch page\n");
 350                 return PTR_ERR(obj);
 351         }
 352
 353         vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
 354         if (IS_ERR(vma)) {
 355                 ret = PTR_ERR(vma);
 356                 goto err_unref;
 357         }
 358
 359         ret = i915_ggtt_pin(vma, 0, PIN_HIGH);
 360         if (ret)
 361                 goto err_unref;
 362
 363         gt->scratch = i915_vma_make_unshrinkable(vma);
 364
 365         return 0;
 366
 367 err_unref:
 368         i915_gem_object_put(obj);
 369         return ret;
 370 }
 371
 372 static void intel_gt_fini_scratch(struct intel_gt *gt)
 373 {
 374         i915_vma_unpin_and_release(&gt->scratch, 0);
 375 }
 376
 377 static struct i915_address_space *kernel_vm(struct intel_gt *gt)
 378 {
 379         if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
 380                 return &i915_ppgtt_create(gt)->vm;
 381         else
 382                 return i915_vm_get(&gt->ggtt->vm);
 383 }
 384
 385 static int __engines_record_defaults(struct intel_gt *gt)
 386 {
 387         struct i915_request *requests[I915_NUM_ENGINES] = {};
 388         struct intel_engine_cs *engine;
 389         enum intel_engine_id id;
 390         int err = 0;
 391
 392         /*
 393          * As we reset the gpu during very early sanitisation, the current
 394          * register state on the GPU should reflect its defaults values.
 395          * We load a context onto the hw (with restore-inhibit), then switch
 396          * over to a second context to save that default register state. We
 397          * can then prime every new context with that state so they all start
 398          * from the same default HW values.
 399          */
 400
 401         for_each_engine(engine, gt, id) {
 402                 struct intel_renderstate so;
 403                 struct intel_context *ce;
 404                 struct i915_request *rq;
 405
 406                 /* We must be able to switch to something! */
 407                 GEM_BUG_ON(!engine->kernel_context);
 408
 409                 err = intel_renderstate_init(&so, engine);
 410                 if (err)
 411                         goto out;
 412
 413                 ce = intel_context_create(engine);
 414                 if (IS_ERR(ce)) {
 415                         err = PTR_ERR(ce);
 416                         goto out;
 417                 }
 418
 419                 rq = intel_context_create_request(ce);
 420                 if (IS_ERR(rq)) {
 421                         err = PTR_ERR(rq);
 422                         intel_context_put(ce);
 423                         goto out;
 424                 }
 425
 426                 err = intel_engine_emit_ctx_wa(rq);
 427                 if (err)
 428                         goto err_rq;
 429
 430                 err = intel_renderstate_emit(&so, rq);
 431                 if (err)
 432                         goto err_rq;
 433
 434 err_rq:
 435                 requests[id] = i915_request_get(rq);
 436                 i915_request_add(rq);
 437                 intel_renderstate_fini(&so);
 438                 if (err)
 439                         goto out;
 440         }
 441
 442         /* Flush the default context image to memory, and enable powersaving. */
 443         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
 444                 err = -EIO;
 445                 goto out;
 446         }
 447
 448         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 449                 struct i915_request *rq;
 450                 struct file *state;
 451
 452                 rq = requests[id];
 453                 if (!rq)
 454                         continue;
 455
 456                 if (rq->fence.error) {
 457                         err = -EIO;
 458                         goto out;
 459                 }
 460
 461                 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
 462                 if (!rq->context->state)
 463                         continue;
 464
 465                 /* Keep a copy of the state's backing pages; free the obj */
 466                 state = shmem_create_from_object(rq->context->state->obj);
 467                 if (IS_ERR(state)) {
 468                         err = PTR_ERR(state);
 469                         goto out;
 470                 }
 471                 rq->engine->default_state = state;
 472         }
 473
 474 out:
 475         /*
 476          * If we have to abandon now, we expect the engines to be idle
 477          * and ready to be torn-down. The quickest way we can accomplish
 478          * this is by declaring ourselves wedged.
 479          */
 480         if (err)
 481                 intel_gt_set_wedged(gt);
 482
 483         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 484                 struct intel_context *ce;
 485                 struct i915_request *rq;
 486
 487                 rq = requests[id];
 488                 if (!rq)
 489                         continue;
 490
 491                 ce = rq->context;
 492                 i915_request_put(rq);
 493                 intel_context_put(ce);
 494         }
 495         return err;
 496 }
 497
 498 static int __engines_verify_workarounds(struct intel_gt *gt)
 499 {
 500         struct intel_engine_cs *engine;
 501         enum intel_engine_id id;
 502         int err = 0;
 503
 504         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 505                 return 0;
 506
 507         for_each_engine(engine, gt, id) {
 508                 if (intel_engine_verify_workarounds(engine, "load"))
 509                         err = -EIO;
 510         }
 511
 512         /* Flush and restore the kernel context for safety */
 513         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
 514                 err = -EIO;
 515
 516         return err;
 517 }
 518
 519 static void __intel_gt_disable(struct intel_gt *gt)
 520 {
 521         intel_gt_set_wedged_on_fini(gt);
 522
 523         intel_gt_suspend_prepare(gt);
 524         intel_gt_suspend_late(gt);
 525
 526         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 527 }
 528
 529 int intel_gt_init(struct intel_gt *gt)
 530 {
 531         int err;
 532
 533         err = i915_inject_probe_error(gt->i915, -ENODEV);
 534         if (err)
 535                 return err;
 536
 537         /*
 538          * This is just a security blanket to placate dragons.
 539          * On some systems, we very sporadically observe that the first TLBs
 540          * used by the CS may be stale, despite us poking the TLB reset. If
 541          * we hold the forcewake during initialisation these problems
 542          * just magically go away.
 543          */
 544         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 545
 546         intel_gt_init_clock_frequency(gt);
 547
 548         err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K);
 549         if (err)
 550                 goto out_fw;
 551
 552         intel_gt_pm_init(gt);
 553
 554         gt->vm = kernel_vm(gt);
 555         if (!gt->vm) {
 556                 err = -ENOMEM;
 557                 goto err_pm;
 558         }
 559
 560         err = intel_engines_init(gt);
 561         if (err)
 562                 goto err_engines;
 563
 564         err = intel_uc_init(&gt->uc);
 565         if (err)
 566                 goto err_engines;
 567
 568         err = intel_gt_resume(gt);
 569         if (err)
 570                 goto err_uc_init;
 571
 572         err = __engines_record_defaults(gt);
 573         if (err)
 574                 goto err_gt;
 575
 576         err = __engines_verify_workarounds(gt);
 577         if (err)
 578                 goto err_gt;
 579
 580         err = i915_inject_probe_error(gt->i915, -EIO);
 581         if (err)
 582                 goto err_gt;
 583
 584         goto out_fw;
 585 err_gt:
 586         __intel_gt_disable(gt);
 587         intel_uc_fini_hw(&gt->uc);
 588 err_uc_init:
 589         intel_uc_fini(&gt->uc);
 590 err_engines:
 591         intel_engines_release(gt);
 592         i915_vm_put(fetch_and_zero(&gt->vm));
 593 err_pm:
 594         intel_gt_pm_fini(gt);
 595         intel_gt_fini_scratch(gt);
 596 out_fw:
 597         if (err)
 598                 intel_gt_set_wedged_on_init(gt);
 599         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 600         return err;
 601 }
 602
 603 void intel_gt_driver_remove(struct intel_gt *gt)
 604 {
 605         __intel_gt_disable(gt);
 606
 607         intel_uc_driver_remove(&gt->uc);
 608
 609         intel_engines_release(gt);
 610 }
 611
 612 void intel_gt_driver_unregister(struct intel_gt *gt)
 613 {
 614         intel_rps_driver_unregister(&gt->rps);
 615
 616         /*
 617          * Upon unregistering the device to prevent any new users, cancel
 618          * all in-flight requests so that we can quickly unbind the active
 619          * resources.
 620          */
 621         intel_gt_set_wedged(gt);
 622 }
 623
 624 void intel_gt_driver_release(struct intel_gt *gt)
 625 {
 626         struct i915_address_space *vm;
 627         intel_wakeref_t wakeref;
 628
 629         /* Scrub all HW state upon release */
 630         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
 631                 __intel_gt_reset(gt, ALL_ENGINES);
 632
 633         vm = fetch_and_zero(&gt->vm);
 634         if (vm) /* FIXME being called twice on error paths :( */
 635                 i915_vm_put(vm);
 636
 637         intel_gt_pm_fini(gt);
 638         intel_gt_fini_scratch(gt);
 639         intel_gt_fini_buffer_pool(gt);
 640 }
 641
 642 void intel_gt_driver_late_release(struct intel_gt *gt)
 643 {
 644         /* We need to wait for inflight RCU frees to release their grip */
 645         rcu_barrier();
 646
 647         intel_uc_driver_late_release(&gt->uc);
 648         intel_gt_fini_requests(gt);
 649         intel_gt_fini_reset(gt);
 650         intel_gt_fini_timelines(gt);
 651         intel_engines_free(gt);
 652 }
 653
 654 void intel_gt_info_print(const struct intel_gt_info *info,
 655                          struct drm_printer *p)
 656 {
 657         drm_printf(p, "available engines: %x\n", info->engine_mask);
 658
 659         intel_sseu_dump(&info->sseu, p);
 660 }