drivers/gpu/drm/i915/gt/intel_gt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_managed.h>
   7 #include <drm/intel-gtt.h>
   8
   9 #include "gem/i915_gem_internal.h"
  10 #include "gem/i915_gem_lmem.h"
  11 #include "pxp/intel_pxp.h"
  12
  13 #include "i915_drv.h"
  14 #include "intel_context.h"
  15 #include "intel_engine_regs.h"
  16 #include "intel_ggtt_gmch.h"
  17 #include "intel_gt.h"
  18 #include "intel_gt_buffer_pool.h"
  19 #include "intel_gt_clock_utils.h"
  20 #include "intel_gt_debugfs.h"
  21 #include "intel_gt_mcr.h"
  22 #include "intel_gt_pm.h"
  23 #include "intel_gt_regs.h"
  24 #include "intel_gt_requests.h"
  25 #include "intel_migrate.h"
  26 #include "intel_mocs.h"
  27 #include "intel_pm.h"
  28 #include "intel_rc6.h"
  29 #include "intel_renderstate.h"
  30 #include "intel_rps.h"
  31 #include "intel_gt_sysfs.h"
  32 #include "intel_uncore.h"
  33 #include "shmem_utils.h"
  34
  35 static void __intel_gt_init_early(struct intel_gt *gt)
  36 {
  37         spin_lock_init(&gt->irq_lock);
  38
  39         mutex_init(&gt->tlb_invalidate_lock);
  40
  41         INIT_LIST_HEAD(&gt->closed_vma);
  42         spin_lock_init(&gt->closed_lock);
  43
  44         init_llist_head(&gt->watchdog.list);
  45         INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work);
  46
  47         intel_gt_init_buffer_pool(gt);
  48         intel_gt_init_reset(gt);
  49         intel_gt_init_requests(gt);
  50         intel_gt_init_timelines(gt);
  51         intel_gt_pm_init_early(gt);
  52
  53         intel_uc_init_early(&gt->uc);
  54         intel_rps_init_early(&gt->rps);
  55 }
  56
  57 /* Preliminary initialization of Tile 0 */
  58 void intel_root_gt_init_early(struct drm_i915_private *i915)
  59 {
  60         struct intel_gt *gt = to_gt(i915);
  61
  62         gt->i915 = i915;
  63         gt->uncore = &i915->uncore;
  64
  65         __intel_gt_init_early(gt);
  66 }
  67
  68 static int intel_gt_probe_lmem(struct intel_gt *gt)
  69 {
  70         struct drm_i915_private *i915 = gt->i915;
  71         unsigned int instance = gt->info.id;
  72         int id = INTEL_REGION_LMEM_0 + instance;
  73         struct intel_memory_region *mem;
  74         int err;
  75
  76         mem = intel_gt_setup_lmem(gt);
  77         if (IS_ERR(mem)) {
  78                 err = PTR_ERR(mem);
  79                 if (err == -ENODEV)
  80                         return 0;
  81
  82                 drm_err(&i915->drm,
  83                         "Failed to setup region(%d) type=%d\n",
  84                         err, INTEL_MEMORY_LOCAL);
  85                 return err;
  86         }
  87
  88         mem->id = id;
  89         mem->instance = instance;
  90
  91         intel_memory_region_set_name(mem, "local%u", mem->instance);
  92
  93         GEM_BUG_ON(!HAS_REGION(i915, id));
  94         GEM_BUG_ON(i915->mm.regions[id]);
  95         i915->mm.regions[id] = mem;
  96
  97         return 0;
  98 }
  99
 100 int intel_gt_assign_ggtt(struct intel_gt *gt)
 101 {
 102         gt->ggtt = drmm_kzalloc(&gt->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL);
 103
 104         return gt->ggtt ? 0 : -ENOMEM;
 105 }
 106
 107 int intel_gt_init_mmio(struct intel_gt *gt)
 108 {
 109         intel_gt_init_clock_frequency(gt);
 110
 111         intel_uc_init_mmio(&gt->uc);
 112         intel_sseu_info_init(gt);
 113         intel_gt_mcr_init(gt);
 114
 115         return intel_engines_init_mmio(gt);
 116 }
 117
 118 static void init_unused_ring(struct intel_gt *gt, u32 base)
 119 {
 120         struct intel_uncore *uncore = gt->uncore;
 121
 122         intel_uncore_write(uncore, RING_CTL(base), 0);
 123         intel_uncore_write(uncore, RING_HEAD(base), 0);
 124         intel_uncore_write(uncore, RING_TAIL(base), 0);
 125         intel_uncore_write(uncore, RING_START(base), 0);
 126 }
 127
 128 static void init_unused_rings(struct intel_gt *gt)
 129 {
 130         struct drm_i915_private *i915 = gt->i915;
 131
 132         if (IS_I830(i915)) {
 133                 init_unused_ring(gt, PRB1_BASE);
 134                 init_unused_ring(gt, SRB0_BASE);
 135                 init_unused_ring(gt, SRB1_BASE);
 136                 init_unused_ring(gt, SRB2_BASE);
 137                 init_unused_ring(gt, SRB3_BASE);
 138         } else if (GRAPHICS_VER(i915) == 2) {
 139                 init_unused_ring(gt, SRB0_BASE);
 140                 init_unused_ring(gt, SRB1_BASE);
 141         } else if (GRAPHICS_VER(i915) == 3) {
 142                 init_unused_ring(gt, PRB1_BASE);
 143                 init_unused_ring(gt, PRB2_BASE);
 144         }
 145 }
 146
 147 int intel_gt_init_hw(struct intel_gt *gt)
 148 {
 149         struct drm_i915_private *i915 = gt->i915;
 150         struct intel_uncore *uncore = gt->uncore;
 151         int ret;
 152
 153         gt->last_init_time = ktime_get();
 154
 155         /* Double layer security blanket, see i915_gem_init() */
 156         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 157
 158         if (HAS_EDRAM(i915) && GRAPHICS_VER(i915) < 9)
 159                 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
 160
 161         if (IS_HASWELL(i915))
 162                 intel_uncore_write(uncore,
 163                                    HSW_MI_PREDICATE_RESULT_2,
 164                                    IS_HSW_GT3(i915) ?
 165                                    LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 166
 167         /* Apply the GT workarounds... */
 168         intel_gt_apply_workarounds(gt);
 169         /* ...and determine whether they are sticking. */
 170         intel_gt_verify_workarounds(gt, "init");
 171
 172         intel_gt_init_swizzling(gt);
 173
 174         /*
 175          * At least 830 can leave some of the unused rings
 176          * "active" (ie. head != tail) after resume which
 177          * will prevent c3 entry. Makes sure all unused rings
 178          * are totally idle.
 179          */
 180         init_unused_rings(gt);
 181
 182         ret = i915_ppgtt_init_hw(gt);
 183         if (ret) {
 184                 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
 185                 goto out;
 186         }
 187
 188         /* We can't enable contexts until all firmware is loaded */
 189         ret = intel_uc_init_hw(&gt->uc);
 190         if (ret) {
 191                 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
 192                 goto out;
 193         }
 194
 195         intel_mocs_init(gt);
 196
 197 out:
 198         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 199         return ret;
 200 }
 201
 202 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
 203 {
 204         intel_uncore_rmw(uncore, reg, 0, set);
 205 }
 206
 207 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
 208 {
 209         intel_uncore_rmw(uncore, reg, clr, 0);
 210 }
 211
 212 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
 213 {
 214         intel_uncore_rmw(uncore, reg, 0, 0);
 215 }
 216
 217 static void gen6_clear_engine_error_register(struct intel_engine_cs *engine)
 218 {
 219         GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
 220         GEN6_RING_FAULT_REG_POSTING_READ(engine);
 221 }
 222
 223 void
 224 intel_gt_clear_error_registers(struct intel_gt *gt,
 225                                intel_engine_mask_t engine_mask)
 226 {
 227         struct drm_i915_private *i915 = gt->i915;
 228         struct intel_uncore *uncore = gt->uncore;
 229         u32 eir;
 230
 231         if (GRAPHICS_VER(i915) != 2)
 232                 clear_register(uncore, PGTBL_ER);
 233
 234         if (GRAPHICS_VER(i915) < 4)
 235                 clear_register(uncore, IPEIR(RENDER_RING_BASE));
 236         else
 237                 clear_register(uncore, IPEIR_I965);
 238
 239         clear_register(uncore, EIR);
 240         eir = intel_uncore_read(uncore, EIR);
 241         if (eir) {
 242                 /*
 243                  * some errors might have become stuck,
 244                  * mask them.
 245                  */
 246                 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
 247                 rmw_set(uncore, EMR, eir);
 248                 intel_uncore_write(uncore, GEN2_IIR,
 249                                    I915_MASTER_ERROR_INTERRUPT);
 250         }
 251
 252         if (GRAPHICS_VER(i915) >= 12) {
 253                 rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
 254                 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
 255         } else if (GRAPHICS_VER(i915) >= 8) {
 256                 rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
 257                 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
 258         } else if (GRAPHICS_VER(i915) >= 6) {
 259                 struct intel_engine_cs *engine;
 260                 enum intel_engine_id id;
 261
 262                 for_each_engine_masked(engine, gt, engine_mask, id)
 263                         gen6_clear_engine_error_register(engine);
 264         }
 265 }
 266
 267 static void gen6_check_faults(struct intel_gt *gt)
 268 {
 269         struct intel_engine_cs *engine;
 270         enum intel_engine_id id;
 271         u32 fault;
 272
 273         for_each_engine(engine, gt, id) {
 274                 fault = GEN6_RING_FAULT_REG_READ(engine);
 275                 if (fault & RING_FAULT_VALID) {
 276                         drm_dbg(&engine->i915->drm, "Unexpected fault\n"
 277                                 "\tAddr: 0x%08lx\n"
 278                                 "\tAddress space: %s\n"
 279                                 "\tSource ID: %d\n"
 280                                 "\tType: %d\n",
 281                                 fault & PAGE_MASK,
 282                                 fault & RING_FAULT_GTTSEL_MASK ?
 283                                 "GGTT" : "PPGTT",
 284                                 RING_FAULT_SRCID(fault),
 285                                 RING_FAULT_FAULT_TYPE(fault));
 286                 }
 287         }
 288 }
 289
 290 static void gen8_check_faults(struct intel_gt *gt)
 291 {
 292         struct intel_uncore *uncore = gt->uncore;
 293         i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
 294         u32 fault;
 295
 296         if (GRAPHICS_VER(gt->i915) >= 12) {
 297                 fault_reg = GEN12_RING_FAULT_REG;
 298                 fault_data0_reg = GEN12_FAULT_TLB_DATA0;
 299                 fault_data1_reg = GEN12_FAULT_TLB_DATA1;
 300         } else {
 301                 fault_reg = GEN8_RING_FAULT_REG;
 302                 fault_data0_reg = GEN8_FAULT_TLB_DATA0;
 303                 fault_data1_reg = GEN8_FAULT_TLB_DATA1;
 304         }
 305
 306         fault = intel_uncore_read(uncore, fault_reg);
 307         if (fault & RING_FAULT_VALID) {
 308                 u32 fault_data0, fault_data1;
 309                 u64 fault_addr;
 310
 311                 fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
 312                 fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
 313
 314                 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
 315                              ((u64)fault_data0 << 12);
 316
 317                 drm_dbg(&uncore->i915->drm, "Unexpected fault\n"
 318                         "\tAddr: 0x%08x_%08x\n"
 319                         "\tAddress space: %s\n"
 320                         "\tEngine ID: %d\n"
 321                         "\tSource ID: %d\n"
 322                         "\tType: %d\n",
 323                         upper_32_bits(fault_addr), lower_32_bits(fault_addr),
 324                         fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
 325                         GEN8_RING_FAULT_ENGINE_ID(fault),
 326                         RING_FAULT_SRCID(fault),
 327                         RING_FAULT_FAULT_TYPE(fault));
 328         }
 329 }
 330
 331 void intel_gt_check_and_clear_faults(struct intel_gt *gt)
 332 {
 333         struct drm_i915_private *i915 = gt->i915;
 334
 335         /* From GEN8 onwards we only have one 'All Engine Fault Register' */
 336         if (GRAPHICS_VER(i915) >= 8)
 337                 gen8_check_faults(gt);
 338         else if (GRAPHICS_VER(i915) >= 6)
 339                 gen6_check_faults(gt);
 340         else
 341                 return;
 342
 343         intel_gt_clear_error_registers(gt, ALL_ENGINES);
 344 }
 345
 346 void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
 347 {
 348         struct intel_uncore *uncore = gt->uncore;
 349         intel_wakeref_t wakeref;
 350
 351         /*
 352          * No actual flushing is required for the GTT write domain for reads
 353          * from the GTT domain. Writes to it "immediately" go to main memory
 354          * as far as we know, so there's no chipset flush. It also doesn't
 355          * land in the GPU render cache.
 356          *
 357          * However, we do have to enforce the order so that all writes through
 358          * the GTT land before any writes to the device, such as updates to
 359          * the GATT itself.
 360          *
 361          * We also have to wait a bit for the writes to land from the GTT.
 362          * An uncached read (i.e. mmio) seems to be ideal for the round-trip
 363          * timing. This issue has only been observed when switching quickly
 364          * between GTT writes and CPU reads from inside the kernel on recent hw,
 365          * and it appears to only affect discrete GTT blocks (i.e. on LLC
 366          * system agents we cannot reproduce this behaviour, until Cannonlake
 367          * that was!).
 368          */
 369
 370         wmb();
 371
 372         if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
 373                 return;
 374
 375         intel_gt_chipset_flush(gt);
 376
 377         with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
 378                 unsigned long flags;
 379
 380                 spin_lock_irqsave(&uncore->lock, flags);
 381                 intel_uncore_posting_read_fw(uncore,
 382                                              RING_HEAD(RENDER_RING_BASE));
 383                 spin_unlock_irqrestore(&uncore->lock, flags);
 384         }
 385 }
 386
 387 void intel_gt_chipset_flush(struct intel_gt *gt)
 388 {
 389         wmb();
 390         if (GRAPHICS_VER(gt->i915) < 6)
 391                 intel_ggtt_gmch_flush();
 392 }
 393
 394 void intel_gt_driver_register(struct intel_gt *gt)
 395 {
 396         intel_gsc_init(&gt->gsc, gt->i915);
 397
 398         intel_rps_driver_register(&gt->rps);
 399
 400         intel_gt_debugfs_register(gt);
 401         intel_gt_sysfs_register(gt);
 402 }
 403
 404 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
 405 {
 406         struct drm_i915_private *i915 = gt->i915;
 407         struct drm_i915_gem_object *obj;
 408         struct i915_vma *vma;
 409         int ret;
 410
 411         obj = i915_gem_object_create_lmem(i915, size,
 412                                           I915_BO_ALLOC_VOLATILE |
 413                                           I915_BO_ALLOC_GPU_ONLY);
 414         if (IS_ERR(obj))
 415                 obj = i915_gem_object_create_stolen(i915, size);
 416         if (IS_ERR(obj))
 417                 obj = i915_gem_object_create_internal(i915, size);
 418         if (IS_ERR(obj)) {
 419                 drm_err(&i915->drm, "Failed to allocate scratch page\n");
 420                 return PTR_ERR(obj);
 421         }
 422
 423         vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
 424         if (IS_ERR(vma)) {
 425                 ret = PTR_ERR(vma);
 426                 goto err_unref;
 427         }
 428
 429         ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
 430         if (ret)
 431                 goto err_unref;
 432
 433         gt->scratch = i915_vma_make_unshrinkable(vma);
 434
 435         return 0;
 436
 437 err_unref:
 438         i915_gem_object_put(obj);
 439         return ret;
 440 }
 441
 442 static void intel_gt_fini_scratch(struct intel_gt *gt)
 443 {
 444         i915_vma_unpin_and_release(&gt->scratch, 0);
 445 }
 446
 447 static struct i915_address_space *kernel_vm(struct intel_gt *gt)
 448 {
 449         if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
 450                 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm;
 451         else
 452                 return i915_vm_get(&gt->ggtt->vm);
 453 }
 454
 455 static int __engines_record_defaults(struct intel_gt *gt)
 456 {
 457         struct i915_request *requests[I915_NUM_ENGINES] = {};
 458         struct intel_engine_cs *engine;
 459         enum intel_engine_id id;
 460         int err = 0;
 461
 462         /*
 463          * As we reset the gpu during very early sanitisation, the current
 464          * register state on the GPU should reflect its defaults values.
 465          * We load a context onto the hw (with restore-inhibit), then switch
 466          * over to a second context to save that default register state. We
 467          * can then prime every new context with that state so they all start
 468          * from the same default HW values.
 469          */
 470
 471         for_each_engine(engine, gt, id) {
 472                 struct intel_renderstate so;
 473                 struct intel_context *ce;
 474                 struct i915_request *rq;
 475
 476                 /* We must be able to switch to something! */
 477                 GEM_BUG_ON(!engine->kernel_context);
 478
 479                 ce = intel_context_create(engine);
 480                 if (IS_ERR(ce)) {
 481                         err = PTR_ERR(ce);
 482                         goto out;
 483                 }
 484
 485                 err = intel_renderstate_init(&so, ce);
 486                 if (err)
 487                         goto err;
 488
 489                 rq = i915_request_create(ce);
 490                 if (IS_ERR(rq)) {
 491                         err = PTR_ERR(rq);
 492                         goto err_fini;
 493                 }
 494
 495                 err = intel_engine_emit_ctx_wa(rq);
 496                 if (err)
 497                         goto err_rq;
 498
 499                 err = intel_renderstate_emit(&so, rq);
 500                 if (err)
 501                         goto err_rq;
 502
 503 err_rq:
 504                 requests[id] = i915_request_get(rq);
 505                 i915_request_add(rq);
 506 err_fini:
 507                 intel_renderstate_fini(&so, ce);
 508 err:
 509                 if (err) {
 510                         intel_context_put(ce);
 511                         goto out;
 512                 }
 513         }
 514
 515         /* Flush the default context image to memory, and enable powersaving. */
 516         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
 517                 err = -EIO;
 518                 goto out;
 519         }
 520
 521         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 522                 struct i915_request *rq;
 523                 struct file *state;
 524
 525                 rq = requests[id];
 526                 if (!rq)
 527                         continue;
 528
 529                 if (rq->fence.error) {
 530                         err = -EIO;
 531                         goto out;
 532                 }
 533
 534                 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
 535                 if (!rq->context->state)
 536                         continue;
 537
 538                 /* Keep a copy of the state's backing pages; free the obj */
 539                 state = shmem_create_from_object(rq->context->state->obj);
 540                 if (IS_ERR(state)) {
 541                         err = PTR_ERR(state);
 542                         goto out;
 543                 }
 544                 rq->engine->default_state = state;
 545         }
 546
 547 out:
 548         /*
 549          * If we have to abandon now, we expect the engines to be idle
 550          * and ready to be torn-down. The quickest way we can accomplish
 551          * this is by declaring ourselves wedged.
 552          */
 553         if (err)
 554                 intel_gt_set_wedged(gt);
 555
 556         for (id = 0; id < ARRAY_SIZE(requests); id++) {
 557                 struct intel_context *ce;
 558                 struct i915_request *rq;
 559
 560                 rq = requests[id];
 561                 if (!rq)
 562                         continue;
 563
 564                 ce = rq->context;
 565                 i915_request_put(rq);
 566                 intel_context_put(ce);
 567         }
 568         return err;
 569 }
 570
 571 static int __engines_verify_workarounds(struct intel_gt *gt)
 572 {
 573         struct intel_engine_cs *engine;
 574         enum intel_engine_id id;
 575         int err = 0;
 576
 577         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 578                 return 0;
 579
 580         for_each_engine(engine, gt, id) {
 581                 if (intel_engine_verify_workarounds(engine, "load"))
 582                         err = -EIO;
 583         }
 584
 585         /* Flush and restore the kernel context for safety */
 586         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
 587                 err = -EIO;
 588
 589         return err;
 590 }
 591
 592 static void __intel_gt_disable(struct intel_gt *gt)
 593 {
 594         intel_gt_set_wedged_on_fini(gt);
 595
 596         intel_gt_suspend_prepare(gt);
 597         intel_gt_suspend_late(gt);
 598
 599         GEM_BUG_ON(intel_gt_pm_is_awake(gt));
 600 }
 601
 602 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
 603 {
 604         long remaining_timeout;
 605
 606         /* If the device is asleep, we have no requests outstanding */
 607         if (!intel_gt_pm_is_awake(gt))
 608                 return 0;
 609
 610         while ((timeout = intel_gt_retire_requests_timeout(gt, timeout,
 611                                                            &remaining_timeout)) > 0) {
 612                 cond_resched();
 613                 if (signal_pending(current))
 614                         return -EINTR;
 615         }
 616
 617         return timeout ? timeout : intel_uc_wait_for_idle(&gt->uc,
 618                                                           remaining_timeout);
 619 }
 620
 621 int intel_gt_init(struct intel_gt *gt)
 622 {
 623         int err;
 624
 625         err = i915_inject_probe_error(gt->i915, -ENODEV);
 626         if (err)
 627                 return err;
 628
 629         intel_gt_init_workarounds(gt);
 630
 631         /*
 632          * This is just a security blanket to placate dragons.
 633          * On some systems, we very sporadically observe that the first TLBs
 634          * used by the CS may be stale, despite us poking the TLB reset. If
 635          * we hold the forcewake during initialisation these problems
 636          * just magically go away.
 637          */
 638         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 639
 640         err = intel_gt_init_scratch(gt,
 641                                     GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K);
 642         if (err)
 643                 goto out_fw;
 644
 645         intel_gt_pm_init(gt);
 646
 647         gt->vm = kernel_vm(gt);
 648         if (!gt->vm) {
 649                 err = -ENOMEM;
 650                 goto err_pm;
 651         }
 652
 653         intel_set_mocs_index(gt);
 654
 655         err = intel_engines_init(gt);
 656         if (err)
 657                 goto err_engines;
 658
 659         err = intel_uc_init(&gt->uc);
 660         if (err)
 661                 goto err_engines;
 662
 663         err = intel_gt_resume(gt);
 664         if (err)
 665                 goto err_uc_init;
 666
 667         err = intel_gt_init_hwconfig(gt);
 668         if (err)
 669                 drm_err(&gt->i915->drm, "Failed to retrieve hwconfig table: %pe\n",
 670                         ERR_PTR(err));
 671
 672         err = __engines_record_defaults(gt);
 673         if (err)
 674                 goto err_gt;
 675
 676         err = __engines_verify_workarounds(gt);
 677         if (err)
 678                 goto err_gt;
 679
 680         intel_uc_init_late(&gt->uc);
 681
 682         err = i915_inject_probe_error(gt->i915, -EIO);
 683         if (err)
 684                 goto err_gt;
 685
 686         intel_migrate_init(&gt->migrate, gt);
 687
 688         intel_pxp_init(&gt->pxp);
 689
 690         goto out_fw;
 691 err_gt:
 692         __intel_gt_disable(gt);
 693         intel_uc_fini_hw(&gt->uc);
 694 err_uc_init:
 695         intel_uc_fini(&gt->uc);
 696 err_engines:
 697         intel_engines_release(gt);
 698         i915_vm_put(fetch_and_zero(&gt->vm));
 699 err_pm:
 700         intel_gt_pm_fini(gt);
 701         intel_gt_fini_scratch(gt);
 702 out_fw:
 703         if (err)
 704                 intel_gt_set_wedged_on_init(gt);
 705         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 706         return err;
 707 }
 708
 709 void intel_gt_driver_remove(struct intel_gt *gt)
 710 {
 711         __intel_gt_disable(gt);
 712
 713         intel_migrate_fini(&gt->migrate);
 714         intel_uc_driver_remove(&gt->uc);
 715
 716         intel_engines_release(gt);
 717
 718         intel_gt_flush_buffer_pool(gt);
 719 }
 720
 721 void intel_gt_driver_unregister(struct intel_gt *gt)
 722 {
 723         intel_wakeref_t wakeref;
 724
 725         intel_gt_sysfs_unregister(gt);
 726         intel_rps_driver_unregister(&gt->rps);
 727         intel_gsc_fini(&gt->gsc);
 728
 729         intel_pxp_fini(&gt->pxp);
 730
 731         /*
 732          * Upon unregistering the device to prevent any new users, cancel
 733          * all in-flight requests so that we can quickly unbind the active
 734          * resources.
 735          */
 736         intel_gt_set_wedged_on_fini(gt);
 737
 738         /* Scrub all HW state upon release */
 739         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
 740                 __intel_gt_reset(gt, ALL_ENGINES);
 741 }
 742
 743 void intel_gt_driver_release(struct intel_gt *gt)
 744 {
 745         struct i915_address_space *vm;
 746
 747         vm = fetch_and_zero(&gt->vm);
 748         if (vm) /* FIXME being called twice on error paths :( */
 749                 i915_vm_put(vm);
 750
 751         intel_wa_list_free(&gt->wa_list);
 752         intel_gt_pm_fini(gt);
 753         intel_gt_fini_scratch(gt);
 754         intel_gt_fini_buffer_pool(gt);
 755         intel_gt_fini_hwconfig(gt);
 756 }
 757
 758 void intel_gt_driver_late_release_all(struct drm_i915_private *i915)
 759 {
 760         struct intel_gt *gt;
 761         unsigned int id;
 762
 763         /* We need to wait for inflight RCU frees to release their grip */
 764         rcu_barrier();
 765
 766         for_each_gt(gt, i915, id) {
 767                 intel_uc_driver_late_release(&gt->uc);
 768                 intel_gt_fini_requests(gt);
 769                 intel_gt_fini_reset(gt);
 770                 intel_gt_fini_timelines(gt);
 771                 intel_engines_free(gt);
 772         }
 773 }
 774
 775 static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)
 776 {
 777         int ret;
 778
 779         if (!gt_is_root(gt)) {
 780                 struct intel_uncore_mmio_debug *mmio_debug;
 781                 struct intel_uncore *uncore;
 782
 783                 uncore = kzalloc(sizeof(*uncore), GFP_KERNEL);
 784                 if (!uncore)
 785                         return -ENOMEM;
 786
 787                 mmio_debug = kzalloc(sizeof(*mmio_debug), GFP_KERNEL);
 788                 if (!mmio_debug) {
 789                         kfree(uncore);
 790                         return -ENOMEM;
 791                 }
 792
 793                 gt->uncore = uncore;
 794                 gt->uncore->debug = mmio_debug;
 795
 796                 __intel_gt_init_early(gt);
 797         }
 798
 799         intel_uncore_init_early(gt->uncore, gt);
 800
 801         ret = intel_uncore_setup_mmio(gt->uncore, phys_addr);
 802         if (ret)
 803                 return ret;
 804
 805         gt->phys_addr = phys_addr;
 806
 807         return 0;
 808 }
 809
 810 static void
 811 intel_gt_tile_cleanup(struct intel_gt *gt)
 812 {
 813         intel_uncore_cleanup_mmio(gt->uncore);
 814
 815         if (!gt_is_root(gt)) {
 816                 kfree(gt->uncore->debug);
 817                 kfree(gt->uncore);
 818                 kfree(gt);
 819         }
 820 }
 821
 822 int intel_gt_probe_all(struct drm_i915_private *i915)
 823 {
 824         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 825         struct intel_gt *gt = &i915->gt0;
 826         phys_addr_t phys_addr;
 827         unsigned int mmio_bar;
 828         int ret;
 829
 830         mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0;
 831         phys_addr = pci_resource_start(pdev, mmio_bar);
 832
 833         /*
 834          * We always have at least one primary GT on any device
 835          * and it has been already initialized early during probe
 836          * in i915_driver_probe()
 837          */
 838         ret = intel_gt_tile_setup(gt, phys_addr);
 839         if (ret)
 840                 return ret;
 841
 842         i915->gt[0] = gt;
 843
 844         /* TODO: add more tiles */
 845         return 0;
 846 }
 847
 848 int intel_gt_tiles_init(struct drm_i915_private *i915)
 849 {
 850         struct intel_gt *gt;
 851         unsigned int id;
 852         int ret;
 853
 854         for_each_gt(gt, i915, id) {
 855                 ret = intel_gt_probe_lmem(gt);
 856                 if (ret)
 857                         return ret;
 858         }
 859
 860         return 0;
 861 }
 862
 863 void intel_gt_release_all(struct drm_i915_private *i915)
 864 {
 865         struct intel_gt *gt;
 866         unsigned int id;
 867
 868         for_each_gt(gt, i915, id) {
 869                 intel_gt_tile_cleanup(gt);
 870                 i915->gt[id] = NULL;
 871         }
 872 }
 873
 874 void intel_gt_info_print(const struct intel_gt_info *info,
 875                          struct drm_printer *p)
 876 {
 877         drm_printf(p, "available engines: %x\n", info->engine_mask);
 878
 879         intel_sseu_dump(&info->sseu, p);
 880 }
 881
 882 struct reg_and_bit {
 883         i915_reg_t reg;
 884         u32 bit;
 885 };
 886
 887 static struct reg_and_bit
 888 get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
 889                 const i915_reg_t *regs, const unsigned int num)
 890 {
 891         const unsigned int class = engine->class;
 892         struct reg_and_bit rb = { };
 893
 894         if (drm_WARN_ON_ONCE(&engine->i915->drm,
 895                              class >= num || !regs[class].reg))
 896                 return rb;
 897
 898         rb.reg = regs[class];
 899         if (gen8 && class == VIDEO_DECODE_CLASS)
 900                 rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */
 901         else
 902                 rb.bit = engine->instance;
 903
 904         rb.bit = BIT(rb.bit);
 905
 906         return rb;
 907 }
 908
 909 void intel_gt_invalidate_tlbs(struct intel_gt *gt)
 910 {
 911         static const i915_reg_t gen8_regs[] = {
 912                 [RENDER_CLASS]                  = GEN8_RTCR,
 913                 [VIDEO_DECODE_CLASS]            = GEN8_M1TCR, /* , GEN8_M2TCR */
 914                 [VIDEO_ENHANCEMENT_CLASS]       = GEN8_VTCR,
 915                 [COPY_ENGINE_CLASS]             = GEN8_BTCR,
 916         };
 917         static const i915_reg_t gen12_regs[] = {
 918                 [RENDER_CLASS]                  = GEN12_GFX_TLB_INV_CR,
 919                 [VIDEO_DECODE_CLASS]            = GEN12_VD_TLB_INV_CR,
 920                 [VIDEO_ENHANCEMENT_CLASS]       = GEN12_VE_TLB_INV_CR,
 921                 [COPY_ENGINE_CLASS]             = GEN12_BLT_TLB_INV_CR,
 922                 [COMPUTE_CLASS]                 = GEN12_COMPCTX_TLB_INV_CR,
 923         };
 924         struct drm_i915_private *i915 = gt->i915;
 925         struct intel_uncore *uncore = gt->uncore;
 926         struct intel_engine_cs *engine;
 927         enum intel_engine_id id;
 928         const i915_reg_t *regs;
 929         unsigned int num = 0;
 930
 931         if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
 932                 return;
 933
 934         if (GRAPHICS_VER(i915) == 12) {
 935                 regs = gen12_regs;
 936                 num = ARRAY_SIZE(gen12_regs);
 937         } else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) {
 938                 regs = gen8_regs;
 939                 num = ARRAY_SIZE(gen8_regs);
 940         } else if (GRAPHICS_VER(i915) < 8) {
 941                 return;
 942         }
 943
 944         if (drm_WARN_ONCE(&i915->drm, !num,
 945                           "Platform does not implement TLB invalidation!"))
 946                 return;
 947
 948         GEM_TRACE("\n");
 949
 950         assert_rpm_wakelock_held(&i915->runtime_pm);
 951
 952         mutex_lock(&gt->tlb_invalidate_lock);
 953         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 954
 955         spin_lock_irq(&uncore->lock); /* serialise invalidate with GT reset */
 956
 957         for_each_engine(engine, gt, id) {
 958                 struct reg_and_bit rb;
 959
 960                 rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
 961                 if (!i915_mmio_reg_offset(rb.reg))
 962                         continue;
 963
 964                 intel_uncore_write_fw(uncore, rb.reg, rb.bit);
 965         }
 966
 967         spin_unlock_irq(&uncore->lock);
 968
 969         for_each_engine(engine, gt, id) {
 970                 /*
 971                  * HW architecture suggest typical invalidation time at 40us,
 972                  * with pessimistic cases up to 100us and a recommendation to
 973                  * cap at 1ms. We go a bit higher just in case.
 974                  */
 975                 const unsigned int timeout_us = 100;
 976                 const unsigned int timeout_ms = 4;
 977                 struct reg_and_bit rb;
 978
 979                 rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
 980                 if (!i915_mmio_reg_offset(rb.reg))
 981                         continue;
 982
 983                 if (__intel_wait_for_register_fw(uncore,
 984                                                  rb.reg, rb.bit, 0,
 985                                                  timeout_us, timeout_ms,
 986                                                  NULL))
 987                         drm_err_ratelimited(&gt->i915->drm,
 988                                             "%s TLB invalidation did not complete in %ums!\n",
 989                                             engine->name, timeout_ms);
 990         }
 991
 992         /*
 993          * Use delayed put since a) we mostly expect a flurry of TLB
 994          * invalidations so it is good to avoid paying the forcewake cost and
 995          * b) it works around a bug in Icelake which cannot cope with too rapid
 996          * transitions.
 997          */
 998         intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL);
 999         mutex_unlock(&gt->tlb_invalidate_lock);
1000 }